1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //==-----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines an instruction selector for the AMDGPU target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUArgumentUsageInfo.h" 16 #include "AMDGPUISelLowering.h" // For AMDGPUISD 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUPerfHintAnalysis.h" 19 #include "AMDGPUSubtarget.h" 20 #include "AMDGPUTargetMachine.h" 21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 22 #include "SIDefines.h" 23 #include "SIISelLowering.h" 24 #include "SIInstrInfo.h" 25 #include "SIMachineFunctionInfo.h" 26 #include "SIRegisterInfo.h" 27 #include "llvm/ADT/APInt.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 31 #include "llvm/Analysis/ValueTracking.h" 32 #include "llvm/CodeGen/FunctionLoweringInfo.h" 33 #include "llvm/CodeGen/ISDOpcodes.h" 34 #include "llvm/CodeGen/MachineFunction.h" 35 #include "llvm/CodeGen/MachineRegisterInfo.h" 36 #include "llvm/CodeGen/SelectionDAG.h" 37 #include "llvm/CodeGen/SelectionDAGISel.h" 38 #include "llvm/CodeGen/SelectionDAGNodes.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/IR/BasicBlock.h" 41 #include "llvm/InitializePasses.h" 42 #ifdef EXPENSIVE_CHECKS 43 #include "llvm/IR/Dominators.h" 44 #endif 45 #include "llvm/IR/Instruction.h" 46 #include "llvm/MC/MCInstrDesc.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/CodeGen.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MachineValueType.h" 51 #include "llvm/Support/MathExtras.h" 52 #include <cassert> 53 #include <cstdint> 54 #include <new> 55 #include <vector> 56 57 #define DEBUG_TYPE "isel" 58 59 using namespace llvm; 60 61 namespace llvm { 62 63 class R600InstrInfo; 64 65 } // end namespace llvm 66 67 //===----------------------------------------------------------------------===// 68 // Instruction Selector Implementation 69 //===----------------------------------------------------------------------===// 70 71 namespace { 72 73 static bool isNullConstantOrUndef(SDValue V) { 74 if (V.isUndef()) 75 return true; 76 77 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 78 return Const != nullptr && Const->isNullValue(); 79 } 80 81 static bool getConstantValue(SDValue N, uint32_t &Out) { 82 // This is only used for packed vectors, where ussing 0 for undef should 83 // always be good. 84 if (N.isUndef()) { 85 Out = 0; 86 return true; 87 } 88 89 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 90 Out = C->getAPIntValue().getSExtValue(); 91 return true; 92 } 93 94 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 95 Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); 96 return true; 97 } 98 99 return false; 100 } 101 102 // TODO: Handle undef as zero 103 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG, 104 bool Negate = false) { 105 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); 106 uint32_t LHSVal, RHSVal; 107 if (getConstantValue(N->getOperand(0), LHSVal) && 108 getConstantValue(N->getOperand(1), RHSVal)) { 109 SDLoc SL(N); 110 uint32_t K = Negate ? 111 (-LHSVal & 0xffff) | (-RHSVal << 16) : 112 (LHSVal & 0xffff) | (RHSVal << 16); 113 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0), 114 DAG.getTargetConstant(K, SL, MVT::i32)); 115 } 116 117 return nullptr; 118 } 119 120 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) { 121 return packConstantV2I16(N, DAG, true); 122 } 123 124 /// AMDGPU specific code to select AMDGPU machine instructions for 125 /// SelectionDAG operations. 126 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 127 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 128 // make the right decision when generating code for different targets. 129 const GCNSubtarget *Subtarget; 130 131 // Default FP mode for the current function. 132 AMDGPU::SIModeRegisterDefaults Mode; 133 134 bool EnableLateStructurizeCFG; 135 136 public: 137 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 138 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 139 : SelectionDAGISel(*TM, OptLevel) { 140 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 141 } 142 ~AMDGPUDAGToDAGISel() override = default; 143 144 void getAnalysisUsage(AnalysisUsage &AU) const override { 145 AU.addRequired<AMDGPUArgumentUsageInfo>(); 146 AU.addRequired<LegacyDivergenceAnalysis>(); 147 #ifdef EXPENSIVE_CHECKS 148 AU.addRequired<DominatorTreeWrapperPass>(); 149 AU.addRequired<LoopInfoWrapperPass>(); 150 #endif 151 SelectionDAGISel::getAnalysisUsage(AU); 152 } 153 154 bool matchLoadD16FromBuildVector(SDNode *N) const; 155 156 bool runOnMachineFunction(MachineFunction &MF) override; 157 void PreprocessISelDAG() override; 158 void Select(SDNode *N) override; 159 StringRef getPassName() const override; 160 void PostprocessISelDAG() override; 161 162 protected: 163 void SelectBuildVector(SDNode *N, unsigned RegClassID); 164 165 private: 166 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 167 bool isNoNanSrc(SDValue N) const; 168 bool isInlineImmediate(const SDNode *N, bool Negated = false) const; 169 bool isNegInlineImmediate(const SDNode *N) const { 170 return isInlineImmediate(N, true); 171 } 172 173 bool isInlineImmediate16(int64_t Imm) const { 174 return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); 175 } 176 177 bool isInlineImmediate32(int64_t Imm) const { 178 return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); 179 } 180 181 bool isInlineImmediate64(int64_t Imm) const { 182 return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); 183 } 184 185 bool isInlineImmediate(const APFloat &Imm) const { 186 return Subtarget->getInstrInfo()->isInlineConstant(Imm); 187 } 188 189 bool isVGPRImm(const SDNode *N) const; 190 bool isUniformLoad(const SDNode *N) const; 191 bool isUniformBr(const SDNode *N) const; 192 193 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; 194 195 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; 196 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; 197 SDNode *glueCopyToM0LDSInit(SDNode *N) const; 198 199 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 200 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 201 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 202 bool isDSOffsetLegal(SDValue Base, unsigned Offset, 203 unsigned OffsetBits) const; 204 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 205 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 206 SDValue &Offset1) const; 207 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 208 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 209 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC, 210 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 211 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 212 SDValue &SOffset, SDValue &Offset, SDValue &GLC, 213 SDValue &SLC, SDValue &TFE, SDValue &DLC, 214 SDValue &SWZ) const; 215 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 216 SDValue &VAddr, SDValue &SOffset, SDValue &Offset, 217 SDValue &SLC) const; 218 bool SelectMUBUFScratchOffen(SDNode *Parent, 219 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 220 SDValue &SOffset, SDValue &ImmOffset) const; 221 bool SelectMUBUFScratchOffset(SDNode *Parent, 222 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 223 SDValue &Offset) const; 224 225 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset, 226 SDValue &Offset, SDValue &GLC, SDValue &SLC, 227 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 228 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 229 SDValue &Offset, SDValue &SLC) const; 230 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 231 SDValue &Offset) const; 232 233 template <bool IsSigned> 234 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 235 SDValue &Offset, SDValue &SLC) const; 236 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr, 237 SDValue &Offset, SDValue &SLC) const; 238 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr, 239 SDValue &Offset, SDValue &SLC) const; 240 241 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 242 bool &Imm) const; 243 SDValue Expand32BitAddress(SDValue Addr) const; 244 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 245 bool &Imm) const; 246 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 247 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 248 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 249 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 250 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 251 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 252 253 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 254 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const; 255 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 256 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 257 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 258 SDValue &Clamp, SDValue &Omod) const; 259 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 260 SDValue &Clamp, SDValue &Omod) const; 261 262 bool SelectVOP3OMods(SDValue In, SDValue &Src, 263 SDValue &Clamp, SDValue &Omod) const; 264 265 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 266 267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 268 269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 271 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 272 273 SDValue getHi16Elt(SDValue In) const; 274 275 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; 276 277 void SelectADD_SUB_I64(SDNode *N); 278 void SelectAddcSubb(SDNode *N); 279 void SelectUADDO_USUBO(SDNode *N); 280 void SelectDIV_SCALE(SDNode *N); 281 void SelectDIV_FMAS(SDNode *N); 282 void SelectMAD_64_32(SDNode *N); 283 void SelectFMA_W_CHAIN(SDNode *N); 284 void SelectFMUL_W_CHAIN(SDNode *N); 285 286 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 287 uint32_t Offset, uint32_t Width); 288 void SelectS_BFEFromShifts(SDNode *N); 289 void SelectS_BFE(SDNode *N); 290 bool isCBranchSCC(const SDNode *N) const; 291 void SelectBRCOND(SDNode *N); 292 void SelectFMAD_FMA(SDNode *N); 293 void SelectATOMIC_CMP_SWAP(SDNode *N); 294 void SelectDSAppendConsume(SDNode *N, unsigned IntrID); 295 void SelectDS_GWS(SDNode *N, unsigned IntrID); 296 void SelectInterpP1F16(SDNode *N); 297 void SelectINTRINSIC_W_CHAIN(SDNode *N); 298 void SelectINTRINSIC_WO_CHAIN(SDNode *N); 299 void SelectINTRINSIC_VOID(SDNode *N); 300 301 protected: 302 // Include the pieces autogenerated from the target description. 303 #include "AMDGPUGenDAGISel.inc" 304 }; 305 306 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 307 const R600Subtarget *Subtarget; 308 309 bool isConstantLoad(const MemSDNode *N, int cbID) const; 310 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 311 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 312 SDValue& Offset); 313 public: 314 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 315 AMDGPUDAGToDAGISel(TM, OptLevel) {} 316 317 void Select(SDNode *N) override; 318 319 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 320 SDValue &Offset) override; 321 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 322 SDValue &Offset) override; 323 324 bool runOnMachineFunction(MachineFunction &MF) override; 325 326 void PreprocessISelDAG() override {} 327 328 protected: 329 // Include the pieces autogenerated from the target description. 330 #include "R600GenDAGISel.inc" 331 }; 332 333 static SDValue stripBitcast(SDValue Val) { 334 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 335 } 336 337 // Figure out if this is really an extract of the high 16-bits of a dword. 338 static bool isExtractHiElt(SDValue In, SDValue &Out) { 339 In = stripBitcast(In); 340 if (In.getOpcode() != ISD::TRUNCATE) 341 return false; 342 343 SDValue Srl = In.getOperand(0); 344 if (Srl.getOpcode() == ISD::SRL) { 345 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 346 if (ShiftAmt->getZExtValue() == 16) { 347 Out = stripBitcast(Srl.getOperand(0)); 348 return true; 349 } 350 } 351 } 352 353 return false; 354 } 355 356 // Look through operations that obscure just looking at the low 16-bits of the 357 // same register. 358 static SDValue stripExtractLoElt(SDValue In) { 359 if (In.getOpcode() == ISD::TRUNCATE) { 360 SDValue Src = In.getOperand(0); 361 if (Src.getValueType().getSizeInBits() == 32) 362 return stripBitcast(Src); 363 } 364 365 return In; 366 } 367 368 } // end anonymous namespace 369 370 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel", 371 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 372 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 373 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 374 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) 375 #ifdef EXPENSIVE_CHECKS 376 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 377 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 378 #endif 379 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel", 380 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 381 382 /// This pass converts a legalized DAG into a AMDGPU-specific 383 // DAG, ready for instruction scheduling. 384 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 385 CodeGenOpt::Level OptLevel) { 386 return new AMDGPUDAGToDAGISel(TM, OptLevel); 387 } 388 389 /// This pass converts a legalized DAG into a R600-specific 390 // DAG, ready for instruction scheduling. 391 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 392 CodeGenOpt::Level OptLevel) { 393 return new R600DAGToDAGISel(TM, OptLevel); 394 } 395 396 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 397 #ifdef EXPENSIVE_CHECKS 398 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 399 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 400 for (auto &L : LI->getLoopsInPreorder()) { 401 assert(L->isLCSSAForm(DT)); 402 } 403 #endif 404 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 405 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction(), *Subtarget); 406 return SelectionDAGISel::runOnMachineFunction(MF); 407 } 408 409 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const { 410 assert(Subtarget->d16PreservesUnusedBits()); 411 MVT VT = N->getValueType(0).getSimpleVT(); 412 if (VT != MVT::v2i16 && VT != MVT::v2f16) 413 return false; 414 415 SDValue Lo = N->getOperand(0); 416 SDValue Hi = N->getOperand(1); 417 418 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi)); 419 420 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo 421 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo 422 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo 423 424 // Need to check for possible indirect dependencies on the other half of the 425 // vector to avoid introducing a cycle. 426 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) { 427 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 428 429 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo); 430 SDValue Ops[] = { 431 LdHi->getChain(), LdHi->getBasePtr(), TiedIn 432 }; 433 434 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI; 435 if (LdHi->getMemoryVT() == MVT::i8) { 436 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ? 437 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8; 438 } else { 439 assert(LdHi->getMemoryVT() == MVT::i16); 440 } 441 442 SDValue NewLoadHi = 443 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList, 444 Ops, LdHi->getMemoryVT(), 445 LdHi->getMemOperand()); 446 447 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi); 448 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1)); 449 return true; 450 } 451 452 // build_vector (load ptr), hi -> load_d16_lo ptr, hi 453 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi 454 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi 455 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo)); 456 if (LdLo && Lo.hasOneUse()) { 457 SDValue TiedIn = getHi16Elt(Hi); 458 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode())) 459 return false; 460 461 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 462 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO; 463 if (LdLo->getMemoryVT() == MVT::i8) { 464 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ? 465 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8; 466 } else { 467 assert(LdLo->getMemoryVT() == MVT::i16); 468 } 469 470 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn); 471 472 SDValue Ops[] = { 473 LdLo->getChain(), LdLo->getBasePtr(), TiedIn 474 }; 475 476 SDValue NewLoadLo = 477 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList, 478 Ops, LdLo->getMemoryVT(), 479 LdLo->getMemOperand()); 480 481 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo); 482 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1)); 483 return true; 484 } 485 486 return false; 487 } 488 489 void AMDGPUDAGToDAGISel::PreprocessISelDAG() { 490 if (!Subtarget->d16PreservesUnusedBits()) 491 return; 492 493 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 494 495 bool MadeChange = false; 496 while (Position != CurDAG->allnodes_begin()) { 497 SDNode *N = &*--Position; 498 if (N->use_empty()) 499 continue; 500 501 switch (N->getOpcode()) { 502 case ISD::BUILD_VECTOR: 503 MadeChange |= matchLoadD16FromBuildVector(N); 504 break; 505 default: 506 break; 507 } 508 } 509 510 if (MadeChange) { 511 CurDAG->RemoveDeadNodes(); 512 LLVM_DEBUG(dbgs() << "After PreProcess:\n"; 513 CurDAG->dump();); 514 } 515 } 516 517 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 518 if (TM.Options.NoNaNsFPMath) 519 return true; 520 521 // TODO: Move into isKnownNeverNaN 522 if (N->getFlags().isDefined()) 523 return N->getFlags().hasNoNaNs(); 524 525 return CurDAG->isKnownNeverNaN(N); 526 } 527 528 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N, 529 bool Negated) const { 530 if (N->isUndef()) 531 return true; 532 533 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 534 if (Negated) { 535 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 536 return TII->isInlineConstant(-C->getAPIntValue()); 537 538 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 539 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt()); 540 541 } else { 542 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 543 return TII->isInlineConstant(C->getAPIntValue()); 544 545 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 546 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 547 } 548 549 return false; 550 } 551 552 /// Determine the register class for \p OpNo 553 /// \returns The register class of the virtual register that will be used for 554 /// the given operand number \OpNo or NULL if the register class cannot be 555 /// determined. 556 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 557 unsigned OpNo) const { 558 if (!N->isMachineOpcode()) { 559 if (N->getOpcode() == ISD::CopyToReg) { 560 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 561 if (Register::isVirtualRegister(Reg)) { 562 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 563 return MRI.getRegClass(Reg); 564 } 565 566 const SIRegisterInfo *TRI 567 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 568 return TRI->getPhysRegClass(Reg); 569 } 570 571 return nullptr; 572 } 573 574 switch (N->getMachineOpcode()) { 575 default: { 576 const MCInstrDesc &Desc = 577 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 578 unsigned OpIdx = Desc.getNumDefs() + OpNo; 579 if (OpIdx >= Desc.getNumOperands()) 580 return nullptr; 581 int RegClass = Desc.OpInfo[OpIdx].RegClass; 582 if (RegClass == -1) 583 return nullptr; 584 585 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 586 } 587 case AMDGPU::REG_SEQUENCE: { 588 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 589 const TargetRegisterClass *SuperRC = 590 Subtarget->getRegisterInfo()->getRegClass(RCID); 591 592 SDValue SubRegOp = N->getOperand(OpNo + 1); 593 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 594 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 595 SubRegIdx); 596 } 597 } 598 } 599 600 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain, 601 SDValue Glue) const { 602 SmallVector <SDValue, 8> Ops; 603 Ops.push_back(NewChain); // Replace the chain. 604 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 605 Ops.push_back(N->getOperand(i)); 606 607 Ops.push_back(Glue); 608 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 609 } 610 611 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const { 612 const SITargetLowering& Lowering = 613 *static_cast<const SITargetLowering*>(getTargetLowering()); 614 615 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain"); 616 617 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val); 618 return glueCopyToOp(N, M0, M0.getValue(1)); 619 } 620 621 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const { 622 unsigned AS = cast<MemSDNode>(N)->getAddressSpace(); 623 if (AS == AMDGPUAS::LOCAL_ADDRESS) { 624 if (Subtarget->ldsRequiresM0Init()) 625 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 626 } else if (AS == AMDGPUAS::REGION_ADDRESS) { 627 MachineFunction &MF = CurDAG->getMachineFunction(); 628 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize(); 629 return 630 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32)); 631 } 632 return N; 633 } 634 635 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm, 636 EVT VT) const { 637 SDNode *Lo = CurDAG->getMachineNode( 638 AMDGPU::S_MOV_B32, DL, MVT::i32, 639 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32)); 640 SDNode *Hi = 641 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 642 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32)); 643 const SDValue Ops[] = { 644 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 645 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 646 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)}; 647 648 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops); 649 } 650 651 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) { 652 switch (NumVectorElts) { 653 case 1: 654 return AMDGPU::SReg_32RegClassID; 655 case 2: 656 return AMDGPU::SReg_64RegClassID; 657 case 3: 658 return AMDGPU::SGPR_96RegClassID; 659 case 4: 660 return AMDGPU::SGPR_128RegClassID; 661 case 5: 662 return AMDGPU::SGPR_160RegClassID; 663 case 8: 664 return AMDGPU::SReg_256RegClassID; 665 case 16: 666 return AMDGPU::SReg_512RegClassID; 667 case 32: 668 return AMDGPU::SReg_1024RegClassID; 669 } 670 671 llvm_unreachable("invalid vector size"); 672 } 673 674 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 675 EVT VT = N->getValueType(0); 676 unsigned NumVectorElts = VT.getVectorNumElements(); 677 EVT EltVT = VT.getVectorElementType(); 678 SDLoc DL(N); 679 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 680 681 if (NumVectorElts == 1) { 682 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 683 RegClass); 684 return; 685 } 686 687 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not " 688 "supported yet"); 689 // 32 = Max Num Vector Elements 690 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 691 // 1 = Vector Register Class 692 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 693 694 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() == 695 Triple::amdgcn; 696 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 697 bool IsRegSeq = true; 698 unsigned NOps = N->getNumOperands(); 699 for (unsigned i = 0; i < NOps; i++) { 700 // XXX: Why is this here? 701 if (isa<RegisterSDNode>(N->getOperand(i))) { 702 IsRegSeq = false; 703 break; 704 } 705 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 706 : R600RegisterInfo::getSubRegFromChannel(i); 707 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 708 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 709 } 710 if (NOps != NumVectorElts) { 711 // Fill in the missing undef elements if this was a scalar_to_vector. 712 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 713 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 714 DL, EltVT); 715 for (unsigned i = NOps; i < NumVectorElts; ++i) { 716 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 717 : R600RegisterInfo::getSubRegFromChannel(i); 718 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 719 RegSeqArgs[1 + (2 * i) + 1] = 720 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 721 } 722 } 723 724 if (!IsRegSeq) 725 SelectCode(N); 726 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 727 } 728 729 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 730 unsigned int Opc = N->getOpcode(); 731 if (N->isMachineOpcode()) { 732 N->setNodeId(-1); 733 return; // Already selected. 734 } 735 736 // isa<MemSDNode> almost works but is slightly too permissive for some DS 737 // intrinsics. 738 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) || 739 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 740 Opc == ISD::ATOMIC_LOAD_FADD || 741 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 742 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) { 743 N = glueCopyToM0LDSInit(N); 744 SelectCode(N); 745 return; 746 } 747 748 switch (Opc) { 749 default: 750 break; 751 // We are selecting i64 ADD here instead of custom lower it during 752 // DAG legalization, so we can fold some i64 ADDs used for address 753 // calculation into the LOAD and STORE instructions. 754 case ISD::ADDC: 755 case ISD::ADDE: 756 case ISD::SUBC: 757 case ISD::SUBE: { 758 if (N->getValueType(0) != MVT::i64) 759 break; 760 761 SelectADD_SUB_I64(N); 762 return; 763 } 764 case ISD::ADDCARRY: 765 case ISD::SUBCARRY: 766 if (N->getValueType(0) != MVT::i32) 767 break; 768 769 SelectAddcSubb(N); 770 return; 771 case ISD::UADDO: 772 case ISD::USUBO: { 773 SelectUADDO_USUBO(N); 774 return; 775 } 776 case AMDGPUISD::FMUL_W_CHAIN: { 777 SelectFMUL_W_CHAIN(N); 778 return; 779 } 780 case AMDGPUISD::FMA_W_CHAIN: { 781 SelectFMA_W_CHAIN(N); 782 return; 783 } 784 785 case ISD::SCALAR_TO_VECTOR: 786 case ISD::BUILD_VECTOR: { 787 EVT VT = N->getValueType(0); 788 unsigned NumVectorElts = VT.getVectorNumElements(); 789 if (VT.getScalarSizeInBits() == 16) { 790 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 791 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) { 792 ReplaceNode(N, Packed); 793 return; 794 } 795 } 796 797 break; 798 } 799 800 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 801 unsigned RegClassID = selectSGPRVectorRegClassID(NumVectorElts); 802 SelectBuildVector(N, RegClassID); 803 return; 804 } 805 case ISD::BUILD_PAIR: { 806 SDValue RC, SubReg0, SubReg1; 807 SDLoc DL(N); 808 if (N->getValueType(0) == MVT::i128) { 809 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32); 810 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 811 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 812 } else if (N->getValueType(0) == MVT::i64) { 813 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 814 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 815 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 816 } else { 817 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 818 } 819 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 820 N->getOperand(1), SubReg1 }; 821 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 822 N->getValueType(0), Ops)); 823 return; 824 } 825 826 case ISD::Constant: 827 case ISD::ConstantFP: { 828 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 829 break; 830 831 uint64_t Imm; 832 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 833 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 834 else { 835 ConstantSDNode *C = cast<ConstantSDNode>(N); 836 Imm = C->getZExtValue(); 837 } 838 839 SDLoc DL(N); 840 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0))); 841 return; 842 } 843 case AMDGPUISD::BFE_I32: 844 case AMDGPUISD::BFE_U32: { 845 // There is a scalar version available, but unlike the vector version which 846 // has a separate operand for the offset and width, the scalar version packs 847 // the width and offset into a single operand. Try to move to the scalar 848 // version if the offsets are constant, so that we can try to keep extended 849 // loads of kernel arguments in SGPRs. 850 851 // TODO: Technically we could try to pattern match scalar bitshifts of 852 // dynamic values, but it's probably not useful. 853 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 854 if (!Offset) 855 break; 856 857 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 858 if (!Width) 859 break; 860 861 bool Signed = Opc == AMDGPUISD::BFE_I32; 862 863 uint32_t OffsetVal = Offset->getZExtValue(); 864 uint32_t WidthVal = Width->getZExtValue(); 865 866 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 867 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 868 return; 869 } 870 case AMDGPUISD::DIV_SCALE: { 871 SelectDIV_SCALE(N); 872 return; 873 } 874 case AMDGPUISD::DIV_FMAS: { 875 SelectDIV_FMAS(N); 876 return; 877 } 878 case AMDGPUISD::MAD_I64_I32: 879 case AMDGPUISD::MAD_U64_U32: { 880 SelectMAD_64_32(N); 881 return; 882 } 883 case ISD::CopyToReg: { 884 const SITargetLowering& Lowering = 885 *static_cast<const SITargetLowering*>(getTargetLowering()); 886 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 887 break; 888 } 889 case ISD::AND: 890 case ISD::SRL: 891 case ISD::SRA: 892 case ISD::SIGN_EXTEND_INREG: 893 if (N->getValueType(0) != MVT::i32) 894 break; 895 896 SelectS_BFE(N); 897 return; 898 case ISD::BRCOND: 899 SelectBRCOND(N); 900 return; 901 case ISD::FMAD: 902 case ISD::FMA: 903 SelectFMAD_FMA(N); 904 return; 905 case AMDGPUISD::ATOMIC_CMP_SWAP: 906 SelectATOMIC_CMP_SWAP(N); 907 return; 908 case AMDGPUISD::CVT_PKRTZ_F16_F32: 909 case AMDGPUISD::CVT_PKNORM_I16_F32: 910 case AMDGPUISD::CVT_PKNORM_U16_F32: 911 case AMDGPUISD::CVT_PK_U16_U32: 912 case AMDGPUISD::CVT_PK_I16_I32: { 913 // Hack around using a legal type if f16 is illegal. 914 if (N->getValueType(0) == MVT::i32) { 915 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16; 916 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT), 917 { N->getOperand(0), N->getOperand(1) }); 918 SelectCode(N); 919 return; 920 } 921 922 break; 923 } 924 case ISD::INTRINSIC_W_CHAIN: { 925 SelectINTRINSIC_W_CHAIN(N); 926 return; 927 } 928 case ISD::INTRINSIC_WO_CHAIN: { 929 SelectINTRINSIC_WO_CHAIN(N); 930 return; 931 } 932 case ISD::INTRINSIC_VOID: { 933 SelectINTRINSIC_VOID(N); 934 return; 935 } 936 } 937 938 SelectCode(N); 939 } 940 941 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 942 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 943 const Instruction *Term = BB->getTerminator(); 944 return Term->getMetadata("amdgpu.uniform") || 945 Term->getMetadata("structurizecfg.uniform"); 946 } 947 948 StringRef AMDGPUDAGToDAGISel::getPassName() const { 949 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 950 } 951 952 //===----------------------------------------------------------------------===// 953 // Complex Patterns 954 //===----------------------------------------------------------------------===// 955 956 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 957 SDValue &Offset) { 958 return false; 959 } 960 961 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 962 SDValue &Offset) { 963 ConstantSDNode *C; 964 SDLoc DL(Addr); 965 966 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 967 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 968 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 969 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 970 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 971 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 972 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 973 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 974 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 975 Base = Addr.getOperand(0); 976 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 977 } else { 978 Base = Addr; 979 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 980 } 981 982 return true; 983 } 984 985 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val, 986 const SDLoc &DL) const { 987 SDNode *Mov = CurDAG->getMachineNode( 988 AMDGPU::S_MOV_B32, DL, MVT::i32, 989 CurDAG->getTargetConstant(Val, DL, MVT::i32)); 990 return SDValue(Mov, 0); 991 } 992 993 // FIXME: Should only handle addcarry/subcarry 994 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 995 SDLoc DL(N); 996 SDValue LHS = N->getOperand(0); 997 SDValue RHS = N->getOperand(1); 998 999 unsigned Opcode = N->getOpcode(); 1000 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 1001 bool ProduceCarry = 1002 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 1003 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 1004 1005 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1006 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1007 1008 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1009 DL, MVT::i32, LHS, Sub0); 1010 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1011 DL, MVT::i32, LHS, Sub1); 1012 1013 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1014 DL, MVT::i32, RHS, Sub0); 1015 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1016 DL, MVT::i32, RHS, Sub1); 1017 1018 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 1019 1020 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; 1021 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; 1022 1023 SDNode *AddLo; 1024 if (!ConsumeCarry) { 1025 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 1026 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 1027 } else { 1028 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 1029 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 1030 } 1031 SDValue AddHiArgs[] = { 1032 SDValue(Hi0, 0), 1033 SDValue(Hi1, 0), 1034 SDValue(AddLo, 1) 1035 }; 1036 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 1037 1038 SDValue RegSequenceArgs[] = { 1039 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 1040 SDValue(AddLo,0), 1041 Sub0, 1042 SDValue(AddHi,0), 1043 Sub1, 1044 }; 1045 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1046 MVT::i64, RegSequenceArgs); 1047 1048 if (ProduceCarry) { 1049 // Replace the carry-use 1050 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 1051 } 1052 1053 // Replace the remaining uses. 1054 ReplaceNode(N, RegSequence); 1055 } 1056 1057 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) { 1058 SDLoc DL(N); 1059 SDValue LHS = N->getOperand(0); 1060 SDValue RHS = N->getOperand(1); 1061 SDValue CI = N->getOperand(2); 1062 1063 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64 1064 : AMDGPU::V_SUBB_U32_e64; 1065 CurDAG->SelectNodeTo( 1066 N, Opc, N->getVTList(), 1067 {LHS, RHS, CI, CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1068 } 1069 1070 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 1071 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 1072 // carry out despite the _i32 name. These were renamed in VI to _U32. 1073 // FIXME: We should probably rename the opcodes here. 1074 unsigned Opc = N->getOpcode() == ISD::UADDO ? 1075 AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 1076 1077 CurDAG->SelectNodeTo( 1078 N, Opc, N->getVTList(), 1079 {N->getOperand(0), N->getOperand(1), 1080 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1081 } 1082 1083 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 1084 SDLoc SL(N); 1085 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 1086 SDValue Ops[10]; 1087 1088 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 1089 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1090 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 1091 Ops[8] = N->getOperand(0); 1092 Ops[9] = N->getOperand(4); 1093 1094 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); 1095 } 1096 1097 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 1098 SDLoc SL(N); 1099 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 1100 SDValue Ops[8]; 1101 1102 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 1103 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1104 Ops[6] = N->getOperand(0); 1105 Ops[7] = N->getOperand(3); 1106 1107 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 1108 } 1109 1110 // We need to handle this here because tablegen doesn't support matching 1111 // instructions with multiple outputs. 1112 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 1113 SDLoc SL(N); 1114 EVT VT = N->getValueType(0); 1115 1116 assert(VT == MVT::f32 || VT == MVT::f64); 1117 1118 unsigned Opc 1119 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32; 1120 1121 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; 1122 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1123 } 1124 1125 void AMDGPUDAGToDAGISel::SelectDIV_FMAS(SDNode *N) { 1126 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 1127 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 1128 1129 SDLoc SL(N); 1130 EVT VT = N->getValueType(0); 1131 1132 assert(VT == MVT::f32 || VT == MVT::f64); 1133 1134 unsigned Opc 1135 = (VT == MVT::f64) ? AMDGPU::V_DIV_FMAS_F64 : AMDGPU::V_DIV_FMAS_F32; 1136 1137 SDValue CarryIn = N->getOperand(3); 1138 // V_DIV_FMAS implicitly reads VCC. 1139 SDValue VCC = CurDAG->getCopyToReg(CurDAG->getEntryNode(), SL, 1140 TRI->getVCC(), CarryIn, SDValue()); 1141 1142 SDValue Ops[10]; 1143 1144 SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]); 1145 SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]); 1146 SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]); 1147 1148 Ops[8] = VCC; 1149 Ops[9] = VCC.getValue(1); 1150 1151 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1152 } 1153 1154 // We need to handle this here because tablegen doesn't support matching 1155 // instructions with multiple outputs. 1156 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 1157 SDLoc SL(N); 1158 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 1159 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32; 1160 1161 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 1162 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 1163 Clamp }; 1164 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1165 } 1166 1167 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset, 1168 unsigned OffsetBits) const { 1169 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 1170 (OffsetBits == 8 && !isUInt<8>(Offset))) 1171 return false; 1172 1173 if (Subtarget->hasUsableDSOffset() || 1174 Subtarget->unsafeDSOffsetFoldingEnabled()) 1175 return true; 1176 1177 // On Southern Islands instruction with a negative base value and an offset 1178 // don't seem to work. 1179 return CurDAG->SignBitIsZero(Base); 1180 } 1181 1182 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 1183 SDValue &Offset) const { 1184 SDLoc DL(Addr); 1185 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1186 SDValue N0 = Addr.getOperand(0); 1187 SDValue N1 = Addr.getOperand(1); 1188 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1189 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) { 1190 // (add n0, c0) 1191 Base = N0; 1192 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1193 return true; 1194 } 1195 } else if (Addr.getOpcode() == ISD::SUB) { 1196 // sub C, x -> add (sub 0, x), C 1197 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1198 int64_t ByteOffset = C->getSExtValue(); 1199 if (isUInt<16>(ByteOffset)) { 1200 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1201 1202 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1203 // the known bits in isDSOffsetLegal. We need to emit the selected node 1204 // here, so this is thrown away. 1205 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1206 Zero, Addr.getOperand(1)); 1207 1208 if (isDSOffsetLegal(Sub, ByteOffset, 16)) { 1209 SmallVector<SDValue, 3> Opnds; 1210 Opnds.push_back(Zero); 1211 Opnds.push_back(Addr.getOperand(1)); 1212 1213 // FIXME: Select to VOP3 version for with-carry. 1214 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1215 if (Subtarget->hasAddNoCarry()) { 1216 SubOp = AMDGPU::V_SUB_U32_e64; 1217 Opnds.push_back( 1218 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1219 } 1220 1221 MachineSDNode *MachineSub = 1222 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1223 1224 Base = SDValue(MachineSub, 0); 1225 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 1226 return true; 1227 } 1228 } 1229 } 1230 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1231 // If we have a constant address, prefer to put the constant into the 1232 // offset. This can save moves to load the constant address since multiple 1233 // operations can share the zero base address register, and enables merging 1234 // into read2 / write2 instructions. 1235 1236 SDLoc DL(Addr); 1237 1238 if (isUInt<16>(CAddr->getZExtValue())) { 1239 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1240 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1241 DL, MVT::i32, Zero); 1242 Base = SDValue(MovZero, 0); 1243 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1244 return true; 1245 } 1246 } 1247 1248 // default case 1249 Base = Addr; 1250 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 1251 return true; 1252 } 1253 1254 // TODO: If offset is too big, put low 16-bit into offset. 1255 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 1256 SDValue &Offset0, 1257 SDValue &Offset1) const { 1258 SDLoc DL(Addr); 1259 1260 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1261 SDValue N0 = Addr.getOperand(0); 1262 SDValue N1 = Addr.getOperand(1); 1263 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1264 unsigned DWordOffset0 = C1->getZExtValue() / 4; 1265 unsigned DWordOffset1 = DWordOffset0 + 1; 1266 // (add n0, c0) 1267 if (isDSOffsetLegal(N0, DWordOffset1, 8)) { 1268 Base = N0; 1269 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1270 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1271 return true; 1272 } 1273 } else if (Addr.getOpcode() == ISD::SUB) { 1274 // sub C, x -> add (sub 0, x), C 1275 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1276 unsigned DWordOffset0 = C->getZExtValue() / 4; 1277 unsigned DWordOffset1 = DWordOffset0 + 1; 1278 1279 if (isUInt<8>(DWordOffset0)) { 1280 SDLoc DL(Addr); 1281 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1282 1283 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1284 // the known bits in isDSOffsetLegal. We need to emit the selected node 1285 // here, so this is thrown away. 1286 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1287 Zero, Addr.getOperand(1)); 1288 1289 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) { 1290 SmallVector<SDValue, 3> Opnds; 1291 Opnds.push_back(Zero); 1292 Opnds.push_back(Addr.getOperand(1)); 1293 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1294 if (Subtarget->hasAddNoCarry()) { 1295 SubOp = AMDGPU::V_SUB_U32_e64; 1296 Opnds.push_back( 1297 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1298 } 1299 1300 MachineSDNode *MachineSub 1301 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1302 1303 Base = SDValue(MachineSub, 0); 1304 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1305 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1306 return true; 1307 } 1308 } 1309 } 1310 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1311 unsigned DWordOffset0 = CAddr->getZExtValue() / 4; 1312 unsigned DWordOffset1 = DWordOffset0 + 1; 1313 assert(4 * DWordOffset0 == CAddr->getZExtValue()); 1314 1315 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) { 1316 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1317 MachineSDNode *MovZero 1318 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1319 DL, MVT::i32, Zero); 1320 Base = SDValue(MovZero, 0); 1321 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1322 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1323 return true; 1324 } 1325 } 1326 1327 // default case 1328 1329 Base = Addr; 1330 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 1331 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 1332 return true; 1333 } 1334 1335 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, 1336 SDValue &VAddr, SDValue &SOffset, 1337 SDValue &Offset, SDValue &Offen, 1338 SDValue &Idxen, SDValue &Addr64, 1339 SDValue &GLC, SDValue &SLC, 1340 SDValue &TFE, SDValue &DLC, 1341 SDValue &SWZ) const { 1342 // Subtarget prefers to use flat instruction 1343 // FIXME: This should be a pattern predicate and not reach here 1344 if (Subtarget->useFlatForGlobal()) 1345 return false; 1346 1347 SDLoc DL(Addr); 1348 1349 if (!GLC.getNode()) 1350 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1351 if (!SLC.getNode()) 1352 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1353 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1); 1354 DLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1355 SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1); 1356 1357 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1358 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1359 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1360 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1361 1362 ConstantSDNode *C1 = nullptr; 1363 SDValue N0 = Addr; 1364 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1365 C1 = cast<ConstantSDNode>(Addr.getOperand(1)); 1366 if (isUInt<32>(C1->getZExtValue())) 1367 N0 = Addr.getOperand(0); 1368 else 1369 C1 = nullptr; 1370 } 1371 1372 if (N0.getOpcode() == ISD::ADD) { 1373 // (add N2, N3) -> addr64, or 1374 // (add (add N2, N3), C1) -> addr64 1375 SDValue N2 = N0.getOperand(0); 1376 SDValue N3 = N0.getOperand(1); 1377 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1378 1379 if (N2->isDivergent()) { 1380 if (N3->isDivergent()) { 1381 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 1382 // addr64, and construct the resource from a 0 address. 1383 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1384 VAddr = N0; 1385 } else { 1386 // N2 is divergent, N3 is not. 1387 Ptr = N3; 1388 VAddr = N2; 1389 } 1390 } else { 1391 // N2 is not divergent. 1392 Ptr = N2; 1393 VAddr = N3; 1394 } 1395 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1396 } else if (N0->isDivergent()) { 1397 // N0 is divergent. Use it as the addr64, and construct the resource from a 1398 // 0 address. 1399 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1400 VAddr = N0; 1401 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1402 } else { 1403 // N0 -> offset, or 1404 // (N0 + C1) -> offset 1405 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1406 Ptr = N0; 1407 } 1408 1409 if (!C1) { 1410 // No offset. 1411 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1412 return true; 1413 } 1414 1415 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1416 // Legal offset for instruction. 1417 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1418 return true; 1419 } 1420 1421 // Illegal offset, store it in soffset. 1422 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1423 SOffset = 1424 SDValue(CurDAG->getMachineNode( 1425 AMDGPU::S_MOV_B32, DL, MVT::i32, 1426 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1427 0); 1428 return true; 1429 } 1430 1431 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1432 SDValue &VAddr, SDValue &SOffset, 1433 SDValue &Offset, SDValue &GLC, 1434 SDValue &SLC, SDValue &TFE, 1435 SDValue &DLC, SDValue &SWZ) const { 1436 SDValue Ptr, Offen, Idxen, Addr64; 1437 1438 // addr64 bit was removed for volcanic islands. 1439 // FIXME: This should be a pattern predicate and not reach here 1440 if (!Subtarget->hasAddr64()) 1441 return false; 1442 1443 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1444 GLC, SLC, TFE, DLC, SWZ)) 1445 return false; 1446 1447 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1448 if (C->getSExtValue()) { 1449 SDLoc DL(Addr); 1450 1451 const SITargetLowering& Lowering = 1452 *static_cast<const SITargetLowering*>(getTargetLowering()); 1453 1454 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1455 return true; 1456 } 1457 1458 return false; 1459 } 1460 1461 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1462 SDValue &VAddr, SDValue &SOffset, 1463 SDValue &Offset, 1464 SDValue &SLC) const { 1465 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1); 1466 SDValue GLC, TFE, DLC, SWZ; 1467 1468 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1469 } 1470 1471 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 1472 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 1473 return PSV && PSV->isStack(); 1474 } 1475 1476 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1477 const MachineFunction &MF = CurDAG->getMachineFunction(); 1478 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1479 1480 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) { 1481 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1482 FI->getValueType(0)); 1483 1484 // If we can resolve this to a frame index access, this will be relative to 1485 // either the stack or frame pointer SGPR. 1486 return std::make_pair( 1487 TFI, CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)); 1488 } 1489 1490 // If we don't know this private access is a local stack object, it needs to 1491 // be relative to the entry point's scratch wave offset register. 1492 return std::make_pair(N, CurDAG->getRegister(Info->getScratchWaveOffsetReg(), 1493 MVT::i32)); 1494 } 1495 1496 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1497 SDValue Addr, SDValue &Rsrc, 1498 SDValue &VAddr, SDValue &SOffset, 1499 SDValue &ImmOffset) const { 1500 1501 SDLoc DL(Addr); 1502 MachineFunction &MF = CurDAG->getMachineFunction(); 1503 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1504 1505 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1506 1507 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1508 unsigned Imm = CAddr->getZExtValue(); 1509 1510 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1511 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1512 DL, MVT::i32, HighBits); 1513 VAddr = SDValue(MovHighBits, 0); 1514 1515 // In a call sequence, stores to the argument stack area are relative to the 1516 // stack pointer. 1517 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1518 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ? 1519 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg(); 1520 1521 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32); 1522 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1523 return true; 1524 } 1525 1526 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1527 // (add n0, c1) 1528 1529 SDValue N0 = Addr.getOperand(0); 1530 SDValue N1 = Addr.getOperand(1); 1531 1532 // Offsets in vaddr must be positive if range checking is enabled. 1533 // 1534 // The total computation of vaddr + soffset + offset must not overflow. If 1535 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1536 // overflowing. 1537 // 1538 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1539 // always perform a range check. If a negative vaddr base index was used, 1540 // this would fail the range check. The overall address computation would 1541 // compute a valid address, but this doesn't happen due to the range 1542 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1543 // 1544 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1545 // MUBUF vaddr, but not on older subtargets which can only do this if the 1546 // sign bit is known 0. 1547 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1548 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1549 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1550 CurDAG->SignBitIsZero(N0))) { 1551 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1552 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1553 return true; 1554 } 1555 } 1556 1557 // (node) 1558 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1559 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1560 return true; 1561 } 1562 1563 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1564 SDValue Addr, 1565 SDValue &SRsrc, 1566 SDValue &SOffset, 1567 SDValue &Offset) const { 1568 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr); 1569 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1570 return false; 1571 1572 SDLoc DL(Addr); 1573 MachineFunction &MF = CurDAG->getMachineFunction(); 1574 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1575 1576 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1577 1578 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1579 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ? 1580 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg(); 1581 1582 // FIXME: Get from MachinePointerInfo? We should only be using the frame 1583 // offset if we know this is in a call sequence. 1584 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32); 1585 1586 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1587 return true; 1588 } 1589 1590 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1591 SDValue &SOffset, SDValue &Offset, 1592 SDValue &GLC, SDValue &SLC, 1593 SDValue &TFE, SDValue &DLC, 1594 SDValue &SWZ) const { 1595 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1596 const SIInstrInfo *TII = 1597 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1598 1599 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1600 GLC, SLC, TFE, DLC, SWZ)) 1601 return false; 1602 1603 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1604 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1605 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1606 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1607 APInt::getAllOnesValue(32).getZExtValue(); // Size 1608 SDLoc DL(Addr); 1609 1610 const SITargetLowering& Lowering = 1611 *static_cast<const SITargetLowering*>(getTargetLowering()); 1612 1613 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1614 return true; 1615 } 1616 return false; 1617 } 1618 1619 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1620 SDValue &Soffset, SDValue &Offset 1621 ) const { 1622 SDValue GLC, SLC, TFE, DLC, SWZ; 1623 1624 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1625 } 1626 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1627 SDValue &Soffset, SDValue &Offset, 1628 SDValue &SLC) const { 1629 SDValue GLC, TFE, DLC, SWZ; 1630 1631 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1632 } 1633 1634 // Find a load or store from corresponding pattern root. 1635 // Roots may be build_vector, bitconvert or their combinations. 1636 static MemSDNode* findMemSDNode(SDNode *N) { 1637 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode(); 1638 if (MemSDNode *MN = dyn_cast<MemSDNode>(N)) 1639 return MN; 1640 assert(isa<BuildVectorSDNode>(N)); 1641 for (SDValue V : N->op_values()) 1642 if (MemSDNode *MN = 1643 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V))) 1644 return MN; 1645 llvm_unreachable("cannot find MemSDNode in the pattern!"); 1646 } 1647 1648 template <bool IsSigned> 1649 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, 1650 SDValue Addr, 1651 SDValue &VAddr, 1652 SDValue &Offset, 1653 SDValue &SLC) const { 1654 int64_t OffsetVal = 0; 1655 1656 if (Subtarget->hasFlatInstOffsets() && 1657 (!Subtarget->hasFlatSegmentOffsetBug() || 1658 findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) && 1659 CurDAG->isBaseWithConstantOffset(Addr)) { 1660 SDValue N0 = Addr.getOperand(0); 1661 SDValue N1 = Addr.getOperand(1); 1662 uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1663 1664 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1665 unsigned AS = findMemSDNode(N)->getAddressSpace(); 1666 if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) { 1667 Addr = N0; 1668 OffsetVal = COffsetVal; 1669 } else { 1670 // If the offset doesn't fit, put the low bits into the offset field and 1671 // add the rest. 1672 1673 SDLoc DL(N); 1674 uint64_t ImmField; 1675 const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned); 1676 if (IsSigned) { 1677 ImmField = SignExtend64(COffsetVal, NumBits); 1678 1679 // Don't use a negative offset field if the base offset is positive. 1680 // Since the scheduler currently relies on the offset field, doing so 1681 // could result in strange scheduling decisions. 1682 1683 // TODO: Should we not do this in the opposite direction as well? 1684 if (static_cast<int64_t>(COffsetVal) > 0) { 1685 if (static_cast<int64_t>(ImmField) < 0) { 1686 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits - 1); 1687 ImmField = COffsetVal & OffsetMask; 1688 } 1689 } 1690 } else { 1691 // TODO: Should we do this for a negative offset? 1692 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits); 1693 ImmField = COffsetVal & OffsetMask; 1694 } 1695 1696 uint64_t RemainderOffset = COffsetVal - ImmField; 1697 1698 assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned)); 1699 assert(RemainderOffset + ImmField == COffsetVal); 1700 1701 OffsetVal = ImmField; 1702 1703 // TODO: Should this try to use a scalar add pseudo if the base address is 1704 // uniform and saddr is usable? 1705 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1706 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1707 1708 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1709 DL, MVT::i32, N0, Sub0); 1710 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1711 DL, MVT::i32, N0, Sub1); 1712 1713 SDValue AddOffsetLo 1714 = getMaterializedScalarImm32(Lo_32(RemainderOffset), DL); 1715 SDValue AddOffsetHi 1716 = getMaterializedScalarImm32(Hi_32(RemainderOffset), DL); 1717 1718 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1); 1719 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1720 1721 SDNode *Add = CurDAG->getMachineNode( 1722 AMDGPU::V_ADD_I32_e64, DL, VTs, 1723 {AddOffsetLo, SDValue(N0Lo, 0), Clamp}); 1724 1725 SDNode *Addc = CurDAG->getMachineNode( 1726 AMDGPU::V_ADDC_U32_e64, DL, VTs, 1727 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp}); 1728 1729 SDValue RegSequenceArgs[] = { 1730 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32), 1731 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1 1732 }; 1733 1734 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1735 MVT::i64, RegSequenceArgs), 0); 1736 } 1737 } 1738 1739 VAddr = Addr; 1740 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1741 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); 1742 return true; 1743 } 1744 1745 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDNode *N, 1746 SDValue Addr, 1747 SDValue &VAddr, 1748 SDValue &Offset, 1749 SDValue &SLC) const { 1750 return SelectFlatOffset<false>(N, Addr, VAddr, Offset, SLC); 1751 } 1752 1753 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N, 1754 SDValue Addr, 1755 SDValue &VAddr, 1756 SDValue &Offset, 1757 SDValue &SLC) const { 1758 return SelectFlatOffset<true>(N, Addr, VAddr, Offset, SLC); 1759 } 1760 1761 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 1762 SDValue &Offset, bool &Imm) const { 1763 1764 // FIXME: Handle non-constant offsets. 1765 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 1766 if (!C) 1767 return false; 1768 1769 SDLoc SL(ByteOffsetNode); 1770 GCNSubtarget::Generation Gen = Subtarget->getGeneration(); 1771 uint64_t ByteOffset = C->getZExtValue(); 1772 Optional<int64_t> EncodedOffset = 1773 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset); 1774 if (EncodedOffset) { 1775 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1776 Imm = true; 1777 return true; 1778 } 1779 1780 if (Gen == AMDGPUSubtarget::SEA_ISLANDS) { 1781 EncodedOffset = 1782 AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset); 1783 if (EncodedOffset) { 1784 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1785 return true; 1786 } 1787 } 1788 1789 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset)) 1790 return false; 1791 1792 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 1793 Offset = SDValue( 1794 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0); 1795 1796 return true; 1797 } 1798 1799 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 1800 if (Addr.getValueType() != MVT::i32) 1801 return Addr; 1802 1803 // Zero-extend a 32-bit address. 1804 SDLoc SL(Addr); 1805 1806 const MachineFunction &MF = CurDAG->getMachineFunction(); 1807 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1808 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 1809 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 1810 1811 const SDValue Ops[] = { 1812 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 1813 Addr, 1814 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 1815 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 1816 0), 1817 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 1818 }; 1819 1820 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 1821 Ops), 0); 1822 } 1823 1824 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 1825 SDValue &Offset, bool &Imm) const { 1826 SDLoc SL(Addr); 1827 1828 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 1829 // wraparound, because s_load instructions perform the addition in 64 bits. 1830 if ((Addr.getValueType() != MVT::i32 || 1831 Addr->getFlags().hasNoUnsignedWrap()) && 1832 CurDAG->isBaseWithConstantOffset(Addr)) { 1833 SDValue N0 = Addr.getOperand(0); 1834 SDValue N1 = Addr.getOperand(1); 1835 1836 if (SelectSMRDOffset(N1, Offset, Imm)) { 1837 SBase = Expand32BitAddress(N0); 1838 return true; 1839 } 1840 } 1841 SBase = Expand32BitAddress(Addr); 1842 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 1843 Imm = true; 1844 return true; 1845 } 1846 1847 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 1848 SDValue &Offset) const { 1849 bool Imm = false; 1850 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 1851 } 1852 1853 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 1854 SDValue &Offset) const { 1855 1856 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1857 1858 bool Imm = false; 1859 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 1860 return false; 1861 1862 return !Imm && isa<ConstantSDNode>(Offset); 1863 } 1864 1865 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 1866 SDValue &Offset) const { 1867 bool Imm = false; 1868 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 1869 !isa<ConstantSDNode>(Offset); 1870 } 1871 1872 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 1873 SDValue &Offset) const { 1874 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1875 if (auto Imm = AMDGPU::getSMRDEncodedOffset(*Subtarget, 1876 C->getZExtValue())) { 1877 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1878 return true; 1879 } 1880 } 1881 1882 return false; 1883 } 1884 1885 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 1886 SDValue &Offset) const { 1887 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1888 1889 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1890 if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, 1891 C->getZExtValue())) { 1892 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1893 return true; 1894 } 1895 } 1896 1897 return false; 1898 } 1899 1900 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 1901 SDValue &Base, 1902 SDValue &Offset) const { 1903 SDLoc DL(Index); 1904 1905 if (CurDAG->isBaseWithConstantOffset(Index)) { 1906 SDValue N0 = Index.getOperand(0); 1907 SDValue N1 = Index.getOperand(1); 1908 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1909 1910 // (add n0, c0) 1911 // Don't peel off the offset (c0) if doing so could possibly lead 1912 // the base (n0) to be negative. 1913 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) { 1914 Base = N0; 1915 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 1916 return true; 1917 } 1918 } 1919 1920 if (isa<ConstantSDNode>(Index)) 1921 return false; 1922 1923 Base = Index; 1924 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1925 return true; 1926 } 1927 1928 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 1929 SDValue Val, uint32_t Offset, 1930 uint32_t Width) { 1931 // Transformation function, pack the offset and width of a BFE into 1932 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1933 // source, bits [5:0] contain the offset and bits [22:16] the width. 1934 uint32_t PackedVal = Offset | (Width << 16); 1935 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 1936 1937 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 1938 } 1939 1940 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 1941 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 1942 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 1943 // Predicate: 0 < b <= c < 32 1944 1945 const SDValue &Shl = N->getOperand(0); 1946 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 1947 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1948 1949 if (B && C) { 1950 uint32_t BVal = B->getZExtValue(); 1951 uint32_t CVal = C->getZExtValue(); 1952 1953 if (0 < BVal && BVal <= CVal && CVal < 32) { 1954 bool Signed = N->getOpcode() == ISD::SRA; 1955 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1956 1957 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 1958 32 - CVal)); 1959 return; 1960 } 1961 } 1962 SelectCode(N); 1963 } 1964 1965 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 1966 switch (N->getOpcode()) { 1967 case ISD::AND: 1968 if (N->getOperand(0).getOpcode() == ISD::SRL) { 1969 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 1970 // Predicate: isMask(mask) 1971 const SDValue &Srl = N->getOperand(0); 1972 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 1973 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1974 1975 if (Shift && Mask) { 1976 uint32_t ShiftVal = Shift->getZExtValue(); 1977 uint32_t MaskVal = Mask->getZExtValue(); 1978 1979 if (isMask_32(MaskVal)) { 1980 uint32_t WidthVal = countPopulation(MaskVal); 1981 1982 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1983 Srl.getOperand(0), ShiftVal, WidthVal)); 1984 return; 1985 } 1986 } 1987 } 1988 break; 1989 case ISD::SRL: 1990 if (N->getOperand(0).getOpcode() == ISD::AND) { 1991 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 1992 // Predicate: isMask(mask >> b) 1993 const SDValue &And = N->getOperand(0); 1994 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1995 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 1996 1997 if (Shift && Mask) { 1998 uint32_t ShiftVal = Shift->getZExtValue(); 1999 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 2000 2001 if (isMask_32(MaskVal)) { 2002 uint32_t WidthVal = countPopulation(MaskVal); 2003 2004 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2005 And.getOperand(0), ShiftVal, WidthVal)); 2006 return; 2007 } 2008 } 2009 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 2010 SelectS_BFEFromShifts(N); 2011 return; 2012 } 2013 break; 2014 case ISD::SRA: 2015 if (N->getOperand(0).getOpcode() == ISD::SHL) { 2016 SelectS_BFEFromShifts(N); 2017 return; 2018 } 2019 break; 2020 2021 case ISD::SIGN_EXTEND_INREG: { 2022 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 2023 SDValue Src = N->getOperand(0); 2024 if (Src.getOpcode() != ISD::SRL) 2025 break; 2026 2027 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 2028 if (!Amt) 2029 break; 2030 2031 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2032 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 2033 Amt->getZExtValue(), Width)); 2034 return; 2035 } 2036 } 2037 2038 SelectCode(N); 2039 } 2040 2041 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 2042 assert(N->getOpcode() == ISD::BRCOND); 2043 if (!N->hasOneUse()) 2044 return false; 2045 2046 SDValue Cond = N->getOperand(1); 2047 if (Cond.getOpcode() == ISD::CopyToReg) 2048 Cond = Cond.getOperand(2); 2049 2050 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 2051 return false; 2052 2053 MVT VT = Cond.getOperand(0).getSimpleValueType(); 2054 if (VT == MVT::i32) 2055 return true; 2056 2057 if (VT == MVT::i64) { 2058 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 2059 2060 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 2061 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 2062 } 2063 2064 return false; 2065 } 2066 2067 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 2068 SDValue Cond = N->getOperand(1); 2069 2070 if (Cond.isUndef()) { 2071 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 2072 N->getOperand(2), N->getOperand(0)); 2073 return; 2074 } 2075 2076 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 2077 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 2078 2079 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 2080 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 2081 unsigned CondReg = UseSCCBr ? (unsigned)AMDGPU::SCC : TRI->getVCC(); 2082 SDLoc SL(N); 2083 2084 if (!UseSCCBr) { 2085 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 2086 // analyzed what generates the vcc value, so we do not know whether vcc 2087 // bits for disabled lanes are 0. Thus we need to mask out bits for 2088 // disabled lanes. 2089 // 2090 // For the case that we select S_CBRANCH_SCC1 and it gets 2091 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 2092 // SIInstrInfo::moveToVALU which inserts the S_AND). 2093 // 2094 // We could add an analysis of what generates the vcc value here and omit 2095 // the S_AND when is unnecessary. But it would be better to add a separate 2096 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 2097 // catches both cases. 2098 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32 2099 : AMDGPU::S_AND_B64, 2100 SL, MVT::i1, 2101 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO 2102 : AMDGPU::EXEC, 2103 MVT::i1), 2104 Cond), 2105 0); 2106 } 2107 2108 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 2109 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 2110 N->getOperand(2), // Basic Block 2111 VCC.getValue(0)); 2112 } 2113 2114 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 2115 MVT VT = N->getSimpleValueType(0); 2116 bool IsFMA = N->getOpcode() == ISD::FMA; 2117 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 2118 !Subtarget->hasFmaMixInsts()) || 2119 ((IsFMA && Subtarget->hasMadMixInsts()) || 2120 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 2121 SelectCode(N); 2122 return; 2123 } 2124 2125 SDValue Src0 = N->getOperand(0); 2126 SDValue Src1 = N->getOperand(1); 2127 SDValue Src2 = N->getOperand(2); 2128 unsigned Src0Mods, Src1Mods, Src2Mods; 2129 2130 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 2131 // using the conversion from f16. 2132 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 2133 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 2134 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 2135 2136 assert((IsFMA || !Mode.allFP32Denormals()) && 2137 "fmad selected with denormals enabled"); 2138 // TODO: We can select this with f32 denormals enabled if all the sources are 2139 // converted from f16 (in which case fmad isn't legal). 2140 2141 if (Sel0 || Sel1 || Sel2) { 2142 // For dummy operands. 2143 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2144 SDValue Ops[] = { 2145 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 2146 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 2147 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 2148 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 2149 Zero, Zero 2150 }; 2151 2152 CurDAG->SelectNodeTo(N, 2153 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 2154 MVT::f32, Ops); 2155 } else { 2156 SelectCode(N); 2157 } 2158 } 2159 2160 // This is here because there isn't a way to use the generated sub0_sub1 as the 2161 // subreg index to EXTRACT_SUBREG in tablegen. 2162 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 2163 MemSDNode *Mem = cast<MemSDNode>(N); 2164 unsigned AS = Mem->getAddressSpace(); 2165 if (AS == AMDGPUAS::FLAT_ADDRESS) { 2166 SelectCode(N); 2167 return; 2168 } 2169 2170 MVT VT = N->getSimpleValueType(0); 2171 bool Is32 = (VT == MVT::i32); 2172 SDLoc SL(N); 2173 2174 MachineSDNode *CmpSwap = nullptr; 2175 if (Subtarget->hasAddr64()) { 2176 SDValue SRsrc, VAddr, SOffset, Offset, SLC; 2177 2178 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) { 2179 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 2180 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 2181 SDValue CmpVal = Mem->getOperand(2); 2182 2183 // XXX - Do we care about glue operands? 2184 2185 SDValue Ops[] = { 2186 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2187 }; 2188 2189 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2190 } 2191 } 2192 2193 if (!CmpSwap) { 2194 SDValue SRsrc, SOffset, Offset, SLC; 2195 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) { 2196 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 2197 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 2198 2199 SDValue CmpVal = Mem->getOperand(2); 2200 SDValue Ops[] = { 2201 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2202 }; 2203 2204 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2205 } 2206 } 2207 2208 if (!CmpSwap) { 2209 SelectCode(N); 2210 return; 2211 } 2212 2213 MachineMemOperand *MMO = Mem->getMemOperand(); 2214 CurDAG->setNodeMemRefs(CmpSwap, {MMO}); 2215 2216 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 2217 SDValue Extract 2218 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 2219 2220 ReplaceUses(SDValue(N, 0), Extract); 2221 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 2222 CurDAG->RemoveDeadNode(N); 2223 } 2224 2225 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { 2226 // The address is assumed to be uniform, so if it ends up in a VGPR, it will 2227 // be copied to an SGPR with readfirstlane. 2228 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ? 2229 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 2230 2231 SDValue Chain = N->getOperand(0); 2232 SDValue Ptr = N->getOperand(2); 2233 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2234 MachineMemOperand *MMO = M->getMemOperand(); 2235 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 2236 2237 SDValue Offset; 2238 if (CurDAG->isBaseWithConstantOffset(Ptr)) { 2239 SDValue PtrBase = Ptr.getOperand(0); 2240 SDValue PtrOffset = Ptr.getOperand(1); 2241 2242 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue(); 2243 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) { 2244 N = glueCopyToM0(N, PtrBase); 2245 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32); 2246 } 2247 } 2248 2249 if (!Offset) { 2250 N = glueCopyToM0(N, Ptr); 2251 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2252 } 2253 2254 SDValue Ops[] = { 2255 Offset, 2256 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32), 2257 Chain, 2258 N->getOperand(N->getNumOperands() - 1) // New glue 2259 }; 2260 2261 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2262 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2263 } 2264 2265 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 2266 switch (IntrID) { 2267 case Intrinsic::amdgcn_ds_gws_init: 2268 return AMDGPU::DS_GWS_INIT; 2269 case Intrinsic::amdgcn_ds_gws_barrier: 2270 return AMDGPU::DS_GWS_BARRIER; 2271 case Intrinsic::amdgcn_ds_gws_sema_v: 2272 return AMDGPU::DS_GWS_SEMA_V; 2273 case Intrinsic::amdgcn_ds_gws_sema_br: 2274 return AMDGPU::DS_GWS_SEMA_BR; 2275 case Intrinsic::amdgcn_ds_gws_sema_p: 2276 return AMDGPU::DS_GWS_SEMA_P; 2277 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2278 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 2279 default: 2280 llvm_unreachable("not a gws intrinsic"); 2281 } 2282 } 2283 2284 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { 2285 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all && 2286 !Subtarget->hasGWSSemaReleaseAll()) { 2287 // Let this error. 2288 SelectCode(N); 2289 return; 2290 } 2291 2292 // Chain, intrinsic ID, vsrc, offset 2293 const bool HasVSrc = N->getNumOperands() == 4; 2294 assert(HasVSrc || N->getNumOperands() == 3); 2295 2296 SDLoc SL(N); 2297 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2); 2298 int ImmOffset = 0; 2299 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2300 MachineMemOperand *MMO = M->getMemOperand(); 2301 2302 // Don't worry if the offset ends up in a VGPR. Only one lane will have 2303 // effect, so SIFixSGPRCopies will validly insert readfirstlane. 2304 2305 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 2306 // offset field) % 64. Some versions of the programming guide omit the m0 2307 // part, or claim it's from offset 0. 2308 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) { 2309 // If we have a constant offset, try to use the 0 in m0 as the base. 2310 // TODO: Look into changing the default m0 initialization value. If the 2311 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 2312 // the immediate offset. 2313 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); 2314 ImmOffset = ConstOffset->getZExtValue(); 2315 } else { 2316 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { 2317 ImmOffset = BaseOffset.getConstantOperandVal(1); 2318 BaseOffset = BaseOffset.getOperand(0); 2319 } 2320 2321 // Prefer to do the shift in an SGPR since it should be possible to use m0 2322 // as the result directly. If it's already an SGPR, it will be eliminated 2323 // later. 2324 SDNode *SGPROffset 2325 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, 2326 BaseOffset); 2327 // Shift to offset in m0 2328 SDNode *M0Base 2329 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32, 2330 SDValue(SGPROffset, 0), 2331 CurDAG->getTargetConstant(16, SL, MVT::i32)); 2332 glueCopyToM0(N, SDValue(M0Base, 0)); 2333 } 2334 2335 SDValue Chain = N->getOperand(0); 2336 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32); 2337 2338 // TODO: Can this just be removed from the instruction? 2339 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1); 2340 2341 const unsigned Opc = gwsIntrinToOpcode(IntrID); 2342 SmallVector<SDValue, 5> Ops; 2343 if (HasVSrc) 2344 Ops.push_back(N->getOperand(2)); 2345 Ops.push_back(OffsetField); 2346 Ops.push_back(GDS); 2347 Ops.push_back(Chain); 2348 2349 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2350 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2351 } 2352 2353 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) { 2354 if (Subtarget->getLDSBankCount() != 16) { 2355 // This is a single instruction with a pattern. 2356 SelectCode(N); 2357 return; 2358 } 2359 2360 SDLoc DL(N); 2361 2362 // This requires 2 instructions. It is possible to write a pattern to support 2363 // this, but the generated isel emitter doesn't correctly deal with multiple 2364 // output instructions using the same physical register input. The copy to m0 2365 // is incorrectly placed before the second instruction. 2366 // 2367 // TODO: Match source modifiers. 2368 // 2369 // def : Pat < 2370 // (int_amdgcn_interp_p1_f16 2371 // (VOP3Mods f32:$src0, i32:$src0_modifiers), 2372 // (i32 timm:$attrchan), (i32 timm:$attr), 2373 // (i1 timm:$high), M0), 2374 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr, 2375 // timm:$attrchan, 0, 2376 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> { 2377 // let Predicates = [has16BankLDS]; 2378 // } 2379 2380 // 16 bank LDS 2381 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0, 2382 N->getOperand(5), SDValue()); 2383 2384 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other); 2385 2386 SDNode *InterpMov = 2387 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, { 2388 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0 2389 N->getOperand(3), // Attr 2390 N->getOperand(2), // Attrchan 2391 ToM0.getValue(1) // In glue 2392 }); 2393 2394 SDNode *InterpP1LV = 2395 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, { 2396 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers 2397 N->getOperand(1), // Src0 2398 N->getOperand(3), // Attr 2399 N->getOperand(2), // Attrchan 2400 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers 2401 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high 2402 N->getOperand(4), // high 2403 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp 2404 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod 2405 SDValue(InterpMov, 1) 2406 }); 2407 2408 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0)); 2409 } 2410 2411 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { 2412 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2413 switch (IntrID) { 2414 case Intrinsic::amdgcn_ds_append: 2415 case Intrinsic::amdgcn_ds_consume: { 2416 if (N->getValueType(0) != MVT::i32) 2417 break; 2418 SelectDSAppendConsume(N, IntrID); 2419 return; 2420 } 2421 } 2422 2423 SelectCode(N); 2424 } 2425 2426 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { 2427 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 2428 unsigned Opcode; 2429 switch (IntrID) { 2430 case Intrinsic::amdgcn_wqm: 2431 Opcode = AMDGPU::WQM; 2432 break; 2433 case Intrinsic::amdgcn_softwqm: 2434 Opcode = AMDGPU::SOFT_WQM; 2435 break; 2436 case Intrinsic::amdgcn_wwm: 2437 Opcode = AMDGPU::WWM; 2438 break; 2439 case Intrinsic::amdgcn_interp_p1_f16: 2440 SelectInterpP1F16(N); 2441 return; 2442 default: 2443 SelectCode(N); 2444 return; 2445 } 2446 2447 SDValue Src = N->getOperand(1); 2448 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); 2449 } 2450 2451 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { 2452 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2453 switch (IntrID) { 2454 case Intrinsic::amdgcn_ds_gws_init: 2455 case Intrinsic::amdgcn_ds_gws_barrier: 2456 case Intrinsic::amdgcn_ds_gws_sema_v: 2457 case Intrinsic::amdgcn_ds_gws_sema_br: 2458 case Intrinsic::amdgcn_ds_gws_sema_p: 2459 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2460 SelectDS_GWS(N, IntrID); 2461 return; 2462 default: 2463 break; 2464 } 2465 2466 SelectCode(N); 2467 } 2468 2469 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 2470 unsigned &Mods) const { 2471 Mods = 0; 2472 Src = In; 2473 2474 if (Src.getOpcode() == ISD::FNEG) { 2475 Mods |= SISrcMods::NEG; 2476 Src = Src.getOperand(0); 2477 } 2478 2479 if (Src.getOpcode() == ISD::FABS) { 2480 Mods |= SISrcMods::ABS; 2481 Src = Src.getOperand(0); 2482 } 2483 2484 return true; 2485 } 2486 2487 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 2488 SDValue &SrcMods) const { 2489 unsigned Mods; 2490 if (SelectVOP3ModsImpl(In, Src, Mods)) { 2491 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2492 return true; 2493 } 2494 2495 return false; 2496 } 2497 2498 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 2499 SDValue &SrcMods) const { 2500 SelectVOP3Mods(In, Src, SrcMods); 2501 return isNoNanSrc(Src); 2502 } 2503 2504 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 2505 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 2506 return false; 2507 2508 Src = In; 2509 return true; 2510 } 2511 2512 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 2513 SDValue &SrcMods, SDValue &Clamp, 2514 SDValue &Omod) const { 2515 SDLoc DL(In); 2516 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2517 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2518 2519 return SelectVOP3Mods(In, Src, SrcMods); 2520 } 2521 2522 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 2523 SDValue &Clamp, SDValue &Omod) const { 2524 Src = In; 2525 2526 SDLoc DL(In); 2527 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2528 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2529 2530 return true; 2531 } 2532 2533 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 2534 SDValue &SrcMods) const { 2535 unsigned Mods = 0; 2536 Src = In; 2537 2538 if (Src.getOpcode() == ISD::FNEG) { 2539 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 2540 Src = Src.getOperand(0); 2541 } 2542 2543 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 2544 unsigned VecMods = Mods; 2545 2546 SDValue Lo = stripBitcast(Src.getOperand(0)); 2547 SDValue Hi = stripBitcast(Src.getOperand(1)); 2548 2549 if (Lo.getOpcode() == ISD::FNEG) { 2550 Lo = stripBitcast(Lo.getOperand(0)); 2551 Mods ^= SISrcMods::NEG; 2552 } 2553 2554 if (Hi.getOpcode() == ISD::FNEG) { 2555 Hi = stripBitcast(Hi.getOperand(0)); 2556 Mods ^= SISrcMods::NEG_HI; 2557 } 2558 2559 if (isExtractHiElt(Lo, Lo)) 2560 Mods |= SISrcMods::OP_SEL_0; 2561 2562 if (isExtractHiElt(Hi, Hi)) 2563 Mods |= SISrcMods::OP_SEL_1; 2564 2565 Lo = stripExtractLoElt(Lo); 2566 Hi = stripExtractLoElt(Hi); 2567 2568 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 2569 // Really a scalar input. Just select from the low half of the register to 2570 // avoid packing. 2571 2572 Src = Lo; 2573 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2574 return true; 2575 } 2576 2577 Mods = VecMods; 2578 } 2579 2580 // Packed instructions do not have abs modifiers. 2581 Mods |= SISrcMods::OP_SEL_1; 2582 2583 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2584 return true; 2585 } 2586 2587 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2588 SDValue &SrcMods) const { 2589 Src = In; 2590 // FIXME: Handle op_sel 2591 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2592 return true; 2593 } 2594 2595 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2596 SDValue &SrcMods) const { 2597 // FIXME: Handle op_sel 2598 return SelectVOP3Mods(In, Src, SrcMods); 2599 } 2600 2601 // The return value is not whether the match is possible (which it always is), 2602 // but whether or not it a conversion is really used. 2603 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2604 unsigned &Mods) const { 2605 Mods = 0; 2606 SelectVOP3ModsImpl(In, Src, Mods); 2607 2608 if (Src.getOpcode() == ISD::FP_EXTEND) { 2609 Src = Src.getOperand(0); 2610 assert(Src.getValueType() == MVT::f16); 2611 Src = stripBitcast(Src); 2612 2613 // Be careful about folding modifiers if we already have an abs. fneg is 2614 // applied last, so we don't want to apply an earlier fneg. 2615 if ((Mods & SISrcMods::ABS) == 0) { 2616 unsigned ModsTmp; 2617 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2618 2619 if ((ModsTmp & SISrcMods::NEG) != 0) 2620 Mods ^= SISrcMods::NEG; 2621 2622 if ((ModsTmp & SISrcMods::ABS) != 0) 2623 Mods |= SISrcMods::ABS; 2624 } 2625 2626 // op_sel/op_sel_hi decide the source type and source. 2627 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2628 // If the sources's op_sel is set, it picks the high half of the source 2629 // register. 2630 2631 Mods |= SISrcMods::OP_SEL_1; 2632 if (isExtractHiElt(Src, Src)) { 2633 Mods |= SISrcMods::OP_SEL_0; 2634 2635 // TODO: Should we try to look for neg/abs here? 2636 } 2637 2638 return true; 2639 } 2640 2641 return false; 2642 } 2643 2644 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2645 SDValue &SrcMods) const { 2646 unsigned Mods = 0; 2647 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2648 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2649 return true; 2650 } 2651 2652 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { 2653 if (In.isUndef()) 2654 return CurDAG->getUNDEF(MVT::i32); 2655 2656 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2657 SDLoc SL(In); 2658 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32); 2659 } 2660 2661 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 2662 SDLoc SL(In); 2663 return CurDAG->getConstant( 2664 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 2665 } 2666 2667 SDValue Src; 2668 if (isExtractHiElt(In, Src)) 2669 return Src; 2670 2671 return SDValue(); 2672 } 2673 2674 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const { 2675 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn); 2676 2677 const SIRegisterInfo *SIRI = 2678 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 2679 const SIInstrInfo * SII = 2680 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 2681 2682 unsigned Limit = 0; 2683 bool AllUsesAcceptSReg = true; 2684 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); 2685 Limit < 10 && U != E; ++U, ++Limit) { 2686 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); 2687 2688 // If the register class is unknown, it could be an unknown 2689 // register class that needs to be an SGPR, e.g. an inline asm 2690 // constraint 2691 if (!RC || SIRI->isSGPRClass(RC)) 2692 return false; 2693 2694 if (RC != &AMDGPU::VS_32RegClass) { 2695 AllUsesAcceptSReg = false; 2696 SDNode * User = *U; 2697 if (User->isMachineOpcode()) { 2698 unsigned Opc = User->getMachineOpcode(); 2699 MCInstrDesc Desc = SII->get(Opc); 2700 if (Desc.isCommutable()) { 2701 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo(); 2702 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 2703 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) { 2704 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs(); 2705 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo); 2706 if (CommutedRC == &AMDGPU::VS_32RegClass) 2707 AllUsesAcceptSReg = true; 2708 } 2709 } 2710 } 2711 // If "AllUsesAcceptSReg == false" so far we haven't suceeded 2712 // commuting current user. This means have at least one use 2713 // that strictly require VGPR. Thus, we will not attempt to commute 2714 // other user instructions. 2715 if (!AllUsesAcceptSReg) 2716 break; 2717 } 2718 } 2719 return !AllUsesAcceptSReg && (Limit < 10); 2720 } 2721 2722 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const { 2723 auto Ld = cast<LoadSDNode>(N); 2724 2725 return Ld->getAlignment() >= 4 && 2726 ( 2727 ( 2728 ( 2729 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2730 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT 2731 ) 2732 && 2733 !N->isDivergent() 2734 ) 2735 || 2736 ( 2737 Subtarget->getScalarizeGlobalBehavior() && 2738 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && 2739 !Ld->isVolatile() && 2740 !N->isDivergent() && 2741 static_cast<const SITargetLowering *>( 2742 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N) 2743 ) 2744 ); 2745 } 2746 2747 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 2748 const AMDGPUTargetLowering& Lowering = 2749 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 2750 bool IsModified = false; 2751 do { 2752 IsModified = false; 2753 2754 // Go over all selected nodes and try to fold them a bit more 2755 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 2756 while (Position != CurDAG->allnodes_end()) { 2757 SDNode *Node = &*Position++; 2758 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 2759 if (!MachineNode) 2760 continue; 2761 2762 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 2763 if (ResNode != Node) { 2764 if (ResNode) 2765 ReplaceUses(Node, ResNode); 2766 IsModified = true; 2767 } 2768 } 2769 CurDAG->RemoveDeadNodes(); 2770 } while (IsModified); 2771 } 2772 2773 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 2774 Subtarget = &MF.getSubtarget<R600Subtarget>(); 2775 return SelectionDAGISel::runOnMachineFunction(MF); 2776 } 2777 2778 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 2779 if (!N->readMem()) 2780 return false; 2781 if (CbId == -1) 2782 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2783 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 2784 2785 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; 2786 } 2787 2788 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 2789 SDValue& IntPtr) { 2790 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 2791 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 2792 true); 2793 return true; 2794 } 2795 return false; 2796 } 2797 2798 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 2799 SDValue& BaseReg, SDValue &Offset) { 2800 if (!isa<ConstantSDNode>(Addr)) { 2801 BaseReg = Addr; 2802 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 2803 return true; 2804 } 2805 return false; 2806 } 2807 2808 void R600DAGToDAGISel::Select(SDNode *N) { 2809 unsigned int Opc = N->getOpcode(); 2810 if (N->isMachineOpcode()) { 2811 N->setNodeId(-1); 2812 return; // Already selected. 2813 } 2814 2815 switch (Opc) { 2816 default: break; 2817 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 2818 case ISD::SCALAR_TO_VECTOR: 2819 case ISD::BUILD_VECTOR: { 2820 EVT VT = N->getValueType(0); 2821 unsigned NumVectorElts = VT.getVectorNumElements(); 2822 unsigned RegClassID; 2823 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 2824 // that adds a 128 bits reg copy when going through TwoAddressInstructions 2825 // pass. We want to avoid 128 bits copies as much as possible because they 2826 // can't be bundled by our scheduler. 2827 switch(NumVectorElts) { 2828 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 2829 case 4: 2830 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 2831 RegClassID = R600::R600_Reg128VerticalRegClassID; 2832 else 2833 RegClassID = R600::R600_Reg128RegClassID; 2834 break; 2835 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 2836 } 2837 SelectBuildVector(N, RegClassID); 2838 return; 2839 } 2840 } 2841 2842 SelectCode(N); 2843 } 2844 2845 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 2846 SDValue &Offset) { 2847 ConstantSDNode *C; 2848 SDLoc DL(Addr); 2849 2850 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 2851 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2852 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2853 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 2854 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 2855 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2856 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2857 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 2858 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 2859 Base = Addr.getOperand(0); 2860 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2861 } else { 2862 Base = Addr; 2863 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2864 } 2865 2866 return true; 2867 } 2868 2869 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 2870 SDValue &Offset) { 2871 ConstantSDNode *IMMOffset; 2872 2873 if (Addr.getOpcode() == ISD::ADD 2874 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 2875 && isInt<16>(IMMOffset->getZExtValue())) { 2876 2877 Base = Addr.getOperand(0); 2878 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2879 MVT::i32); 2880 return true; 2881 // If the pointer address is constant, we can move it to the offset field. 2882 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 2883 && isInt<16>(IMMOffset->getZExtValue())) { 2884 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 2885 SDLoc(CurDAG->getEntryNode()), 2886 R600::ZERO, MVT::i32); 2887 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2888 MVT::i32); 2889 return true; 2890 } 2891 2892 // Default case, no offset 2893 Base = Addr; 2894 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 2895 return true; 2896 } 2897