1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //==-----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines an instruction selector for the AMDGPU target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUArgumentUsageInfo.h" 16 #include "AMDGPUISelLowering.h" // For AMDGPUISD 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUPerfHintAnalysis.h" 19 #include "AMDGPUSubtarget.h" 20 #include "AMDGPUTargetMachine.h" 21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 22 #include "SIDefines.h" 23 #include "SIISelLowering.h" 24 #include "SIInstrInfo.h" 25 #include "SIMachineFunctionInfo.h" 26 #include "SIRegisterInfo.h" 27 #include "llvm/ADT/APInt.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 31 #include "llvm/Analysis/ValueTracking.h" 32 #include "llvm/CodeGen/FunctionLoweringInfo.h" 33 #include "llvm/CodeGen/ISDOpcodes.h" 34 #include "llvm/CodeGen/MachineFunction.h" 35 #include "llvm/CodeGen/MachineRegisterInfo.h" 36 #include "llvm/CodeGen/SelectionDAG.h" 37 #include "llvm/CodeGen/SelectionDAGISel.h" 38 #include "llvm/CodeGen/SelectionDAGNodes.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/IR/BasicBlock.h" 41 #include "llvm/InitializePasses.h" 42 #ifdef EXPENSIVE_CHECKS 43 #include "llvm/IR/Dominators.h" 44 #endif 45 #include "llvm/IR/Instruction.h" 46 #include "llvm/MC/MCInstrDesc.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/CodeGen.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MachineValueType.h" 51 #include "llvm/Support/MathExtras.h" 52 #include <cassert> 53 #include <cstdint> 54 #include <new> 55 #include <vector> 56 57 #define DEBUG_TYPE "isel" 58 59 using namespace llvm; 60 61 namespace llvm { 62 63 class R600InstrInfo; 64 65 } // end namespace llvm 66 67 //===----------------------------------------------------------------------===// 68 // Instruction Selector Implementation 69 //===----------------------------------------------------------------------===// 70 71 namespace { 72 73 static bool isNullConstantOrUndef(SDValue V) { 74 if (V.isUndef()) 75 return true; 76 77 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 78 return Const != nullptr && Const->isNullValue(); 79 } 80 81 static bool getConstantValue(SDValue N, uint32_t &Out) { 82 // This is only used for packed vectors, where ussing 0 for undef should 83 // always be good. 84 if (N.isUndef()) { 85 Out = 0; 86 return true; 87 } 88 89 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 90 Out = C->getAPIntValue().getSExtValue(); 91 return true; 92 } 93 94 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 95 Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); 96 return true; 97 } 98 99 return false; 100 } 101 102 // TODO: Handle undef as zero 103 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG, 104 bool Negate = false) { 105 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); 106 uint32_t LHSVal, RHSVal; 107 if (getConstantValue(N->getOperand(0), LHSVal) && 108 getConstantValue(N->getOperand(1), RHSVal)) { 109 SDLoc SL(N); 110 uint32_t K = Negate ? 111 (-LHSVal & 0xffff) | (-RHSVal << 16) : 112 (LHSVal & 0xffff) | (RHSVal << 16); 113 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0), 114 DAG.getTargetConstant(K, SL, MVT::i32)); 115 } 116 117 return nullptr; 118 } 119 120 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) { 121 return packConstantV2I16(N, DAG, true); 122 } 123 124 /// AMDGPU specific code to select AMDGPU machine instructions for 125 /// SelectionDAG operations. 126 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 127 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 128 // make the right decision when generating code for different targets. 129 const GCNSubtarget *Subtarget; 130 131 // Default FP mode for the current function. 132 AMDGPU::SIModeRegisterDefaults Mode; 133 134 bool EnableLateStructurizeCFG; 135 136 public: 137 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 138 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 139 : SelectionDAGISel(*TM, OptLevel) { 140 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 141 } 142 ~AMDGPUDAGToDAGISel() override = default; 143 144 void getAnalysisUsage(AnalysisUsage &AU) const override { 145 AU.addRequired<AMDGPUArgumentUsageInfo>(); 146 AU.addRequired<LegacyDivergenceAnalysis>(); 147 #ifdef EXPENSIVE_CHECKS 148 AU.addRequired<DominatorTreeWrapperPass>(); 149 AU.addRequired<LoopInfoWrapperPass>(); 150 #endif 151 SelectionDAGISel::getAnalysisUsage(AU); 152 } 153 154 bool matchLoadD16FromBuildVector(SDNode *N) const; 155 156 bool runOnMachineFunction(MachineFunction &MF) override; 157 void PreprocessISelDAG() override; 158 void Select(SDNode *N) override; 159 StringRef getPassName() const override; 160 void PostprocessISelDAG() override; 161 162 protected: 163 void SelectBuildVector(SDNode *N, unsigned RegClassID); 164 165 private: 166 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 167 bool isNoNanSrc(SDValue N) const; 168 bool isInlineImmediate(const SDNode *N, bool Negated = false) const; 169 bool isNegInlineImmediate(const SDNode *N) const { 170 return isInlineImmediate(N, true); 171 } 172 173 bool isInlineImmediate16(int64_t Imm) const { 174 return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); 175 } 176 177 bool isInlineImmediate32(int64_t Imm) const { 178 return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); 179 } 180 181 bool isInlineImmediate64(int64_t Imm) const { 182 return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); 183 } 184 185 bool isInlineImmediate(const APFloat &Imm) const { 186 return Subtarget->getInstrInfo()->isInlineConstant(Imm); 187 } 188 189 bool isVGPRImm(const SDNode *N) const; 190 bool isUniformLoad(const SDNode *N) const; 191 bool isUniformBr(const SDNode *N) const; 192 193 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; 194 195 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; 196 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; 197 SDNode *glueCopyToM0LDSInit(SDNode *N) const; 198 199 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 200 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 201 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 202 bool isDSOffsetLegal(SDValue Base, unsigned Offset, 203 unsigned OffsetBits) const; 204 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 205 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 206 SDValue &Offset1) const; 207 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 208 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 209 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC, 210 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 211 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 212 SDValue &SOffset, SDValue &Offset, SDValue &GLC, 213 SDValue &SLC, SDValue &TFE, SDValue &DLC, 214 SDValue &SWZ) const; 215 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 216 SDValue &VAddr, SDValue &SOffset, SDValue &Offset, 217 SDValue &SLC) const; 218 bool SelectMUBUFScratchOffen(SDNode *Parent, 219 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 220 SDValue &SOffset, SDValue &ImmOffset) const; 221 bool SelectMUBUFScratchOffset(SDNode *Parent, 222 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 223 SDValue &Offset) const; 224 225 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset, 226 SDValue &Offset, SDValue &GLC, SDValue &SLC, 227 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 228 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 229 SDValue &Offset, SDValue &SLC) const; 230 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 231 SDValue &Offset) const; 232 233 template <bool IsSigned> 234 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 235 SDValue &Offset, SDValue &SLC) const; 236 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr, 237 SDValue &Offset, SDValue &SLC) const; 238 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr, 239 SDValue &Offset, SDValue &SLC) const; 240 241 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 242 bool &Imm) const; 243 SDValue Expand32BitAddress(SDValue Addr) const; 244 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 245 bool &Imm) const; 246 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 247 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 248 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 249 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 250 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 251 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 252 253 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 254 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const; 255 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 256 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 257 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 258 SDValue &Clamp, SDValue &Omod) const; 259 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 260 SDValue &Clamp, SDValue &Omod) const; 261 262 bool SelectVOP3OMods(SDValue In, SDValue &Src, 263 SDValue &Clamp, SDValue &Omod) const; 264 265 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 266 267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 268 269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 271 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 272 273 SDValue getHi16Elt(SDValue In) const; 274 275 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; 276 277 void SelectADD_SUB_I64(SDNode *N); 278 void SelectAddcSubb(SDNode *N); 279 void SelectUADDO_USUBO(SDNode *N); 280 void SelectDIV_SCALE(SDNode *N); 281 void SelectDIV_FMAS(SDNode *N); 282 void SelectMAD_64_32(SDNode *N); 283 void SelectFMA_W_CHAIN(SDNode *N); 284 void SelectFMUL_W_CHAIN(SDNode *N); 285 286 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 287 uint32_t Offset, uint32_t Width); 288 void SelectS_BFEFromShifts(SDNode *N); 289 void SelectS_BFE(SDNode *N); 290 bool isCBranchSCC(const SDNode *N) const; 291 void SelectBRCOND(SDNode *N); 292 void SelectFMAD_FMA(SDNode *N); 293 void SelectATOMIC_CMP_SWAP(SDNode *N); 294 void SelectDSAppendConsume(SDNode *N, unsigned IntrID); 295 void SelectDS_GWS(SDNode *N, unsigned IntrID); 296 void SelectInterpP1F16(SDNode *N); 297 void SelectINTRINSIC_W_CHAIN(SDNode *N); 298 void SelectINTRINSIC_WO_CHAIN(SDNode *N); 299 void SelectINTRINSIC_VOID(SDNode *N); 300 301 protected: 302 // Include the pieces autogenerated from the target description. 303 #include "AMDGPUGenDAGISel.inc" 304 }; 305 306 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 307 const R600Subtarget *Subtarget; 308 309 bool isConstantLoad(const MemSDNode *N, int cbID) const; 310 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 311 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 312 SDValue& Offset); 313 public: 314 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 315 AMDGPUDAGToDAGISel(TM, OptLevel) {} 316 317 void Select(SDNode *N) override; 318 319 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 320 SDValue &Offset) override; 321 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 322 SDValue &Offset) override; 323 324 bool runOnMachineFunction(MachineFunction &MF) override; 325 326 void PreprocessISelDAG() override {} 327 328 protected: 329 // Include the pieces autogenerated from the target description. 330 #include "R600GenDAGISel.inc" 331 }; 332 333 static SDValue stripBitcast(SDValue Val) { 334 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 335 } 336 337 // Figure out if this is really an extract of the high 16-bits of a dword. 338 static bool isExtractHiElt(SDValue In, SDValue &Out) { 339 In = stripBitcast(In); 340 if (In.getOpcode() != ISD::TRUNCATE) 341 return false; 342 343 SDValue Srl = In.getOperand(0); 344 if (Srl.getOpcode() == ISD::SRL) { 345 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 346 if (ShiftAmt->getZExtValue() == 16) { 347 Out = stripBitcast(Srl.getOperand(0)); 348 return true; 349 } 350 } 351 } 352 353 return false; 354 } 355 356 // Look through operations that obscure just looking at the low 16-bits of the 357 // same register. 358 static SDValue stripExtractLoElt(SDValue In) { 359 if (In.getOpcode() == ISD::TRUNCATE) { 360 SDValue Src = In.getOperand(0); 361 if (Src.getValueType().getSizeInBits() == 32) 362 return stripBitcast(Src); 363 } 364 365 return In; 366 } 367 368 } // end anonymous namespace 369 370 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel", 371 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 372 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 373 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 374 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) 375 #ifdef EXPENSIVE_CHECKS 376 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 377 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 378 #endif 379 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel", 380 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 381 382 /// This pass converts a legalized DAG into a AMDGPU-specific 383 // DAG, ready for instruction scheduling. 384 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 385 CodeGenOpt::Level OptLevel) { 386 return new AMDGPUDAGToDAGISel(TM, OptLevel); 387 } 388 389 /// This pass converts a legalized DAG into a R600-specific 390 // DAG, ready for instruction scheduling. 391 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 392 CodeGenOpt::Level OptLevel) { 393 return new R600DAGToDAGISel(TM, OptLevel); 394 } 395 396 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 397 #ifdef EXPENSIVE_CHECKS 398 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 399 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 400 for (auto &L : LI->getLoopsInPreorder()) { 401 assert(L->isLCSSAForm(DT)); 402 } 403 #endif 404 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 405 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction(), *Subtarget); 406 return SelectionDAGISel::runOnMachineFunction(MF); 407 } 408 409 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const { 410 assert(Subtarget->d16PreservesUnusedBits()); 411 MVT VT = N->getValueType(0).getSimpleVT(); 412 if (VT != MVT::v2i16 && VT != MVT::v2f16) 413 return false; 414 415 SDValue Lo = N->getOperand(0); 416 SDValue Hi = N->getOperand(1); 417 418 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi)); 419 420 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo 421 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo 422 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo 423 424 // Need to check for possible indirect dependencies on the other half of the 425 // vector to avoid introducing a cycle. 426 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) { 427 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 428 429 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo); 430 SDValue Ops[] = { 431 LdHi->getChain(), LdHi->getBasePtr(), TiedIn 432 }; 433 434 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI; 435 if (LdHi->getMemoryVT() == MVT::i8) { 436 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ? 437 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8; 438 } else { 439 assert(LdHi->getMemoryVT() == MVT::i16); 440 } 441 442 SDValue NewLoadHi = 443 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList, 444 Ops, LdHi->getMemoryVT(), 445 LdHi->getMemOperand()); 446 447 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi); 448 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1)); 449 return true; 450 } 451 452 // build_vector (load ptr), hi -> load_d16_lo ptr, hi 453 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi 454 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi 455 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo)); 456 if (LdLo && Lo.hasOneUse()) { 457 SDValue TiedIn = getHi16Elt(Hi); 458 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode())) 459 return false; 460 461 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 462 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO; 463 if (LdLo->getMemoryVT() == MVT::i8) { 464 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ? 465 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8; 466 } else { 467 assert(LdLo->getMemoryVT() == MVT::i16); 468 } 469 470 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn); 471 472 SDValue Ops[] = { 473 LdLo->getChain(), LdLo->getBasePtr(), TiedIn 474 }; 475 476 SDValue NewLoadLo = 477 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList, 478 Ops, LdLo->getMemoryVT(), 479 LdLo->getMemOperand()); 480 481 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo); 482 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1)); 483 return true; 484 } 485 486 return false; 487 } 488 489 void AMDGPUDAGToDAGISel::PreprocessISelDAG() { 490 if (!Subtarget->d16PreservesUnusedBits()) 491 return; 492 493 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 494 495 bool MadeChange = false; 496 while (Position != CurDAG->allnodes_begin()) { 497 SDNode *N = &*--Position; 498 if (N->use_empty()) 499 continue; 500 501 switch (N->getOpcode()) { 502 case ISD::BUILD_VECTOR: 503 MadeChange |= matchLoadD16FromBuildVector(N); 504 break; 505 default: 506 break; 507 } 508 } 509 510 if (MadeChange) { 511 CurDAG->RemoveDeadNodes(); 512 LLVM_DEBUG(dbgs() << "After PreProcess:\n"; 513 CurDAG->dump();); 514 } 515 } 516 517 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 518 if (TM.Options.NoNaNsFPMath) 519 return true; 520 521 // TODO: Move into isKnownNeverNaN 522 if (N->getFlags().isDefined()) 523 return N->getFlags().hasNoNaNs(); 524 525 return CurDAG->isKnownNeverNaN(N); 526 } 527 528 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N, 529 bool Negated) const { 530 if (N->isUndef()) 531 return true; 532 533 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 534 if (Negated) { 535 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 536 return TII->isInlineConstant(-C->getAPIntValue()); 537 538 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 539 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt()); 540 541 } else { 542 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 543 return TII->isInlineConstant(C->getAPIntValue()); 544 545 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 546 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 547 } 548 549 return false; 550 } 551 552 /// Determine the register class for \p OpNo 553 /// \returns The register class of the virtual register that will be used for 554 /// the given operand number \OpNo or NULL if the register class cannot be 555 /// determined. 556 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 557 unsigned OpNo) const { 558 if (!N->isMachineOpcode()) { 559 if (N->getOpcode() == ISD::CopyToReg) { 560 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 561 if (Register::isVirtualRegister(Reg)) { 562 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 563 return MRI.getRegClass(Reg); 564 } 565 566 const SIRegisterInfo *TRI 567 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 568 return TRI->getPhysRegClass(Reg); 569 } 570 571 return nullptr; 572 } 573 574 switch (N->getMachineOpcode()) { 575 default: { 576 const MCInstrDesc &Desc = 577 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 578 unsigned OpIdx = Desc.getNumDefs() + OpNo; 579 if (OpIdx >= Desc.getNumOperands()) 580 return nullptr; 581 int RegClass = Desc.OpInfo[OpIdx].RegClass; 582 if (RegClass == -1) 583 return nullptr; 584 585 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 586 } 587 case AMDGPU::REG_SEQUENCE: { 588 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 589 const TargetRegisterClass *SuperRC = 590 Subtarget->getRegisterInfo()->getRegClass(RCID); 591 592 SDValue SubRegOp = N->getOperand(OpNo + 1); 593 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 594 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 595 SubRegIdx); 596 } 597 } 598 } 599 600 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain, 601 SDValue Glue) const { 602 SmallVector <SDValue, 8> Ops; 603 Ops.push_back(NewChain); // Replace the chain. 604 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 605 Ops.push_back(N->getOperand(i)); 606 607 Ops.push_back(Glue); 608 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 609 } 610 611 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const { 612 const SITargetLowering& Lowering = 613 *static_cast<const SITargetLowering*>(getTargetLowering()); 614 615 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain"); 616 617 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val); 618 return glueCopyToOp(N, M0, M0.getValue(1)); 619 } 620 621 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const { 622 unsigned AS = cast<MemSDNode>(N)->getAddressSpace(); 623 if (AS == AMDGPUAS::LOCAL_ADDRESS) { 624 if (Subtarget->ldsRequiresM0Init()) 625 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 626 } else if (AS == AMDGPUAS::REGION_ADDRESS) { 627 MachineFunction &MF = CurDAG->getMachineFunction(); 628 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize(); 629 return 630 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32)); 631 } 632 return N; 633 } 634 635 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm, 636 EVT VT) const { 637 SDNode *Lo = CurDAG->getMachineNode( 638 AMDGPU::S_MOV_B32, DL, MVT::i32, 639 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32)); 640 SDNode *Hi = 641 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 642 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32)); 643 const SDValue Ops[] = { 644 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 645 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 646 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)}; 647 648 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops); 649 } 650 651 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) { 652 switch (NumVectorElts) { 653 case 1: 654 return AMDGPU::SReg_32RegClassID; 655 case 2: 656 return AMDGPU::SReg_64RegClassID; 657 case 3: 658 return AMDGPU::SGPR_96RegClassID; 659 case 4: 660 return AMDGPU::SGPR_128RegClassID; 661 case 5: 662 return AMDGPU::SGPR_160RegClassID; 663 case 8: 664 return AMDGPU::SReg_256RegClassID; 665 case 16: 666 return AMDGPU::SReg_512RegClassID; 667 case 32: 668 return AMDGPU::SReg_1024RegClassID; 669 } 670 671 llvm_unreachable("invalid vector size"); 672 } 673 674 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 675 EVT VT = N->getValueType(0); 676 unsigned NumVectorElts = VT.getVectorNumElements(); 677 EVT EltVT = VT.getVectorElementType(); 678 SDLoc DL(N); 679 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 680 681 if (NumVectorElts == 1) { 682 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 683 RegClass); 684 return; 685 } 686 687 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not " 688 "supported yet"); 689 // 32 = Max Num Vector Elements 690 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 691 // 1 = Vector Register Class 692 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 693 694 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() == 695 Triple::amdgcn; 696 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 697 bool IsRegSeq = true; 698 unsigned NOps = N->getNumOperands(); 699 for (unsigned i = 0; i < NOps; i++) { 700 // XXX: Why is this here? 701 if (isa<RegisterSDNode>(N->getOperand(i))) { 702 IsRegSeq = false; 703 break; 704 } 705 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 706 : R600RegisterInfo::getSubRegFromChannel(i); 707 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 708 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 709 } 710 if (NOps != NumVectorElts) { 711 // Fill in the missing undef elements if this was a scalar_to_vector. 712 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 713 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 714 DL, EltVT); 715 for (unsigned i = NOps; i < NumVectorElts; ++i) { 716 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 717 : R600RegisterInfo::getSubRegFromChannel(i); 718 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 719 RegSeqArgs[1 + (2 * i) + 1] = 720 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 721 } 722 } 723 724 if (!IsRegSeq) 725 SelectCode(N); 726 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 727 } 728 729 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 730 unsigned int Opc = N->getOpcode(); 731 if (N->isMachineOpcode()) { 732 N->setNodeId(-1); 733 return; // Already selected. 734 } 735 736 // isa<MemSDNode> almost works but is slightly too permissive for some DS 737 // intrinsics. 738 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) || 739 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 740 Opc == ISD::ATOMIC_LOAD_FADD || 741 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 742 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) { 743 N = glueCopyToM0LDSInit(N); 744 SelectCode(N); 745 return; 746 } 747 748 switch (Opc) { 749 default: 750 break; 751 // We are selecting i64 ADD here instead of custom lower it during 752 // DAG legalization, so we can fold some i64 ADDs used for address 753 // calculation into the LOAD and STORE instructions. 754 case ISD::ADDC: 755 case ISD::ADDE: 756 case ISD::SUBC: 757 case ISD::SUBE: { 758 if (N->getValueType(0) != MVT::i64) 759 break; 760 761 SelectADD_SUB_I64(N); 762 return; 763 } 764 case ISD::ADDCARRY: 765 case ISD::SUBCARRY: 766 if (N->getValueType(0) != MVT::i32) 767 break; 768 769 SelectAddcSubb(N); 770 return; 771 case ISD::UADDO: 772 case ISD::USUBO: { 773 SelectUADDO_USUBO(N); 774 return; 775 } 776 case AMDGPUISD::FMUL_W_CHAIN: { 777 SelectFMUL_W_CHAIN(N); 778 return; 779 } 780 case AMDGPUISD::FMA_W_CHAIN: { 781 SelectFMA_W_CHAIN(N); 782 return; 783 } 784 785 case ISD::SCALAR_TO_VECTOR: 786 case ISD::BUILD_VECTOR: { 787 EVT VT = N->getValueType(0); 788 unsigned NumVectorElts = VT.getVectorNumElements(); 789 if (VT.getScalarSizeInBits() == 16) { 790 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 791 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) { 792 ReplaceNode(N, Packed); 793 return; 794 } 795 } 796 797 break; 798 } 799 800 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 801 unsigned RegClassID = selectSGPRVectorRegClassID(NumVectorElts); 802 SelectBuildVector(N, RegClassID); 803 return; 804 } 805 case ISD::BUILD_PAIR: { 806 SDValue RC, SubReg0, SubReg1; 807 SDLoc DL(N); 808 if (N->getValueType(0) == MVT::i128) { 809 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32); 810 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 811 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 812 } else if (N->getValueType(0) == MVT::i64) { 813 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 814 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 815 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 816 } else { 817 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 818 } 819 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 820 N->getOperand(1), SubReg1 }; 821 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 822 N->getValueType(0), Ops)); 823 return; 824 } 825 826 case ISD::Constant: 827 case ISD::ConstantFP: { 828 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 829 break; 830 831 uint64_t Imm; 832 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 833 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 834 else { 835 ConstantSDNode *C = cast<ConstantSDNode>(N); 836 Imm = C->getZExtValue(); 837 } 838 839 SDLoc DL(N); 840 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0))); 841 return; 842 } 843 case AMDGPUISD::BFE_I32: 844 case AMDGPUISD::BFE_U32: { 845 // There is a scalar version available, but unlike the vector version which 846 // has a separate operand for the offset and width, the scalar version packs 847 // the width and offset into a single operand. Try to move to the scalar 848 // version if the offsets are constant, so that we can try to keep extended 849 // loads of kernel arguments in SGPRs. 850 851 // TODO: Technically we could try to pattern match scalar bitshifts of 852 // dynamic values, but it's probably not useful. 853 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 854 if (!Offset) 855 break; 856 857 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 858 if (!Width) 859 break; 860 861 bool Signed = Opc == AMDGPUISD::BFE_I32; 862 863 uint32_t OffsetVal = Offset->getZExtValue(); 864 uint32_t WidthVal = Width->getZExtValue(); 865 866 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 867 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 868 return; 869 } 870 case AMDGPUISD::DIV_SCALE: { 871 SelectDIV_SCALE(N); 872 return; 873 } 874 case AMDGPUISD::DIV_FMAS: { 875 SelectDIV_FMAS(N); 876 return; 877 } 878 case AMDGPUISD::MAD_I64_I32: 879 case AMDGPUISD::MAD_U64_U32: { 880 SelectMAD_64_32(N); 881 return; 882 } 883 case ISD::CopyToReg: { 884 const SITargetLowering& Lowering = 885 *static_cast<const SITargetLowering*>(getTargetLowering()); 886 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 887 break; 888 } 889 case ISD::AND: 890 case ISD::SRL: 891 case ISD::SRA: 892 case ISD::SIGN_EXTEND_INREG: 893 if (N->getValueType(0) != MVT::i32) 894 break; 895 896 SelectS_BFE(N); 897 return; 898 case ISD::BRCOND: 899 SelectBRCOND(N); 900 return; 901 case ISD::FMAD: 902 case ISD::FMA: 903 SelectFMAD_FMA(N); 904 return; 905 case AMDGPUISD::ATOMIC_CMP_SWAP: 906 SelectATOMIC_CMP_SWAP(N); 907 return; 908 case AMDGPUISD::CVT_PKRTZ_F16_F32: 909 case AMDGPUISD::CVT_PKNORM_I16_F32: 910 case AMDGPUISD::CVT_PKNORM_U16_F32: 911 case AMDGPUISD::CVT_PK_U16_U32: 912 case AMDGPUISD::CVT_PK_I16_I32: { 913 // Hack around using a legal type if f16 is illegal. 914 if (N->getValueType(0) == MVT::i32) { 915 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16; 916 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT), 917 { N->getOperand(0), N->getOperand(1) }); 918 SelectCode(N); 919 return; 920 } 921 922 break; 923 } 924 case ISD::INTRINSIC_W_CHAIN: { 925 SelectINTRINSIC_W_CHAIN(N); 926 return; 927 } 928 case ISD::INTRINSIC_WO_CHAIN: { 929 SelectINTRINSIC_WO_CHAIN(N); 930 return; 931 } 932 case ISD::INTRINSIC_VOID: { 933 SelectINTRINSIC_VOID(N); 934 return; 935 } 936 } 937 938 SelectCode(N); 939 } 940 941 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 942 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 943 const Instruction *Term = BB->getTerminator(); 944 return Term->getMetadata("amdgpu.uniform") || 945 Term->getMetadata("structurizecfg.uniform"); 946 } 947 948 StringRef AMDGPUDAGToDAGISel::getPassName() const { 949 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 950 } 951 952 //===----------------------------------------------------------------------===// 953 // Complex Patterns 954 //===----------------------------------------------------------------------===// 955 956 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 957 SDValue &Offset) { 958 return false; 959 } 960 961 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 962 SDValue &Offset) { 963 ConstantSDNode *C; 964 SDLoc DL(Addr); 965 966 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 967 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 968 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 969 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 970 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 971 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 972 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 973 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 974 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 975 Base = Addr.getOperand(0); 976 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 977 } else { 978 Base = Addr; 979 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 980 } 981 982 return true; 983 } 984 985 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val, 986 const SDLoc &DL) const { 987 SDNode *Mov = CurDAG->getMachineNode( 988 AMDGPU::S_MOV_B32, DL, MVT::i32, 989 CurDAG->getTargetConstant(Val, DL, MVT::i32)); 990 return SDValue(Mov, 0); 991 } 992 993 // FIXME: Should only handle addcarry/subcarry 994 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 995 SDLoc DL(N); 996 SDValue LHS = N->getOperand(0); 997 SDValue RHS = N->getOperand(1); 998 999 unsigned Opcode = N->getOpcode(); 1000 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 1001 bool ProduceCarry = 1002 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 1003 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 1004 1005 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1006 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1007 1008 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1009 DL, MVT::i32, LHS, Sub0); 1010 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1011 DL, MVT::i32, LHS, Sub1); 1012 1013 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1014 DL, MVT::i32, RHS, Sub0); 1015 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1016 DL, MVT::i32, RHS, Sub1); 1017 1018 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 1019 1020 static const unsigned OpcMap[2][2][2] = { 1021 {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32}, 1022 {AMDGPU::V_SUB_I32_e32, AMDGPU::V_ADD_I32_e32}}, 1023 {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32}, 1024 {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}}; 1025 1026 unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd]; 1027 unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd]; 1028 1029 SDNode *AddLo; 1030 if (!ConsumeCarry) { 1031 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 1032 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 1033 } else { 1034 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 1035 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 1036 } 1037 SDValue AddHiArgs[] = { 1038 SDValue(Hi0, 0), 1039 SDValue(Hi1, 0), 1040 SDValue(AddLo, 1) 1041 }; 1042 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 1043 1044 SDValue RegSequenceArgs[] = { 1045 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 1046 SDValue(AddLo,0), 1047 Sub0, 1048 SDValue(AddHi,0), 1049 Sub1, 1050 }; 1051 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1052 MVT::i64, RegSequenceArgs); 1053 1054 if (ProduceCarry) { 1055 // Replace the carry-use 1056 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 1057 } 1058 1059 // Replace the remaining uses. 1060 ReplaceNode(N, RegSequence); 1061 } 1062 1063 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) { 1064 SDLoc DL(N); 1065 SDValue LHS = N->getOperand(0); 1066 SDValue RHS = N->getOperand(1); 1067 SDValue CI = N->getOperand(2); 1068 1069 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64 1070 : AMDGPU::V_SUBB_U32_e64; 1071 CurDAG->SelectNodeTo( 1072 N, Opc, N->getVTList(), 1073 {LHS, RHS, CI, CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1074 } 1075 1076 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 1077 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 1078 // carry out despite the _i32 name. These were renamed in VI to _U32. 1079 // FIXME: We should probably rename the opcodes here. 1080 unsigned Opc = N->getOpcode() == ISD::UADDO ? 1081 AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 1082 1083 CurDAG->SelectNodeTo( 1084 N, Opc, N->getVTList(), 1085 {N->getOperand(0), N->getOperand(1), 1086 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1087 } 1088 1089 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 1090 SDLoc SL(N); 1091 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 1092 SDValue Ops[10]; 1093 1094 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 1095 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1096 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 1097 Ops[8] = N->getOperand(0); 1098 Ops[9] = N->getOperand(4); 1099 1100 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); 1101 } 1102 1103 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 1104 SDLoc SL(N); 1105 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 1106 SDValue Ops[8]; 1107 1108 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 1109 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1110 Ops[6] = N->getOperand(0); 1111 Ops[7] = N->getOperand(3); 1112 1113 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 1114 } 1115 1116 // We need to handle this here because tablegen doesn't support matching 1117 // instructions with multiple outputs. 1118 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 1119 SDLoc SL(N); 1120 EVT VT = N->getValueType(0); 1121 1122 assert(VT == MVT::f32 || VT == MVT::f64); 1123 1124 unsigned Opc 1125 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32; 1126 1127 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; 1128 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1129 } 1130 1131 void AMDGPUDAGToDAGISel::SelectDIV_FMAS(SDNode *N) { 1132 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 1133 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 1134 1135 SDLoc SL(N); 1136 EVT VT = N->getValueType(0); 1137 1138 assert(VT == MVT::f32 || VT == MVT::f64); 1139 1140 unsigned Opc 1141 = (VT == MVT::f64) ? AMDGPU::V_DIV_FMAS_F64 : AMDGPU::V_DIV_FMAS_F32; 1142 1143 SDValue CarryIn = N->getOperand(3); 1144 // V_DIV_FMAS implicitly reads VCC. 1145 SDValue VCC = CurDAG->getCopyToReg(CurDAG->getEntryNode(), SL, 1146 TRI->getVCC(), CarryIn, SDValue()); 1147 1148 SDValue Ops[10]; 1149 1150 SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]); 1151 SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]); 1152 SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]); 1153 1154 Ops[8] = VCC; 1155 Ops[9] = VCC.getValue(1); 1156 1157 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1158 } 1159 1160 // We need to handle this here because tablegen doesn't support matching 1161 // instructions with multiple outputs. 1162 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 1163 SDLoc SL(N); 1164 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 1165 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32; 1166 1167 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 1168 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 1169 Clamp }; 1170 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1171 } 1172 1173 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset, 1174 unsigned OffsetBits) const { 1175 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 1176 (OffsetBits == 8 && !isUInt<8>(Offset))) 1177 return false; 1178 1179 if (Subtarget->hasUsableDSOffset() || 1180 Subtarget->unsafeDSOffsetFoldingEnabled()) 1181 return true; 1182 1183 // On Southern Islands instruction with a negative base value and an offset 1184 // don't seem to work. 1185 return CurDAG->SignBitIsZero(Base); 1186 } 1187 1188 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 1189 SDValue &Offset) const { 1190 SDLoc DL(Addr); 1191 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1192 SDValue N0 = Addr.getOperand(0); 1193 SDValue N1 = Addr.getOperand(1); 1194 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1195 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) { 1196 // (add n0, c0) 1197 Base = N0; 1198 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1199 return true; 1200 } 1201 } else if (Addr.getOpcode() == ISD::SUB) { 1202 // sub C, x -> add (sub 0, x), C 1203 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1204 int64_t ByteOffset = C->getSExtValue(); 1205 if (isUInt<16>(ByteOffset)) { 1206 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1207 1208 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1209 // the known bits in isDSOffsetLegal. We need to emit the selected node 1210 // here, so this is thrown away. 1211 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1212 Zero, Addr.getOperand(1)); 1213 1214 if (isDSOffsetLegal(Sub, ByteOffset, 16)) { 1215 SmallVector<SDValue, 3> Opnds; 1216 Opnds.push_back(Zero); 1217 Opnds.push_back(Addr.getOperand(1)); 1218 1219 // FIXME: Select to VOP3 version for with-carry. 1220 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1221 if (Subtarget->hasAddNoCarry()) { 1222 SubOp = AMDGPU::V_SUB_U32_e64; 1223 Opnds.push_back( 1224 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1225 } 1226 1227 MachineSDNode *MachineSub = 1228 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1229 1230 Base = SDValue(MachineSub, 0); 1231 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 1232 return true; 1233 } 1234 } 1235 } 1236 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1237 // If we have a constant address, prefer to put the constant into the 1238 // offset. This can save moves to load the constant address since multiple 1239 // operations can share the zero base address register, and enables merging 1240 // into read2 / write2 instructions. 1241 1242 SDLoc DL(Addr); 1243 1244 if (isUInt<16>(CAddr->getZExtValue())) { 1245 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1246 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1247 DL, MVT::i32, Zero); 1248 Base = SDValue(MovZero, 0); 1249 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1250 return true; 1251 } 1252 } 1253 1254 // default case 1255 Base = Addr; 1256 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 1257 return true; 1258 } 1259 1260 // TODO: If offset is too big, put low 16-bit into offset. 1261 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 1262 SDValue &Offset0, 1263 SDValue &Offset1) const { 1264 SDLoc DL(Addr); 1265 1266 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1267 SDValue N0 = Addr.getOperand(0); 1268 SDValue N1 = Addr.getOperand(1); 1269 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1270 unsigned DWordOffset0 = C1->getZExtValue() / 4; 1271 unsigned DWordOffset1 = DWordOffset0 + 1; 1272 // (add n0, c0) 1273 if (isDSOffsetLegal(N0, DWordOffset1, 8)) { 1274 Base = N0; 1275 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1276 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1277 return true; 1278 } 1279 } else if (Addr.getOpcode() == ISD::SUB) { 1280 // sub C, x -> add (sub 0, x), C 1281 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1282 unsigned DWordOffset0 = C->getZExtValue() / 4; 1283 unsigned DWordOffset1 = DWordOffset0 + 1; 1284 1285 if (isUInt<8>(DWordOffset0)) { 1286 SDLoc DL(Addr); 1287 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1288 1289 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1290 // the known bits in isDSOffsetLegal. We need to emit the selected node 1291 // here, so this is thrown away. 1292 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1293 Zero, Addr.getOperand(1)); 1294 1295 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) { 1296 SmallVector<SDValue, 3> Opnds; 1297 Opnds.push_back(Zero); 1298 Opnds.push_back(Addr.getOperand(1)); 1299 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1300 if (Subtarget->hasAddNoCarry()) { 1301 SubOp = AMDGPU::V_SUB_U32_e64; 1302 Opnds.push_back( 1303 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1304 } 1305 1306 MachineSDNode *MachineSub 1307 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1308 1309 Base = SDValue(MachineSub, 0); 1310 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1311 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1312 return true; 1313 } 1314 } 1315 } 1316 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1317 unsigned DWordOffset0 = CAddr->getZExtValue() / 4; 1318 unsigned DWordOffset1 = DWordOffset0 + 1; 1319 assert(4 * DWordOffset0 == CAddr->getZExtValue()); 1320 1321 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) { 1322 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1323 MachineSDNode *MovZero 1324 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1325 DL, MVT::i32, Zero); 1326 Base = SDValue(MovZero, 0); 1327 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1328 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1329 return true; 1330 } 1331 } 1332 1333 // default case 1334 1335 Base = Addr; 1336 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 1337 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 1338 return true; 1339 } 1340 1341 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, 1342 SDValue &VAddr, SDValue &SOffset, 1343 SDValue &Offset, SDValue &Offen, 1344 SDValue &Idxen, SDValue &Addr64, 1345 SDValue &GLC, SDValue &SLC, 1346 SDValue &TFE, SDValue &DLC, 1347 SDValue &SWZ) const { 1348 // Subtarget prefers to use flat instruction 1349 // FIXME: This should be a pattern predicate and not reach here 1350 if (Subtarget->useFlatForGlobal()) 1351 return false; 1352 1353 SDLoc DL(Addr); 1354 1355 if (!GLC.getNode()) 1356 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1357 if (!SLC.getNode()) 1358 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1359 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1); 1360 DLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1361 SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1); 1362 1363 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1364 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1365 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1366 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1367 1368 ConstantSDNode *C1 = nullptr; 1369 SDValue N0 = Addr; 1370 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1371 C1 = cast<ConstantSDNode>(Addr.getOperand(1)); 1372 if (isUInt<32>(C1->getZExtValue())) 1373 N0 = Addr.getOperand(0); 1374 else 1375 C1 = nullptr; 1376 } 1377 1378 if (N0.getOpcode() == ISD::ADD) { 1379 // (add N2, N3) -> addr64, or 1380 // (add (add N2, N3), C1) -> addr64 1381 SDValue N2 = N0.getOperand(0); 1382 SDValue N3 = N0.getOperand(1); 1383 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1384 1385 if (N2->isDivergent()) { 1386 if (N3->isDivergent()) { 1387 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 1388 // addr64, and construct the resource from a 0 address. 1389 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1390 VAddr = N0; 1391 } else { 1392 // N2 is divergent, N3 is not. 1393 Ptr = N3; 1394 VAddr = N2; 1395 } 1396 } else { 1397 // N2 is not divergent. 1398 Ptr = N2; 1399 VAddr = N3; 1400 } 1401 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1402 } else if (N0->isDivergent()) { 1403 // N0 is divergent. Use it as the addr64, and construct the resource from a 1404 // 0 address. 1405 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1406 VAddr = N0; 1407 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1408 } else { 1409 // N0 -> offset, or 1410 // (N0 + C1) -> offset 1411 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1412 Ptr = N0; 1413 } 1414 1415 if (!C1) { 1416 // No offset. 1417 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1418 return true; 1419 } 1420 1421 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1422 // Legal offset for instruction. 1423 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1424 return true; 1425 } 1426 1427 // Illegal offset, store it in soffset. 1428 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1429 SOffset = 1430 SDValue(CurDAG->getMachineNode( 1431 AMDGPU::S_MOV_B32, DL, MVT::i32, 1432 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1433 0); 1434 return true; 1435 } 1436 1437 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1438 SDValue &VAddr, SDValue &SOffset, 1439 SDValue &Offset, SDValue &GLC, 1440 SDValue &SLC, SDValue &TFE, 1441 SDValue &DLC, SDValue &SWZ) const { 1442 SDValue Ptr, Offen, Idxen, Addr64; 1443 1444 // addr64 bit was removed for volcanic islands. 1445 // FIXME: This should be a pattern predicate and not reach here 1446 if (!Subtarget->hasAddr64()) 1447 return false; 1448 1449 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1450 GLC, SLC, TFE, DLC, SWZ)) 1451 return false; 1452 1453 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1454 if (C->getSExtValue()) { 1455 SDLoc DL(Addr); 1456 1457 const SITargetLowering& Lowering = 1458 *static_cast<const SITargetLowering*>(getTargetLowering()); 1459 1460 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1461 return true; 1462 } 1463 1464 return false; 1465 } 1466 1467 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1468 SDValue &VAddr, SDValue &SOffset, 1469 SDValue &Offset, 1470 SDValue &SLC) const { 1471 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1); 1472 SDValue GLC, TFE, DLC, SWZ; 1473 1474 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1475 } 1476 1477 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 1478 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 1479 return PSV && PSV->isStack(); 1480 } 1481 1482 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1483 SDLoc DL(N); 1484 const MachineFunction &MF = CurDAG->getMachineFunction(); 1485 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1486 1487 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) { 1488 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1489 FI->getValueType(0)); 1490 1491 // If we can resolve this to a frame index access, this will be relative to 1492 // either the stack or frame pointer SGPR. 1493 return std::make_pair( 1494 TFI, CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)); 1495 } 1496 1497 // If we don't know this private access is a local stack object, it needs to 1498 // be relative to the entry point's scratch wave offset. 1499 return std::make_pair(N, CurDAG->getTargetConstant(0, DL, MVT::i32)); 1500 } 1501 1502 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1503 SDValue Addr, SDValue &Rsrc, 1504 SDValue &VAddr, SDValue &SOffset, 1505 SDValue &ImmOffset) const { 1506 1507 SDLoc DL(Addr); 1508 MachineFunction &MF = CurDAG->getMachineFunction(); 1509 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1510 1511 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1512 1513 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1514 unsigned Imm = CAddr->getZExtValue(); 1515 1516 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1517 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1518 DL, MVT::i32, HighBits); 1519 VAddr = SDValue(MovHighBits, 0); 1520 1521 // In a call sequence, stores to the argument stack area are relative to the 1522 // stack pointer. 1523 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1524 1525 SOffset = isStackPtrRelative(PtrInfo) 1526 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1527 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1528 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1529 return true; 1530 } 1531 1532 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1533 // (add n0, c1) 1534 1535 SDValue N0 = Addr.getOperand(0); 1536 SDValue N1 = Addr.getOperand(1); 1537 1538 // Offsets in vaddr must be positive if range checking is enabled. 1539 // 1540 // The total computation of vaddr + soffset + offset must not overflow. If 1541 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1542 // overflowing. 1543 // 1544 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1545 // always perform a range check. If a negative vaddr base index was used, 1546 // this would fail the range check. The overall address computation would 1547 // compute a valid address, but this doesn't happen due to the range 1548 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1549 // 1550 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1551 // MUBUF vaddr, but not on older subtargets which can only do this if the 1552 // sign bit is known 0. 1553 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1554 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1555 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1556 CurDAG->SignBitIsZero(N0))) { 1557 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1558 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1559 return true; 1560 } 1561 } 1562 1563 // (node) 1564 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1565 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1566 return true; 1567 } 1568 1569 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1570 SDValue Addr, 1571 SDValue &SRsrc, 1572 SDValue &SOffset, 1573 SDValue &Offset) const { 1574 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr); 1575 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1576 return false; 1577 1578 SDLoc DL(Addr); 1579 MachineFunction &MF = CurDAG->getMachineFunction(); 1580 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1581 1582 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1583 1584 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1585 1586 // FIXME: Get from MachinePointerInfo? We should only be using the frame 1587 // offset if we know this is in a call sequence. 1588 SOffset = isStackPtrRelative(PtrInfo) 1589 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1590 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1591 1592 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1593 return true; 1594 } 1595 1596 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1597 SDValue &SOffset, SDValue &Offset, 1598 SDValue &GLC, SDValue &SLC, 1599 SDValue &TFE, SDValue &DLC, 1600 SDValue &SWZ) const { 1601 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1602 const SIInstrInfo *TII = 1603 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1604 1605 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1606 GLC, SLC, TFE, DLC, SWZ)) 1607 return false; 1608 1609 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1610 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1611 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1612 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1613 APInt::getAllOnesValue(32).getZExtValue(); // Size 1614 SDLoc DL(Addr); 1615 1616 const SITargetLowering& Lowering = 1617 *static_cast<const SITargetLowering*>(getTargetLowering()); 1618 1619 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1620 return true; 1621 } 1622 return false; 1623 } 1624 1625 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1626 SDValue &Soffset, SDValue &Offset 1627 ) const { 1628 SDValue GLC, SLC, TFE, DLC, SWZ; 1629 1630 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1631 } 1632 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1633 SDValue &Soffset, SDValue &Offset, 1634 SDValue &SLC) const { 1635 SDValue GLC, TFE, DLC, SWZ; 1636 1637 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1638 } 1639 1640 // Find a load or store from corresponding pattern root. 1641 // Roots may be build_vector, bitconvert or their combinations. 1642 static MemSDNode* findMemSDNode(SDNode *N) { 1643 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode(); 1644 if (MemSDNode *MN = dyn_cast<MemSDNode>(N)) 1645 return MN; 1646 assert(isa<BuildVectorSDNode>(N)); 1647 for (SDValue V : N->op_values()) 1648 if (MemSDNode *MN = 1649 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V))) 1650 return MN; 1651 llvm_unreachable("cannot find MemSDNode in the pattern!"); 1652 } 1653 1654 template <bool IsSigned> 1655 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, 1656 SDValue Addr, 1657 SDValue &VAddr, 1658 SDValue &Offset, 1659 SDValue &SLC) const { 1660 int64_t OffsetVal = 0; 1661 1662 if (Subtarget->hasFlatInstOffsets() && 1663 (!Subtarget->hasFlatSegmentOffsetBug() || 1664 findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) && 1665 CurDAG->isBaseWithConstantOffset(Addr)) { 1666 SDValue N0 = Addr.getOperand(0); 1667 SDValue N1 = Addr.getOperand(1); 1668 uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1669 1670 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1671 unsigned AS = findMemSDNode(N)->getAddressSpace(); 1672 if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) { 1673 Addr = N0; 1674 OffsetVal = COffsetVal; 1675 } else { 1676 // If the offset doesn't fit, put the low bits into the offset field and 1677 // add the rest. 1678 1679 SDLoc DL(N); 1680 uint64_t ImmField; 1681 const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned); 1682 if (IsSigned) { 1683 ImmField = SignExtend64(COffsetVal, NumBits); 1684 1685 // Don't use a negative offset field if the base offset is positive. 1686 // Since the scheduler currently relies on the offset field, doing so 1687 // could result in strange scheduling decisions. 1688 1689 // TODO: Should we not do this in the opposite direction as well? 1690 if (static_cast<int64_t>(COffsetVal) > 0) { 1691 if (static_cast<int64_t>(ImmField) < 0) { 1692 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits - 1); 1693 ImmField = COffsetVal & OffsetMask; 1694 } 1695 } 1696 } else { 1697 // TODO: Should we do this for a negative offset? 1698 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits); 1699 ImmField = COffsetVal & OffsetMask; 1700 } 1701 1702 uint64_t RemainderOffset = COffsetVal - ImmField; 1703 1704 assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned)); 1705 assert(RemainderOffset + ImmField == COffsetVal); 1706 1707 OffsetVal = ImmField; 1708 1709 // TODO: Should this try to use a scalar add pseudo if the base address is 1710 // uniform and saddr is usable? 1711 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1712 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1713 1714 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1715 DL, MVT::i32, N0, Sub0); 1716 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1717 DL, MVT::i32, N0, Sub1); 1718 1719 SDValue AddOffsetLo 1720 = getMaterializedScalarImm32(Lo_32(RemainderOffset), DL); 1721 SDValue AddOffsetHi 1722 = getMaterializedScalarImm32(Hi_32(RemainderOffset), DL); 1723 1724 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1); 1725 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1726 1727 SDNode *Add = CurDAG->getMachineNode( 1728 AMDGPU::V_ADD_I32_e64, DL, VTs, 1729 {AddOffsetLo, SDValue(N0Lo, 0), Clamp}); 1730 1731 SDNode *Addc = CurDAG->getMachineNode( 1732 AMDGPU::V_ADDC_U32_e64, DL, VTs, 1733 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp}); 1734 1735 SDValue RegSequenceArgs[] = { 1736 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32), 1737 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1 1738 }; 1739 1740 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1741 MVT::i64, RegSequenceArgs), 0); 1742 } 1743 } 1744 1745 VAddr = Addr; 1746 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1747 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); 1748 return true; 1749 } 1750 1751 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDNode *N, 1752 SDValue Addr, 1753 SDValue &VAddr, 1754 SDValue &Offset, 1755 SDValue &SLC) const { 1756 return SelectFlatOffset<false>(N, Addr, VAddr, Offset, SLC); 1757 } 1758 1759 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N, 1760 SDValue Addr, 1761 SDValue &VAddr, 1762 SDValue &Offset, 1763 SDValue &SLC) const { 1764 return SelectFlatOffset<true>(N, Addr, VAddr, Offset, SLC); 1765 } 1766 1767 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 1768 SDValue &Offset, bool &Imm) const { 1769 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 1770 if (!C) { 1771 if (ByteOffsetNode.getValueType().isScalarInteger() && 1772 ByteOffsetNode.getValueType().getSizeInBits() == 32) { 1773 Offset = ByteOffsetNode; 1774 Imm = false; 1775 return true; 1776 } 1777 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) { 1778 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) { 1779 Offset = ByteOffsetNode.getOperand(0); 1780 Imm = false; 1781 return true; 1782 } 1783 } 1784 return false; 1785 } 1786 1787 SDLoc SL(ByteOffsetNode); 1788 GCNSubtarget::Generation Gen = Subtarget->getGeneration(); 1789 uint64_t ByteOffset = C->getZExtValue(); 1790 Optional<int64_t> EncodedOffset = 1791 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset); 1792 if (EncodedOffset) { 1793 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1794 Imm = true; 1795 return true; 1796 } 1797 1798 if (Gen == AMDGPUSubtarget::SEA_ISLANDS) { 1799 EncodedOffset = 1800 AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset); 1801 if (EncodedOffset) { 1802 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1803 return true; 1804 } 1805 } 1806 1807 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset)) 1808 return false; 1809 1810 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 1811 Offset = SDValue( 1812 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0); 1813 1814 return true; 1815 } 1816 1817 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 1818 if (Addr.getValueType() != MVT::i32) 1819 return Addr; 1820 1821 // Zero-extend a 32-bit address. 1822 SDLoc SL(Addr); 1823 1824 const MachineFunction &MF = CurDAG->getMachineFunction(); 1825 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1826 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 1827 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 1828 1829 const SDValue Ops[] = { 1830 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 1831 Addr, 1832 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 1833 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 1834 0), 1835 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 1836 }; 1837 1838 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 1839 Ops), 0); 1840 } 1841 1842 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 1843 SDValue &Offset, bool &Imm) const { 1844 SDLoc SL(Addr); 1845 1846 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 1847 // wraparound, because s_load instructions perform the addition in 64 bits. 1848 if ((Addr.getValueType() != MVT::i32 || 1849 Addr->getFlags().hasNoUnsignedWrap()) && 1850 (CurDAG->isBaseWithConstantOffset(Addr) || 1851 Addr.getOpcode() == ISD::ADD)) { 1852 SDValue N0 = Addr.getOperand(0); 1853 SDValue N1 = Addr.getOperand(1); 1854 1855 if (SelectSMRDOffset(N1, Offset, Imm)) { 1856 SBase = Expand32BitAddress(N0); 1857 return true; 1858 } 1859 } 1860 SBase = Expand32BitAddress(Addr); 1861 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 1862 Imm = true; 1863 return true; 1864 } 1865 1866 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 1867 SDValue &Offset) const { 1868 bool Imm = false; 1869 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 1870 } 1871 1872 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 1873 SDValue &Offset) const { 1874 1875 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1876 1877 bool Imm = false; 1878 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 1879 return false; 1880 1881 return !Imm && isa<ConstantSDNode>(Offset); 1882 } 1883 1884 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 1885 SDValue &Offset) const { 1886 bool Imm = false; 1887 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 1888 !isa<ConstantSDNode>(Offset); 1889 } 1890 1891 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 1892 SDValue &Offset) const { 1893 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1894 if (auto Imm = AMDGPU::getSMRDEncodedOffset(*Subtarget, 1895 C->getZExtValue())) { 1896 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1897 return true; 1898 } 1899 } 1900 1901 return false; 1902 } 1903 1904 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 1905 SDValue &Offset) const { 1906 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1907 1908 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1909 if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, 1910 C->getZExtValue())) { 1911 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1912 return true; 1913 } 1914 } 1915 1916 return false; 1917 } 1918 1919 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 1920 SDValue &Base, 1921 SDValue &Offset) const { 1922 SDLoc DL(Index); 1923 1924 if (CurDAG->isBaseWithConstantOffset(Index)) { 1925 SDValue N0 = Index.getOperand(0); 1926 SDValue N1 = Index.getOperand(1); 1927 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1928 1929 // (add n0, c0) 1930 // Don't peel off the offset (c0) if doing so could possibly lead 1931 // the base (n0) to be negative. 1932 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) { 1933 Base = N0; 1934 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 1935 return true; 1936 } 1937 } 1938 1939 if (isa<ConstantSDNode>(Index)) 1940 return false; 1941 1942 Base = Index; 1943 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1944 return true; 1945 } 1946 1947 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 1948 SDValue Val, uint32_t Offset, 1949 uint32_t Width) { 1950 // Transformation function, pack the offset and width of a BFE into 1951 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1952 // source, bits [5:0] contain the offset and bits [22:16] the width. 1953 uint32_t PackedVal = Offset | (Width << 16); 1954 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 1955 1956 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 1957 } 1958 1959 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 1960 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 1961 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 1962 // Predicate: 0 < b <= c < 32 1963 1964 const SDValue &Shl = N->getOperand(0); 1965 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 1966 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1967 1968 if (B && C) { 1969 uint32_t BVal = B->getZExtValue(); 1970 uint32_t CVal = C->getZExtValue(); 1971 1972 if (0 < BVal && BVal <= CVal && CVal < 32) { 1973 bool Signed = N->getOpcode() == ISD::SRA; 1974 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1975 1976 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 1977 32 - CVal)); 1978 return; 1979 } 1980 } 1981 SelectCode(N); 1982 } 1983 1984 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 1985 switch (N->getOpcode()) { 1986 case ISD::AND: 1987 if (N->getOperand(0).getOpcode() == ISD::SRL) { 1988 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 1989 // Predicate: isMask(mask) 1990 const SDValue &Srl = N->getOperand(0); 1991 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 1992 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1993 1994 if (Shift && Mask) { 1995 uint32_t ShiftVal = Shift->getZExtValue(); 1996 uint32_t MaskVal = Mask->getZExtValue(); 1997 1998 if (isMask_32(MaskVal)) { 1999 uint32_t WidthVal = countPopulation(MaskVal); 2000 2001 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2002 Srl.getOperand(0), ShiftVal, WidthVal)); 2003 return; 2004 } 2005 } 2006 } 2007 break; 2008 case ISD::SRL: 2009 if (N->getOperand(0).getOpcode() == ISD::AND) { 2010 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 2011 // Predicate: isMask(mask >> b) 2012 const SDValue &And = N->getOperand(0); 2013 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 2014 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 2015 2016 if (Shift && Mask) { 2017 uint32_t ShiftVal = Shift->getZExtValue(); 2018 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 2019 2020 if (isMask_32(MaskVal)) { 2021 uint32_t WidthVal = countPopulation(MaskVal); 2022 2023 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2024 And.getOperand(0), ShiftVal, WidthVal)); 2025 return; 2026 } 2027 } 2028 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 2029 SelectS_BFEFromShifts(N); 2030 return; 2031 } 2032 break; 2033 case ISD::SRA: 2034 if (N->getOperand(0).getOpcode() == ISD::SHL) { 2035 SelectS_BFEFromShifts(N); 2036 return; 2037 } 2038 break; 2039 2040 case ISD::SIGN_EXTEND_INREG: { 2041 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 2042 SDValue Src = N->getOperand(0); 2043 if (Src.getOpcode() != ISD::SRL) 2044 break; 2045 2046 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 2047 if (!Amt) 2048 break; 2049 2050 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2051 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 2052 Amt->getZExtValue(), Width)); 2053 return; 2054 } 2055 } 2056 2057 SelectCode(N); 2058 } 2059 2060 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 2061 assert(N->getOpcode() == ISD::BRCOND); 2062 if (!N->hasOneUse()) 2063 return false; 2064 2065 SDValue Cond = N->getOperand(1); 2066 if (Cond.getOpcode() == ISD::CopyToReg) 2067 Cond = Cond.getOperand(2); 2068 2069 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 2070 return false; 2071 2072 MVT VT = Cond.getOperand(0).getSimpleValueType(); 2073 if (VT == MVT::i32) 2074 return true; 2075 2076 if (VT == MVT::i64) { 2077 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 2078 2079 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 2080 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 2081 } 2082 2083 return false; 2084 } 2085 2086 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 2087 SDValue Cond = N->getOperand(1); 2088 2089 if (Cond.isUndef()) { 2090 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 2091 N->getOperand(2), N->getOperand(0)); 2092 return; 2093 } 2094 2095 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 2096 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 2097 2098 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 2099 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 2100 unsigned CondReg = UseSCCBr ? (unsigned)AMDGPU::SCC : TRI->getVCC(); 2101 SDLoc SL(N); 2102 2103 if (!UseSCCBr) { 2104 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 2105 // analyzed what generates the vcc value, so we do not know whether vcc 2106 // bits for disabled lanes are 0. Thus we need to mask out bits for 2107 // disabled lanes. 2108 // 2109 // For the case that we select S_CBRANCH_SCC1 and it gets 2110 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 2111 // SIInstrInfo::moveToVALU which inserts the S_AND). 2112 // 2113 // We could add an analysis of what generates the vcc value here and omit 2114 // the S_AND when is unnecessary. But it would be better to add a separate 2115 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 2116 // catches both cases. 2117 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32 2118 : AMDGPU::S_AND_B64, 2119 SL, MVT::i1, 2120 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO 2121 : AMDGPU::EXEC, 2122 MVT::i1), 2123 Cond), 2124 0); 2125 } 2126 2127 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 2128 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 2129 N->getOperand(2), // Basic Block 2130 VCC.getValue(0)); 2131 } 2132 2133 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 2134 MVT VT = N->getSimpleValueType(0); 2135 bool IsFMA = N->getOpcode() == ISD::FMA; 2136 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 2137 !Subtarget->hasFmaMixInsts()) || 2138 ((IsFMA && Subtarget->hasMadMixInsts()) || 2139 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 2140 SelectCode(N); 2141 return; 2142 } 2143 2144 SDValue Src0 = N->getOperand(0); 2145 SDValue Src1 = N->getOperand(1); 2146 SDValue Src2 = N->getOperand(2); 2147 unsigned Src0Mods, Src1Mods, Src2Mods; 2148 2149 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 2150 // using the conversion from f16. 2151 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 2152 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 2153 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 2154 2155 assert((IsFMA || !Mode.allFP32Denormals()) && 2156 "fmad selected with denormals enabled"); 2157 // TODO: We can select this with f32 denormals enabled if all the sources are 2158 // converted from f16 (in which case fmad isn't legal). 2159 2160 if (Sel0 || Sel1 || Sel2) { 2161 // For dummy operands. 2162 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2163 SDValue Ops[] = { 2164 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 2165 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 2166 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 2167 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 2168 Zero, Zero 2169 }; 2170 2171 CurDAG->SelectNodeTo(N, 2172 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 2173 MVT::f32, Ops); 2174 } else { 2175 SelectCode(N); 2176 } 2177 } 2178 2179 // This is here because there isn't a way to use the generated sub0_sub1 as the 2180 // subreg index to EXTRACT_SUBREG in tablegen. 2181 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 2182 MemSDNode *Mem = cast<MemSDNode>(N); 2183 unsigned AS = Mem->getAddressSpace(); 2184 if (AS == AMDGPUAS::FLAT_ADDRESS) { 2185 SelectCode(N); 2186 return; 2187 } 2188 2189 MVT VT = N->getSimpleValueType(0); 2190 bool Is32 = (VT == MVT::i32); 2191 SDLoc SL(N); 2192 2193 MachineSDNode *CmpSwap = nullptr; 2194 if (Subtarget->hasAddr64()) { 2195 SDValue SRsrc, VAddr, SOffset, Offset, SLC; 2196 2197 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) { 2198 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 2199 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 2200 SDValue CmpVal = Mem->getOperand(2); 2201 2202 // XXX - Do we care about glue operands? 2203 2204 SDValue Ops[] = { 2205 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2206 }; 2207 2208 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2209 } 2210 } 2211 2212 if (!CmpSwap) { 2213 SDValue SRsrc, SOffset, Offset, SLC; 2214 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) { 2215 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 2216 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 2217 2218 SDValue CmpVal = Mem->getOperand(2); 2219 SDValue Ops[] = { 2220 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2221 }; 2222 2223 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2224 } 2225 } 2226 2227 if (!CmpSwap) { 2228 SelectCode(N); 2229 return; 2230 } 2231 2232 MachineMemOperand *MMO = Mem->getMemOperand(); 2233 CurDAG->setNodeMemRefs(CmpSwap, {MMO}); 2234 2235 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 2236 SDValue Extract 2237 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 2238 2239 ReplaceUses(SDValue(N, 0), Extract); 2240 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 2241 CurDAG->RemoveDeadNode(N); 2242 } 2243 2244 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { 2245 // The address is assumed to be uniform, so if it ends up in a VGPR, it will 2246 // be copied to an SGPR with readfirstlane. 2247 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ? 2248 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 2249 2250 SDValue Chain = N->getOperand(0); 2251 SDValue Ptr = N->getOperand(2); 2252 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2253 MachineMemOperand *MMO = M->getMemOperand(); 2254 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 2255 2256 SDValue Offset; 2257 if (CurDAG->isBaseWithConstantOffset(Ptr)) { 2258 SDValue PtrBase = Ptr.getOperand(0); 2259 SDValue PtrOffset = Ptr.getOperand(1); 2260 2261 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue(); 2262 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) { 2263 N = glueCopyToM0(N, PtrBase); 2264 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32); 2265 } 2266 } 2267 2268 if (!Offset) { 2269 N = glueCopyToM0(N, Ptr); 2270 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2271 } 2272 2273 SDValue Ops[] = { 2274 Offset, 2275 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32), 2276 Chain, 2277 N->getOperand(N->getNumOperands() - 1) // New glue 2278 }; 2279 2280 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2281 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2282 } 2283 2284 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 2285 switch (IntrID) { 2286 case Intrinsic::amdgcn_ds_gws_init: 2287 return AMDGPU::DS_GWS_INIT; 2288 case Intrinsic::amdgcn_ds_gws_barrier: 2289 return AMDGPU::DS_GWS_BARRIER; 2290 case Intrinsic::amdgcn_ds_gws_sema_v: 2291 return AMDGPU::DS_GWS_SEMA_V; 2292 case Intrinsic::amdgcn_ds_gws_sema_br: 2293 return AMDGPU::DS_GWS_SEMA_BR; 2294 case Intrinsic::amdgcn_ds_gws_sema_p: 2295 return AMDGPU::DS_GWS_SEMA_P; 2296 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2297 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 2298 default: 2299 llvm_unreachable("not a gws intrinsic"); 2300 } 2301 } 2302 2303 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { 2304 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all && 2305 !Subtarget->hasGWSSemaReleaseAll()) { 2306 // Let this error. 2307 SelectCode(N); 2308 return; 2309 } 2310 2311 // Chain, intrinsic ID, vsrc, offset 2312 const bool HasVSrc = N->getNumOperands() == 4; 2313 assert(HasVSrc || N->getNumOperands() == 3); 2314 2315 SDLoc SL(N); 2316 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2); 2317 int ImmOffset = 0; 2318 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2319 MachineMemOperand *MMO = M->getMemOperand(); 2320 2321 // Don't worry if the offset ends up in a VGPR. Only one lane will have 2322 // effect, so SIFixSGPRCopies will validly insert readfirstlane. 2323 2324 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 2325 // offset field) % 64. Some versions of the programming guide omit the m0 2326 // part, or claim it's from offset 0. 2327 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) { 2328 // If we have a constant offset, try to use the 0 in m0 as the base. 2329 // TODO: Look into changing the default m0 initialization value. If the 2330 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 2331 // the immediate offset. 2332 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); 2333 ImmOffset = ConstOffset->getZExtValue(); 2334 } else { 2335 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { 2336 ImmOffset = BaseOffset.getConstantOperandVal(1); 2337 BaseOffset = BaseOffset.getOperand(0); 2338 } 2339 2340 // Prefer to do the shift in an SGPR since it should be possible to use m0 2341 // as the result directly. If it's already an SGPR, it will be eliminated 2342 // later. 2343 SDNode *SGPROffset 2344 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, 2345 BaseOffset); 2346 // Shift to offset in m0 2347 SDNode *M0Base 2348 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32, 2349 SDValue(SGPROffset, 0), 2350 CurDAG->getTargetConstant(16, SL, MVT::i32)); 2351 glueCopyToM0(N, SDValue(M0Base, 0)); 2352 } 2353 2354 SDValue Chain = N->getOperand(0); 2355 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32); 2356 2357 // TODO: Can this just be removed from the instruction? 2358 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1); 2359 2360 const unsigned Opc = gwsIntrinToOpcode(IntrID); 2361 SmallVector<SDValue, 5> Ops; 2362 if (HasVSrc) 2363 Ops.push_back(N->getOperand(2)); 2364 Ops.push_back(OffsetField); 2365 Ops.push_back(GDS); 2366 Ops.push_back(Chain); 2367 2368 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2369 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2370 } 2371 2372 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) { 2373 if (Subtarget->getLDSBankCount() != 16) { 2374 // This is a single instruction with a pattern. 2375 SelectCode(N); 2376 return; 2377 } 2378 2379 SDLoc DL(N); 2380 2381 // This requires 2 instructions. It is possible to write a pattern to support 2382 // this, but the generated isel emitter doesn't correctly deal with multiple 2383 // output instructions using the same physical register input. The copy to m0 2384 // is incorrectly placed before the second instruction. 2385 // 2386 // TODO: Match source modifiers. 2387 // 2388 // def : Pat < 2389 // (int_amdgcn_interp_p1_f16 2390 // (VOP3Mods f32:$src0, i32:$src0_modifiers), 2391 // (i32 timm:$attrchan), (i32 timm:$attr), 2392 // (i1 timm:$high), M0), 2393 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr, 2394 // timm:$attrchan, 0, 2395 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> { 2396 // let Predicates = [has16BankLDS]; 2397 // } 2398 2399 // 16 bank LDS 2400 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0, 2401 N->getOperand(5), SDValue()); 2402 2403 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other); 2404 2405 SDNode *InterpMov = 2406 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, { 2407 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0 2408 N->getOperand(3), // Attr 2409 N->getOperand(2), // Attrchan 2410 ToM0.getValue(1) // In glue 2411 }); 2412 2413 SDNode *InterpP1LV = 2414 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, { 2415 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers 2416 N->getOperand(1), // Src0 2417 N->getOperand(3), // Attr 2418 N->getOperand(2), // Attrchan 2419 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers 2420 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high 2421 N->getOperand(4), // high 2422 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp 2423 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod 2424 SDValue(InterpMov, 1) 2425 }); 2426 2427 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0)); 2428 } 2429 2430 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { 2431 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2432 switch (IntrID) { 2433 case Intrinsic::amdgcn_ds_append: 2434 case Intrinsic::amdgcn_ds_consume: { 2435 if (N->getValueType(0) != MVT::i32) 2436 break; 2437 SelectDSAppendConsume(N, IntrID); 2438 return; 2439 } 2440 } 2441 2442 SelectCode(N); 2443 } 2444 2445 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { 2446 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 2447 unsigned Opcode; 2448 switch (IntrID) { 2449 case Intrinsic::amdgcn_wqm: 2450 Opcode = AMDGPU::WQM; 2451 break; 2452 case Intrinsic::amdgcn_softwqm: 2453 Opcode = AMDGPU::SOFT_WQM; 2454 break; 2455 case Intrinsic::amdgcn_wwm: 2456 Opcode = AMDGPU::WWM; 2457 break; 2458 case Intrinsic::amdgcn_interp_p1_f16: 2459 SelectInterpP1F16(N); 2460 return; 2461 default: 2462 SelectCode(N); 2463 return; 2464 } 2465 2466 SDValue Src = N->getOperand(1); 2467 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); 2468 } 2469 2470 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { 2471 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2472 switch (IntrID) { 2473 case Intrinsic::amdgcn_ds_gws_init: 2474 case Intrinsic::amdgcn_ds_gws_barrier: 2475 case Intrinsic::amdgcn_ds_gws_sema_v: 2476 case Intrinsic::amdgcn_ds_gws_sema_br: 2477 case Intrinsic::amdgcn_ds_gws_sema_p: 2478 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2479 SelectDS_GWS(N, IntrID); 2480 return; 2481 default: 2482 break; 2483 } 2484 2485 SelectCode(N); 2486 } 2487 2488 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 2489 unsigned &Mods) const { 2490 Mods = 0; 2491 Src = In; 2492 2493 if (Src.getOpcode() == ISD::FNEG) { 2494 Mods |= SISrcMods::NEG; 2495 Src = Src.getOperand(0); 2496 } 2497 2498 if (Src.getOpcode() == ISD::FABS) { 2499 Mods |= SISrcMods::ABS; 2500 Src = Src.getOperand(0); 2501 } 2502 2503 return true; 2504 } 2505 2506 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 2507 SDValue &SrcMods) const { 2508 unsigned Mods; 2509 if (SelectVOP3ModsImpl(In, Src, Mods)) { 2510 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2511 return true; 2512 } 2513 2514 return false; 2515 } 2516 2517 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 2518 SDValue &SrcMods) const { 2519 SelectVOP3Mods(In, Src, SrcMods); 2520 return isNoNanSrc(Src); 2521 } 2522 2523 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 2524 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 2525 return false; 2526 2527 Src = In; 2528 return true; 2529 } 2530 2531 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 2532 SDValue &SrcMods, SDValue &Clamp, 2533 SDValue &Omod) const { 2534 SDLoc DL(In); 2535 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2536 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2537 2538 return SelectVOP3Mods(In, Src, SrcMods); 2539 } 2540 2541 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 2542 SDValue &Clamp, SDValue &Omod) const { 2543 Src = In; 2544 2545 SDLoc DL(In); 2546 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2547 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2548 2549 return true; 2550 } 2551 2552 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 2553 SDValue &SrcMods) const { 2554 unsigned Mods = 0; 2555 Src = In; 2556 2557 if (Src.getOpcode() == ISD::FNEG) { 2558 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 2559 Src = Src.getOperand(0); 2560 } 2561 2562 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 2563 unsigned VecMods = Mods; 2564 2565 SDValue Lo = stripBitcast(Src.getOperand(0)); 2566 SDValue Hi = stripBitcast(Src.getOperand(1)); 2567 2568 if (Lo.getOpcode() == ISD::FNEG) { 2569 Lo = stripBitcast(Lo.getOperand(0)); 2570 Mods ^= SISrcMods::NEG; 2571 } 2572 2573 if (Hi.getOpcode() == ISD::FNEG) { 2574 Hi = stripBitcast(Hi.getOperand(0)); 2575 Mods ^= SISrcMods::NEG_HI; 2576 } 2577 2578 if (isExtractHiElt(Lo, Lo)) 2579 Mods |= SISrcMods::OP_SEL_0; 2580 2581 if (isExtractHiElt(Hi, Hi)) 2582 Mods |= SISrcMods::OP_SEL_1; 2583 2584 Lo = stripExtractLoElt(Lo); 2585 Hi = stripExtractLoElt(Hi); 2586 2587 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 2588 // Really a scalar input. Just select from the low half of the register to 2589 // avoid packing. 2590 2591 Src = Lo; 2592 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2593 return true; 2594 } 2595 2596 Mods = VecMods; 2597 } 2598 2599 // Packed instructions do not have abs modifiers. 2600 Mods |= SISrcMods::OP_SEL_1; 2601 2602 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2603 return true; 2604 } 2605 2606 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2607 SDValue &SrcMods) const { 2608 Src = In; 2609 // FIXME: Handle op_sel 2610 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2611 return true; 2612 } 2613 2614 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2615 SDValue &SrcMods) const { 2616 // FIXME: Handle op_sel 2617 return SelectVOP3Mods(In, Src, SrcMods); 2618 } 2619 2620 // The return value is not whether the match is possible (which it always is), 2621 // but whether or not it a conversion is really used. 2622 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2623 unsigned &Mods) const { 2624 Mods = 0; 2625 SelectVOP3ModsImpl(In, Src, Mods); 2626 2627 if (Src.getOpcode() == ISD::FP_EXTEND) { 2628 Src = Src.getOperand(0); 2629 assert(Src.getValueType() == MVT::f16); 2630 Src = stripBitcast(Src); 2631 2632 // Be careful about folding modifiers if we already have an abs. fneg is 2633 // applied last, so we don't want to apply an earlier fneg. 2634 if ((Mods & SISrcMods::ABS) == 0) { 2635 unsigned ModsTmp; 2636 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2637 2638 if ((ModsTmp & SISrcMods::NEG) != 0) 2639 Mods ^= SISrcMods::NEG; 2640 2641 if ((ModsTmp & SISrcMods::ABS) != 0) 2642 Mods |= SISrcMods::ABS; 2643 } 2644 2645 // op_sel/op_sel_hi decide the source type and source. 2646 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2647 // If the sources's op_sel is set, it picks the high half of the source 2648 // register. 2649 2650 Mods |= SISrcMods::OP_SEL_1; 2651 if (isExtractHiElt(Src, Src)) { 2652 Mods |= SISrcMods::OP_SEL_0; 2653 2654 // TODO: Should we try to look for neg/abs here? 2655 } 2656 2657 return true; 2658 } 2659 2660 return false; 2661 } 2662 2663 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2664 SDValue &SrcMods) const { 2665 unsigned Mods = 0; 2666 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2667 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2668 return true; 2669 } 2670 2671 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { 2672 if (In.isUndef()) 2673 return CurDAG->getUNDEF(MVT::i32); 2674 2675 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2676 SDLoc SL(In); 2677 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32); 2678 } 2679 2680 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 2681 SDLoc SL(In); 2682 return CurDAG->getConstant( 2683 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 2684 } 2685 2686 SDValue Src; 2687 if (isExtractHiElt(In, Src)) 2688 return Src; 2689 2690 return SDValue(); 2691 } 2692 2693 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const { 2694 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn); 2695 2696 const SIRegisterInfo *SIRI = 2697 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 2698 const SIInstrInfo * SII = 2699 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 2700 2701 unsigned Limit = 0; 2702 bool AllUsesAcceptSReg = true; 2703 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); 2704 Limit < 10 && U != E; ++U, ++Limit) { 2705 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); 2706 2707 // If the register class is unknown, it could be an unknown 2708 // register class that needs to be an SGPR, e.g. an inline asm 2709 // constraint 2710 if (!RC || SIRI->isSGPRClass(RC)) 2711 return false; 2712 2713 if (RC != &AMDGPU::VS_32RegClass) { 2714 AllUsesAcceptSReg = false; 2715 SDNode * User = *U; 2716 if (User->isMachineOpcode()) { 2717 unsigned Opc = User->getMachineOpcode(); 2718 MCInstrDesc Desc = SII->get(Opc); 2719 if (Desc.isCommutable()) { 2720 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo(); 2721 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 2722 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) { 2723 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs(); 2724 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo); 2725 if (CommutedRC == &AMDGPU::VS_32RegClass) 2726 AllUsesAcceptSReg = true; 2727 } 2728 } 2729 } 2730 // If "AllUsesAcceptSReg == false" so far we haven't suceeded 2731 // commuting current user. This means have at least one use 2732 // that strictly require VGPR. Thus, we will not attempt to commute 2733 // other user instructions. 2734 if (!AllUsesAcceptSReg) 2735 break; 2736 } 2737 } 2738 return !AllUsesAcceptSReg && (Limit < 10); 2739 } 2740 2741 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const { 2742 auto Ld = cast<LoadSDNode>(N); 2743 2744 return Ld->getAlignment() >= 4 && 2745 ( 2746 ( 2747 ( 2748 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2749 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT 2750 ) 2751 && 2752 !N->isDivergent() 2753 ) 2754 || 2755 ( 2756 Subtarget->getScalarizeGlobalBehavior() && 2757 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && 2758 !Ld->isVolatile() && 2759 !N->isDivergent() && 2760 static_cast<const SITargetLowering *>( 2761 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N) 2762 ) 2763 ); 2764 } 2765 2766 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 2767 const AMDGPUTargetLowering& Lowering = 2768 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 2769 bool IsModified = false; 2770 do { 2771 IsModified = false; 2772 2773 // Go over all selected nodes and try to fold them a bit more 2774 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 2775 while (Position != CurDAG->allnodes_end()) { 2776 SDNode *Node = &*Position++; 2777 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 2778 if (!MachineNode) 2779 continue; 2780 2781 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 2782 if (ResNode != Node) { 2783 if (ResNode) 2784 ReplaceUses(Node, ResNode); 2785 IsModified = true; 2786 } 2787 } 2788 CurDAG->RemoveDeadNodes(); 2789 } while (IsModified); 2790 } 2791 2792 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 2793 Subtarget = &MF.getSubtarget<R600Subtarget>(); 2794 return SelectionDAGISel::runOnMachineFunction(MF); 2795 } 2796 2797 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 2798 if (!N->readMem()) 2799 return false; 2800 if (CbId == -1) 2801 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2802 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 2803 2804 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; 2805 } 2806 2807 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 2808 SDValue& IntPtr) { 2809 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 2810 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 2811 true); 2812 return true; 2813 } 2814 return false; 2815 } 2816 2817 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 2818 SDValue& BaseReg, SDValue &Offset) { 2819 if (!isa<ConstantSDNode>(Addr)) { 2820 BaseReg = Addr; 2821 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 2822 return true; 2823 } 2824 return false; 2825 } 2826 2827 void R600DAGToDAGISel::Select(SDNode *N) { 2828 unsigned int Opc = N->getOpcode(); 2829 if (N->isMachineOpcode()) { 2830 N->setNodeId(-1); 2831 return; // Already selected. 2832 } 2833 2834 switch (Opc) { 2835 default: break; 2836 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 2837 case ISD::SCALAR_TO_VECTOR: 2838 case ISD::BUILD_VECTOR: { 2839 EVT VT = N->getValueType(0); 2840 unsigned NumVectorElts = VT.getVectorNumElements(); 2841 unsigned RegClassID; 2842 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 2843 // that adds a 128 bits reg copy when going through TwoAddressInstructions 2844 // pass. We want to avoid 128 bits copies as much as possible because they 2845 // can't be bundled by our scheduler. 2846 switch(NumVectorElts) { 2847 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 2848 case 4: 2849 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 2850 RegClassID = R600::R600_Reg128VerticalRegClassID; 2851 else 2852 RegClassID = R600::R600_Reg128RegClassID; 2853 break; 2854 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 2855 } 2856 SelectBuildVector(N, RegClassID); 2857 return; 2858 } 2859 } 2860 2861 SelectCode(N); 2862 } 2863 2864 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 2865 SDValue &Offset) { 2866 ConstantSDNode *C; 2867 SDLoc DL(Addr); 2868 2869 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 2870 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2871 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2872 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 2873 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 2874 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2875 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2876 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 2877 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 2878 Base = Addr.getOperand(0); 2879 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2880 } else { 2881 Base = Addr; 2882 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2883 } 2884 2885 return true; 2886 } 2887 2888 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 2889 SDValue &Offset) { 2890 ConstantSDNode *IMMOffset; 2891 2892 if (Addr.getOpcode() == ISD::ADD 2893 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 2894 && isInt<16>(IMMOffset->getZExtValue())) { 2895 2896 Base = Addr.getOperand(0); 2897 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2898 MVT::i32); 2899 return true; 2900 // If the pointer address is constant, we can move it to the offset field. 2901 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 2902 && isInt<16>(IMMOffset->getZExtValue())) { 2903 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 2904 SDLoc(CurDAG->getEntryNode()), 2905 R600::ZERO, MVT::i32); 2906 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2907 MVT::i32); 2908 return true; 2909 } 2910 2911 // Default case, no offset 2912 Base = Addr; 2913 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 2914 return true; 2915 } 2916