1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //==-----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines an instruction selector for the AMDGPU target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUArgumentUsageInfo.h" 16 #include "AMDGPUISelLowering.h" // For AMDGPUISD 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUPerfHintAnalysis.h" 19 #include "AMDGPUSubtarget.h" 20 #include "AMDGPUTargetMachine.h" 21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 22 #include "SIDefines.h" 23 #include "SIISelLowering.h" 24 #include "SIInstrInfo.h" 25 #include "SIMachineFunctionInfo.h" 26 #include "SIRegisterInfo.h" 27 #include "llvm/ADT/APInt.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 31 #include "llvm/Analysis/ValueTracking.h" 32 #include "llvm/CodeGen/FunctionLoweringInfo.h" 33 #include "llvm/CodeGen/ISDOpcodes.h" 34 #include "llvm/CodeGen/MachineFunction.h" 35 #include "llvm/CodeGen/MachineRegisterInfo.h" 36 #include "llvm/CodeGen/SelectionDAG.h" 37 #include "llvm/CodeGen/SelectionDAGISel.h" 38 #include "llvm/CodeGen/SelectionDAGNodes.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/IR/BasicBlock.h" 41 #include "llvm/InitializePasses.h" 42 #ifdef EXPENSIVE_CHECKS 43 #include "llvm/IR/Dominators.h" 44 #endif 45 #include "llvm/IR/Instruction.h" 46 #include "llvm/MC/MCInstrDesc.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/CodeGen.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MachineValueType.h" 51 #include "llvm/Support/MathExtras.h" 52 #include <cassert> 53 #include <cstdint> 54 #include <new> 55 #include <vector> 56 57 #define DEBUG_TYPE "isel" 58 59 using namespace llvm; 60 61 namespace llvm { 62 63 class R600InstrInfo; 64 65 } // end namespace llvm 66 67 //===----------------------------------------------------------------------===// 68 // Instruction Selector Implementation 69 //===----------------------------------------------------------------------===// 70 71 namespace { 72 73 static bool isNullConstantOrUndef(SDValue V) { 74 if (V.isUndef()) 75 return true; 76 77 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 78 return Const != nullptr && Const->isNullValue(); 79 } 80 81 static bool getConstantValue(SDValue N, uint32_t &Out) { 82 // This is only used for packed vectors, where ussing 0 for undef should 83 // always be good. 84 if (N.isUndef()) { 85 Out = 0; 86 return true; 87 } 88 89 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 90 Out = C->getAPIntValue().getSExtValue(); 91 return true; 92 } 93 94 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 95 Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); 96 return true; 97 } 98 99 return false; 100 } 101 102 // TODO: Handle undef as zero 103 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG, 104 bool Negate = false) { 105 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); 106 uint32_t LHSVal, RHSVal; 107 if (getConstantValue(N->getOperand(0), LHSVal) && 108 getConstantValue(N->getOperand(1), RHSVal)) { 109 SDLoc SL(N); 110 uint32_t K = Negate ? 111 (-LHSVal & 0xffff) | (-RHSVal << 16) : 112 (LHSVal & 0xffff) | (RHSVal << 16); 113 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0), 114 DAG.getTargetConstant(K, SL, MVT::i32)); 115 } 116 117 return nullptr; 118 } 119 120 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) { 121 return packConstantV2I16(N, DAG, true); 122 } 123 124 /// AMDGPU specific code to select AMDGPU machine instructions for 125 /// SelectionDAG operations. 126 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 127 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 128 // make the right decision when generating code for different targets. 129 const GCNSubtarget *Subtarget; 130 131 // Default FP mode for the current function. 132 AMDGPU::SIModeRegisterDefaults Mode; 133 134 bool EnableLateStructurizeCFG; 135 136 public: 137 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 138 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 139 : SelectionDAGISel(*TM, OptLevel) { 140 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 141 } 142 ~AMDGPUDAGToDAGISel() override = default; 143 144 void getAnalysisUsage(AnalysisUsage &AU) const override { 145 AU.addRequired<AMDGPUArgumentUsageInfo>(); 146 AU.addRequired<LegacyDivergenceAnalysis>(); 147 #ifdef EXPENSIVE_CHECKS 148 AU.addRequired<DominatorTreeWrapperPass>(); 149 AU.addRequired<LoopInfoWrapperPass>(); 150 #endif 151 SelectionDAGISel::getAnalysisUsage(AU); 152 } 153 154 bool matchLoadD16FromBuildVector(SDNode *N) const; 155 156 bool runOnMachineFunction(MachineFunction &MF) override; 157 void PreprocessISelDAG() override; 158 void Select(SDNode *N) override; 159 StringRef getPassName() const override; 160 void PostprocessISelDAG() override; 161 162 protected: 163 void SelectBuildVector(SDNode *N, unsigned RegClassID); 164 165 private: 166 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 167 bool isNoNanSrc(SDValue N) const; 168 bool isInlineImmediate(const SDNode *N, bool Negated = false) const; 169 bool isNegInlineImmediate(const SDNode *N) const { 170 return isInlineImmediate(N, true); 171 } 172 173 bool isInlineImmediate16(int64_t Imm) const { 174 return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); 175 } 176 177 bool isInlineImmediate32(int64_t Imm) const { 178 return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); 179 } 180 181 bool isInlineImmediate64(int64_t Imm) const { 182 return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); 183 } 184 185 bool isInlineImmediate(const APFloat &Imm) const { 186 return Subtarget->getInstrInfo()->isInlineConstant(Imm); 187 } 188 189 bool isVGPRImm(const SDNode *N) const; 190 bool isUniformLoad(const SDNode *N) const; 191 bool isUniformBr(const SDNode *N) const; 192 193 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; 194 195 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; 196 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; 197 SDNode *glueCopyToM0LDSInit(SDNode *N) const; 198 199 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 200 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 201 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 202 bool isDSOffsetLegal(SDValue Base, unsigned Offset, 203 unsigned OffsetBits) const; 204 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 205 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 206 SDValue &Offset1) const; 207 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 208 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 209 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC, 210 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 211 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 212 SDValue &SOffset, SDValue &Offset, SDValue &GLC, 213 SDValue &SLC, SDValue &TFE, SDValue &DLC, 214 SDValue &SWZ) const; 215 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 216 SDValue &VAddr, SDValue &SOffset, SDValue &Offset, 217 SDValue &SLC) const; 218 bool SelectMUBUFScratchOffen(SDNode *Parent, 219 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 220 SDValue &SOffset, SDValue &ImmOffset) const; 221 bool SelectMUBUFScratchOffset(SDNode *Parent, 222 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 223 SDValue &Offset) const; 224 225 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset, 226 SDValue &Offset, SDValue &GLC, SDValue &SLC, 227 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 228 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 229 SDValue &Offset, SDValue &SLC) const; 230 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 231 SDValue &Offset) const; 232 233 template <bool IsSigned> 234 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 235 SDValue &Offset, SDValue &SLC) const; 236 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr, 237 SDValue &Offset, SDValue &SLC) const; 238 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr, 239 SDValue &Offset, SDValue &SLC) const; 240 241 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 242 bool &Imm) const; 243 SDValue Expand32BitAddress(SDValue Addr) const; 244 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 245 bool &Imm) const; 246 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 247 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 248 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 249 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 250 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 251 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 252 253 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 254 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const; 255 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 256 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 257 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 258 SDValue &Clamp, SDValue &Omod) const; 259 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 260 SDValue &Clamp, SDValue &Omod) const; 261 262 bool SelectVOP3OMods(SDValue In, SDValue &Src, 263 SDValue &Clamp, SDValue &Omod) const; 264 265 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 266 267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 268 269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 271 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 272 273 SDValue getHi16Elt(SDValue In) const; 274 275 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; 276 277 void SelectADD_SUB_I64(SDNode *N); 278 void SelectAddcSubb(SDNode *N); 279 void SelectUADDO_USUBO(SDNode *N); 280 void SelectDIV_SCALE(SDNode *N); 281 void SelectMAD_64_32(SDNode *N); 282 void SelectFMA_W_CHAIN(SDNode *N); 283 void SelectFMUL_W_CHAIN(SDNode *N); 284 285 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 286 uint32_t Offset, uint32_t Width); 287 void SelectS_BFEFromShifts(SDNode *N); 288 void SelectS_BFE(SDNode *N); 289 bool isCBranchSCC(const SDNode *N) const; 290 void SelectBRCOND(SDNode *N); 291 void SelectFMAD_FMA(SDNode *N); 292 void SelectATOMIC_CMP_SWAP(SDNode *N); 293 void SelectDSAppendConsume(SDNode *N, unsigned IntrID); 294 void SelectDS_GWS(SDNode *N, unsigned IntrID); 295 void SelectInterpP1F16(SDNode *N); 296 void SelectINTRINSIC_W_CHAIN(SDNode *N); 297 void SelectINTRINSIC_WO_CHAIN(SDNode *N); 298 void SelectINTRINSIC_VOID(SDNode *N); 299 300 protected: 301 // Include the pieces autogenerated from the target description. 302 #include "AMDGPUGenDAGISel.inc" 303 }; 304 305 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 306 const R600Subtarget *Subtarget; 307 308 bool isConstantLoad(const MemSDNode *N, int cbID) const; 309 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 310 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 311 SDValue& Offset); 312 public: 313 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 314 AMDGPUDAGToDAGISel(TM, OptLevel) {} 315 316 void Select(SDNode *N) override; 317 318 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 319 SDValue &Offset) override; 320 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 321 SDValue &Offset) override; 322 323 bool runOnMachineFunction(MachineFunction &MF) override; 324 325 void PreprocessISelDAG() override {} 326 327 protected: 328 // Include the pieces autogenerated from the target description. 329 #include "R600GenDAGISel.inc" 330 }; 331 332 static SDValue stripBitcast(SDValue Val) { 333 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 334 } 335 336 // Figure out if this is really an extract of the high 16-bits of a dword. 337 static bool isExtractHiElt(SDValue In, SDValue &Out) { 338 In = stripBitcast(In); 339 if (In.getOpcode() != ISD::TRUNCATE) 340 return false; 341 342 SDValue Srl = In.getOperand(0); 343 if (Srl.getOpcode() == ISD::SRL) { 344 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 345 if (ShiftAmt->getZExtValue() == 16) { 346 Out = stripBitcast(Srl.getOperand(0)); 347 return true; 348 } 349 } 350 } 351 352 return false; 353 } 354 355 // Look through operations that obscure just looking at the low 16-bits of the 356 // same register. 357 static SDValue stripExtractLoElt(SDValue In) { 358 if (In.getOpcode() == ISD::TRUNCATE) { 359 SDValue Src = In.getOperand(0); 360 if (Src.getValueType().getSizeInBits() == 32) 361 return stripBitcast(Src); 362 } 363 364 return In; 365 } 366 367 } // end anonymous namespace 368 369 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel", 370 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 371 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 372 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 373 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) 374 #ifdef EXPENSIVE_CHECKS 375 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 376 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 377 #endif 378 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel", 379 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 380 381 /// This pass converts a legalized DAG into a AMDGPU-specific 382 // DAG, ready for instruction scheduling. 383 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 384 CodeGenOpt::Level OptLevel) { 385 return new AMDGPUDAGToDAGISel(TM, OptLevel); 386 } 387 388 /// This pass converts a legalized DAG into a R600-specific 389 // DAG, ready for instruction scheduling. 390 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 391 CodeGenOpt::Level OptLevel) { 392 return new R600DAGToDAGISel(TM, OptLevel); 393 } 394 395 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 396 #ifdef EXPENSIVE_CHECKS 397 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 398 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 399 for (auto &L : LI->getLoopsInPreorder()) { 400 assert(L->isLCSSAForm(DT)); 401 } 402 #endif 403 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 404 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction()); 405 return SelectionDAGISel::runOnMachineFunction(MF); 406 } 407 408 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const { 409 assert(Subtarget->d16PreservesUnusedBits()); 410 MVT VT = N->getValueType(0).getSimpleVT(); 411 if (VT != MVT::v2i16 && VT != MVT::v2f16) 412 return false; 413 414 SDValue Lo = N->getOperand(0); 415 SDValue Hi = N->getOperand(1); 416 417 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi)); 418 419 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo 420 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo 421 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo 422 423 // Need to check for possible indirect dependencies on the other half of the 424 // vector to avoid introducing a cycle. 425 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) { 426 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 427 428 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo); 429 SDValue Ops[] = { 430 LdHi->getChain(), LdHi->getBasePtr(), TiedIn 431 }; 432 433 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI; 434 if (LdHi->getMemoryVT() == MVT::i8) { 435 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ? 436 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8; 437 } else { 438 assert(LdHi->getMemoryVT() == MVT::i16); 439 } 440 441 SDValue NewLoadHi = 442 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList, 443 Ops, LdHi->getMemoryVT(), 444 LdHi->getMemOperand()); 445 446 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi); 447 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1)); 448 return true; 449 } 450 451 // build_vector (load ptr), hi -> load_d16_lo ptr, hi 452 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi 453 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi 454 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo)); 455 if (LdLo && Lo.hasOneUse()) { 456 SDValue TiedIn = getHi16Elt(Hi); 457 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode())) 458 return false; 459 460 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 461 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO; 462 if (LdLo->getMemoryVT() == MVT::i8) { 463 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ? 464 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8; 465 } else { 466 assert(LdLo->getMemoryVT() == MVT::i16); 467 } 468 469 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn); 470 471 SDValue Ops[] = { 472 LdLo->getChain(), LdLo->getBasePtr(), TiedIn 473 }; 474 475 SDValue NewLoadLo = 476 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList, 477 Ops, LdLo->getMemoryVT(), 478 LdLo->getMemOperand()); 479 480 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo); 481 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1)); 482 return true; 483 } 484 485 return false; 486 } 487 488 void AMDGPUDAGToDAGISel::PreprocessISelDAG() { 489 if (!Subtarget->d16PreservesUnusedBits()) 490 return; 491 492 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 493 494 bool MadeChange = false; 495 while (Position != CurDAG->allnodes_begin()) { 496 SDNode *N = &*--Position; 497 if (N->use_empty()) 498 continue; 499 500 switch (N->getOpcode()) { 501 case ISD::BUILD_VECTOR: 502 MadeChange |= matchLoadD16FromBuildVector(N); 503 break; 504 default: 505 break; 506 } 507 } 508 509 if (MadeChange) { 510 CurDAG->RemoveDeadNodes(); 511 LLVM_DEBUG(dbgs() << "After PreProcess:\n"; 512 CurDAG->dump();); 513 } 514 } 515 516 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 517 if (TM.Options.NoNaNsFPMath) 518 return true; 519 520 // TODO: Move into isKnownNeverNaN 521 if (N->getFlags().isDefined()) 522 return N->getFlags().hasNoNaNs(); 523 524 return CurDAG->isKnownNeverNaN(N); 525 } 526 527 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N, 528 bool Negated) const { 529 if (N->isUndef()) 530 return true; 531 532 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 533 if (Negated) { 534 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 535 return TII->isInlineConstant(-C->getAPIntValue()); 536 537 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 538 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt()); 539 540 } else { 541 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 542 return TII->isInlineConstant(C->getAPIntValue()); 543 544 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 545 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 546 } 547 548 return false; 549 } 550 551 /// Determine the register class for \p OpNo 552 /// \returns The register class of the virtual register that will be used for 553 /// the given operand number \OpNo or NULL if the register class cannot be 554 /// determined. 555 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 556 unsigned OpNo) const { 557 if (!N->isMachineOpcode()) { 558 if (N->getOpcode() == ISD::CopyToReg) { 559 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 560 if (Register::isVirtualRegister(Reg)) { 561 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 562 return MRI.getRegClass(Reg); 563 } 564 565 const SIRegisterInfo *TRI 566 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 567 return TRI->getPhysRegClass(Reg); 568 } 569 570 return nullptr; 571 } 572 573 switch (N->getMachineOpcode()) { 574 default: { 575 const MCInstrDesc &Desc = 576 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 577 unsigned OpIdx = Desc.getNumDefs() + OpNo; 578 if (OpIdx >= Desc.getNumOperands()) 579 return nullptr; 580 int RegClass = Desc.OpInfo[OpIdx].RegClass; 581 if (RegClass == -1) 582 return nullptr; 583 584 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 585 } 586 case AMDGPU::REG_SEQUENCE: { 587 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 588 const TargetRegisterClass *SuperRC = 589 Subtarget->getRegisterInfo()->getRegClass(RCID); 590 591 SDValue SubRegOp = N->getOperand(OpNo + 1); 592 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 593 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 594 SubRegIdx); 595 } 596 } 597 } 598 599 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain, 600 SDValue Glue) const { 601 SmallVector <SDValue, 8> Ops; 602 Ops.push_back(NewChain); // Replace the chain. 603 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 604 Ops.push_back(N->getOperand(i)); 605 606 Ops.push_back(Glue); 607 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 608 } 609 610 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const { 611 const SITargetLowering& Lowering = 612 *static_cast<const SITargetLowering*>(getTargetLowering()); 613 614 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain"); 615 616 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val); 617 return glueCopyToOp(N, M0, M0.getValue(1)); 618 } 619 620 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const { 621 unsigned AS = cast<MemSDNode>(N)->getAddressSpace(); 622 if (AS == AMDGPUAS::LOCAL_ADDRESS) { 623 if (Subtarget->ldsRequiresM0Init()) 624 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 625 } else if (AS == AMDGPUAS::REGION_ADDRESS) { 626 MachineFunction &MF = CurDAG->getMachineFunction(); 627 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize(); 628 return 629 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32)); 630 } 631 return N; 632 } 633 634 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm, 635 EVT VT) const { 636 SDNode *Lo = CurDAG->getMachineNode( 637 AMDGPU::S_MOV_B32, DL, MVT::i32, 638 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32)); 639 SDNode *Hi = 640 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 641 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32)); 642 const SDValue Ops[] = { 643 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 644 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 645 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)}; 646 647 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops); 648 } 649 650 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 651 EVT VT = N->getValueType(0); 652 unsigned NumVectorElts = VT.getVectorNumElements(); 653 EVT EltVT = VT.getVectorElementType(); 654 SDLoc DL(N); 655 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 656 657 if (NumVectorElts == 1) { 658 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 659 RegClass); 660 return; 661 } 662 663 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not " 664 "supported yet"); 665 // 32 = Max Num Vector Elements 666 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 667 // 1 = Vector Register Class 668 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 669 670 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() == 671 Triple::amdgcn; 672 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 673 bool IsRegSeq = true; 674 unsigned NOps = N->getNumOperands(); 675 for (unsigned i = 0; i < NOps; i++) { 676 // XXX: Why is this here? 677 if (isa<RegisterSDNode>(N->getOperand(i))) { 678 IsRegSeq = false; 679 break; 680 } 681 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 682 : R600RegisterInfo::getSubRegFromChannel(i); 683 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 684 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 685 } 686 if (NOps != NumVectorElts) { 687 // Fill in the missing undef elements if this was a scalar_to_vector. 688 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 689 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 690 DL, EltVT); 691 for (unsigned i = NOps; i < NumVectorElts; ++i) { 692 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 693 : R600RegisterInfo::getSubRegFromChannel(i); 694 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 695 RegSeqArgs[1 + (2 * i) + 1] = 696 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 697 } 698 } 699 700 if (!IsRegSeq) 701 SelectCode(N); 702 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 703 } 704 705 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 706 unsigned int Opc = N->getOpcode(); 707 if (N->isMachineOpcode()) { 708 N->setNodeId(-1); 709 return; // Already selected. 710 } 711 712 // isa<MemSDNode> almost works but is slightly too permissive for some DS 713 // intrinsics. 714 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) || 715 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 716 Opc == ISD::ATOMIC_LOAD_FADD || 717 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 718 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) { 719 N = glueCopyToM0LDSInit(N); 720 SelectCode(N); 721 return; 722 } 723 724 switch (Opc) { 725 default: 726 break; 727 // We are selecting i64 ADD here instead of custom lower it during 728 // DAG legalization, so we can fold some i64 ADDs used for address 729 // calculation into the LOAD and STORE instructions. 730 case ISD::ADDC: 731 case ISD::ADDE: 732 case ISD::SUBC: 733 case ISD::SUBE: { 734 if (N->getValueType(0) != MVT::i64) 735 break; 736 737 SelectADD_SUB_I64(N); 738 return; 739 } 740 case ISD::ADDCARRY: 741 case ISD::SUBCARRY: 742 if (N->getValueType(0) != MVT::i32) 743 break; 744 745 SelectAddcSubb(N); 746 return; 747 case ISD::UADDO: 748 case ISD::USUBO: { 749 SelectUADDO_USUBO(N); 750 return; 751 } 752 case AMDGPUISD::FMUL_W_CHAIN: { 753 SelectFMUL_W_CHAIN(N); 754 return; 755 } 756 case AMDGPUISD::FMA_W_CHAIN: { 757 SelectFMA_W_CHAIN(N); 758 return; 759 } 760 761 case ISD::SCALAR_TO_VECTOR: 762 case ISD::BUILD_VECTOR: { 763 EVT VT = N->getValueType(0); 764 unsigned NumVectorElts = VT.getVectorNumElements(); 765 if (VT.getScalarSizeInBits() == 16) { 766 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 767 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) { 768 ReplaceNode(N, Packed); 769 return; 770 } 771 } 772 773 break; 774 } 775 776 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 777 unsigned RegClassID = 778 SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID(); 779 SelectBuildVector(N, RegClassID); 780 return; 781 } 782 case ISD::BUILD_PAIR: { 783 SDValue RC, SubReg0, SubReg1; 784 SDLoc DL(N); 785 if (N->getValueType(0) == MVT::i128) { 786 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32); 787 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 788 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 789 } else if (N->getValueType(0) == MVT::i64) { 790 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 791 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 792 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 793 } else { 794 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 795 } 796 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 797 N->getOperand(1), SubReg1 }; 798 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 799 N->getValueType(0), Ops)); 800 return; 801 } 802 803 case ISD::Constant: 804 case ISD::ConstantFP: { 805 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 806 break; 807 808 uint64_t Imm; 809 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 810 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 811 else { 812 ConstantSDNode *C = cast<ConstantSDNode>(N); 813 Imm = C->getZExtValue(); 814 } 815 816 SDLoc DL(N); 817 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0))); 818 return; 819 } 820 case AMDGPUISD::BFE_I32: 821 case AMDGPUISD::BFE_U32: { 822 // There is a scalar version available, but unlike the vector version which 823 // has a separate operand for the offset and width, the scalar version packs 824 // the width and offset into a single operand. Try to move to the scalar 825 // version if the offsets are constant, so that we can try to keep extended 826 // loads of kernel arguments in SGPRs. 827 828 // TODO: Technically we could try to pattern match scalar bitshifts of 829 // dynamic values, but it's probably not useful. 830 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 831 if (!Offset) 832 break; 833 834 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 835 if (!Width) 836 break; 837 838 bool Signed = Opc == AMDGPUISD::BFE_I32; 839 840 uint32_t OffsetVal = Offset->getZExtValue(); 841 uint32_t WidthVal = Width->getZExtValue(); 842 843 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 844 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 845 return; 846 } 847 case AMDGPUISD::DIV_SCALE: { 848 SelectDIV_SCALE(N); 849 return; 850 } 851 case AMDGPUISD::MAD_I64_I32: 852 case AMDGPUISD::MAD_U64_U32: { 853 SelectMAD_64_32(N); 854 return; 855 } 856 case ISD::CopyToReg: { 857 const SITargetLowering& Lowering = 858 *static_cast<const SITargetLowering*>(getTargetLowering()); 859 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 860 break; 861 } 862 case ISD::AND: 863 case ISD::SRL: 864 case ISD::SRA: 865 case ISD::SIGN_EXTEND_INREG: 866 if (N->getValueType(0) != MVT::i32) 867 break; 868 869 SelectS_BFE(N); 870 return; 871 case ISD::BRCOND: 872 SelectBRCOND(N); 873 return; 874 case ISD::FMAD: 875 case ISD::FMA: 876 SelectFMAD_FMA(N); 877 return; 878 case AMDGPUISD::ATOMIC_CMP_SWAP: 879 SelectATOMIC_CMP_SWAP(N); 880 return; 881 case AMDGPUISD::CVT_PKRTZ_F16_F32: 882 case AMDGPUISD::CVT_PKNORM_I16_F32: 883 case AMDGPUISD::CVT_PKNORM_U16_F32: 884 case AMDGPUISD::CVT_PK_U16_U32: 885 case AMDGPUISD::CVT_PK_I16_I32: { 886 // Hack around using a legal type if f16 is illegal. 887 if (N->getValueType(0) == MVT::i32) { 888 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16; 889 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT), 890 { N->getOperand(0), N->getOperand(1) }); 891 SelectCode(N); 892 return; 893 } 894 895 break; 896 } 897 case ISD::INTRINSIC_W_CHAIN: { 898 SelectINTRINSIC_W_CHAIN(N); 899 return; 900 } 901 case ISD::INTRINSIC_WO_CHAIN: { 902 SelectINTRINSIC_WO_CHAIN(N); 903 return; 904 } 905 case ISD::INTRINSIC_VOID: { 906 SelectINTRINSIC_VOID(N); 907 return; 908 } 909 } 910 911 SelectCode(N); 912 } 913 914 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 915 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 916 const Instruction *Term = BB->getTerminator(); 917 return Term->getMetadata("amdgpu.uniform") || 918 Term->getMetadata("structurizecfg.uniform"); 919 } 920 921 StringRef AMDGPUDAGToDAGISel::getPassName() const { 922 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 923 } 924 925 //===----------------------------------------------------------------------===// 926 // Complex Patterns 927 //===----------------------------------------------------------------------===// 928 929 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 930 SDValue &Offset) { 931 return false; 932 } 933 934 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 935 SDValue &Offset) { 936 ConstantSDNode *C; 937 SDLoc DL(Addr); 938 939 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 940 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 941 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 942 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 943 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 944 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 945 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 946 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 947 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 948 Base = Addr.getOperand(0); 949 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 950 } else { 951 Base = Addr; 952 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 953 } 954 955 return true; 956 } 957 958 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val, 959 const SDLoc &DL) const { 960 SDNode *Mov = CurDAG->getMachineNode( 961 AMDGPU::S_MOV_B32, DL, MVT::i32, 962 CurDAG->getTargetConstant(Val, DL, MVT::i32)); 963 return SDValue(Mov, 0); 964 } 965 966 // FIXME: Should only handle addcarry/subcarry 967 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 968 SDLoc DL(N); 969 SDValue LHS = N->getOperand(0); 970 SDValue RHS = N->getOperand(1); 971 972 unsigned Opcode = N->getOpcode(); 973 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 974 bool ProduceCarry = 975 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 976 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 977 978 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 979 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 980 981 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 982 DL, MVT::i32, LHS, Sub0); 983 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 984 DL, MVT::i32, LHS, Sub1); 985 986 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 987 DL, MVT::i32, RHS, Sub0); 988 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 989 DL, MVT::i32, RHS, Sub1); 990 991 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 992 993 static const unsigned OpcMap[2][2][2] = { 994 {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32}, 995 {AMDGPU::V_SUB_I32_e32, AMDGPU::V_ADD_I32_e32}}, 996 {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32}, 997 {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}}; 998 999 unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd]; 1000 unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd]; 1001 1002 SDNode *AddLo; 1003 if (!ConsumeCarry) { 1004 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 1005 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 1006 } else { 1007 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 1008 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 1009 } 1010 SDValue AddHiArgs[] = { 1011 SDValue(Hi0, 0), 1012 SDValue(Hi1, 0), 1013 SDValue(AddLo, 1) 1014 }; 1015 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 1016 1017 SDValue RegSequenceArgs[] = { 1018 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 1019 SDValue(AddLo,0), 1020 Sub0, 1021 SDValue(AddHi,0), 1022 Sub1, 1023 }; 1024 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1025 MVT::i64, RegSequenceArgs); 1026 1027 if (ProduceCarry) { 1028 // Replace the carry-use 1029 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 1030 } 1031 1032 // Replace the remaining uses. 1033 ReplaceNode(N, RegSequence); 1034 } 1035 1036 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) { 1037 SDLoc DL(N); 1038 SDValue LHS = N->getOperand(0); 1039 SDValue RHS = N->getOperand(1); 1040 SDValue CI = N->getOperand(2); 1041 1042 if (N->isDivergent()) { 1043 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64 1044 : AMDGPU::V_SUBB_U32_e64; 1045 CurDAG->SelectNodeTo( 1046 N, Opc, N->getVTList(), 1047 {LHS, RHS, CI, 1048 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1049 } else { 1050 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO 1051 : AMDGPU::S_SUB_CO_PSEUDO; 1052 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI}); 1053 } 1054 } 1055 1056 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 1057 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 1058 // carry out despite the _i32 name. These were renamed in VI to _U32. 1059 // FIXME: We should probably rename the opcodes here. 1060 bool IsAdd = N->getOpcode() == ISD::UADDO; 1061 bool IsVALU = N->isDivergent(); 1062 1063 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; 1064 ++UI) 1065 if (UI.getUse().getResNo() == 1) { 1066 if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) || 1067 (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) { 1068 IsVALU = true; 1069 break; 1070 } 1071 } 1072 1073 if (IsVALU) { 1074 unsigned Opc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 1075 1076 CurDAG->SelectNodeTo( 1077 N, Opc, N->getVTList(), 1078 {N->getOperand(0), N->getOperand(1), 1079 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1080 } else { 1081 unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO 1082 : AMDGPU::S_USUBO_PSEUDO; 1083 1084 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), 1085 {N->getOperand(0), N->getOperand(1)}); 1086 } 1087 } 1088 1089 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 1090 SDLoc SL(N); 1091 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 1092 SDValue Ops[10]; 1093 1094 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 1095 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1096 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 1097 Ops[8] = N->getOperand(0); 1098 Ops[9] = N->getOperand(4); 1099 1100 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); 1101 } 1102 1103 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 1104 SDLoc SL(N); 1105 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 1106 SDValue Ops[8]; 1107 1108 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 1109 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1110 Ops[6] = N->getOperand(0); 1111 Ops[7] = N->getOperand(3); 1112 1113 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 1114 } 1115 1116 // We need to handle this here because tablegen doesn't support matching 1117 // instructions with multiple outputs. 1118 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 1119 SDLoc SL(N); 1120 EVT VT = N->getValueType(0); 1121 1122 assert(VT == MVT::f32 || VT == MVT::f64); 1123 1124 unsigned Opc 1125 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32; 1126 1127 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; 1128 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1129 } 1130 1131 // We need to handle this here because tablegen doesn't support matching 1132 // instructions with multiple outputs. 1133 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 1134 SDLoc SL(N); 1135 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 1136 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32; 1137 1138 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 1139 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 1140 Clamp }; 1141 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1142 } 1143 1144 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset, 1145 unsigned OffsetBits) const { 1146 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 1147 (OffsetBits == 8 && !isUInt<8>(Offset))) 1148 return false; 1149 1150 if (Subtarget->hasUsableDSOffset() || 1151 Subtarget->unsafeDSOffsetFoldingEnabled()) 1152 return true; 1153 1154 // On Southern Islands instruction with a negative base value and an offset 1155 // don't seem to work. 1156 return CurDAG->SignBitIsZero(Base); 1157 } 1158 1159 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 1160 SDValue &Offset) const { 1161 SDLoc DL(Addr); 1162 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1163 SDValue N0 = Addr.getOperand(0); 1164 SDValue N1 = Addr.getOperand(1); 1165 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1166 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) { 1167 // (add n0, c0) 1168 Base = N0; 1169 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1170 return true; 1171 } 1172 } else if (Addr.getOpcode() == ISD::SUB) { 1173 // sub C, x -> add (sub 0, x), C 1174 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1175 int64_t ByteOffset = C->getSExtValue(); 1176 if (isUInt<16>(ByteOffset)) { 1177 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1178 1179 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1180 // the known bits in isDSOffsetLegal. We need to emit the selected node 1181 // here, so this is thrown away. 1182 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1183 Zero, Addr.getOperand(1)); 1184 1185 if (isDSOffsetLegal(Sub, ByteOffset, 16)) { 1186 SmallVector<SDValue, 3> Opnds; 1187 Opnds.push_back(Zero); 1188 Opnds.push_back(Addr.getOperand(1)); 1189 1190 // FIXME: Select to VOP3 version for with-carry. 1191 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1192 if (Subtarget->hasAddNoCarry()) { 1193 SubOp = AMDGPU::V_SUB_U32_e64; 1194 Opnds.push_back( 1195 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1196 } 1197 1198 MachineSDNode *MachineSub = 1199 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1200 1201 Base = SDValue(MachineSub, 0); 1202 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 1203 return true; 1204 } 1205 } 1206 } 1207 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1208 // If we have a constant address, prefer to put the constant into the 1209 // offset. This can save moves to load the constant address since multiple 1210 // operations can share the zero base address register, and enables merging 1211 // into read2 / write2 instructions. 1212 1213 SDLoc DL(Addr); 1214 1215 if (isUInt<16>(CAddr->getZExtValue())) { 1216 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1217 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1218 DL, MVT::i32, Zero); 1219 Base = SDValue(MovZero, 0); 1220 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1221 return true; 1222 } 1223 } 1224 1225 // default case 1226 Base = Addr; 1227 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 1228 return true; 1229 } 1230 1231 // TODO: If offset is too big, put low 16-bit into offset. 1232 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 1233 SDValue &Offset0, 1234 SDValue &Offset1) const { 1235 SDLoc DL(Addr); 1236 1237 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1238 SDValue N0 = Addr.getOperand(0); 1239 SDValue N1 = Addr.getOperand(1); 1240 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1241 unsigned DWordOffset0 = C1->getZExtValue() / 4; 1242 unsigned DWordOffset1 = DWordOffset0 + 1; 1243 // (add n0, c0) 1244 if (isDSOffsetLegal(N0, DWordOffset1, 8)) { 1245 Base = N0; 1246 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1247 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1248 return true; 1249 } 1250 } else if (Addr.getOpcode() == ISD::SUB) { 1251 // sub C, x -> add (sub 0, x), C 1252 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1253 unsigned DWordOffset0 = C->getZExtValue() / 4; 1254 unsigned DWordOffset1 = DWordOffset0 + 1; 1255 1256 if (isUInt<8>(DWordOffset0)) { 1257 SDLoc DL(Addr); 1258 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1259 1260 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1261 // the known bits in isDSOffsetLegal. We need to emit the selected node 1262 // here, so this is thrown away. 1263 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1264 Zero, Addr.getOperand(1)); 1265 1266 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) { 1267 SmallVector<SDValue, 3> Opnds; 1268 Opnds.push_back(Zero); 1269 Opnds.push_back(Addr.getOperand(1)); 1270 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1271 if (Subtarget->hasAddNoCarry()) { 1272 SubOp = AMDGPU::V_SUB_U32_e64; 1273 Opnds.push_back( 1274 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1275 } 1276 1277 MachineSDNode *MachineSub 1278 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1279 1280 Base = SDValue(MachineSub, 0); 1281 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1282 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1283 return true; 1284 } 1285 } 1286 } 1287 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1288 unsigned DWordOffset0 = CAddr->getZExtValue() / 4; 1289 unsigned DWordOffset1 = DWordOffset0 + 1; 1290 assert(4 * DWordOffset0 == CAddr->getZExtValue()); 1291 1292 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) { 1293 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1294 MachineSDNode *MovZero 1295 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1296 DL, MVT::i32, Zero); 1297 Base = SDValue(MovZero, 0); 1298 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1299 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1300 return true; 1301 } 1302 } 1303 1304 // default case 1305 1306 Base = Addr; 1307 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 1308 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 1309 return true; 1310 } 1311 1312 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, 1313 SDValue &VAddr, SDValue &SOffset, 1314 SDValue &Offset, SDValue &Offen, 1315 SDValue &Idxen, SDValue &Addr64, 1316 SDValue &GLC, SDValue &SLC, 1317 SDValue &TFE, SDValue &DLC, 1318 SDValue &SWZ) const { 1319 // Subtarget prefers to use flat instruction 1320 // FIXME: This should be a pattern predicate and not reach here 1321 if (Subtarget->useFlatForGlobal()) 1322 return false; 1323 1324 SDLoc DL(Addr); 1325 1326 if (!GLC.getNode()) 1327 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1328 if (!SLC.getNode()) 1329 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1330 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1); 1331 DLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1332 SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1); 1333 1334 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1335 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1336 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1337 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1338 1339 ConstantSDNode *C1 = nullptr; 1340 SDValue N0 = Addr; 1341 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1342 C1 = cast<ConstantSDNode>(Addr.getOperand(1)); 1343 if (isUInt<32>(C1->getZExtValue())) 1344 N0 = Addr.getOperand(0); 1345 else 1346 C1 = nullptr; 1347 } 1348 1349 if (N0.getOpcode() == ISD::ADD) { 1350 // (add N2, N3) -> addr64, or 1351 // (add (add N2, N3), C1) -> addr64 1352 SDValue N2 = N0.getOperand(0); 1353 SDValue N3 = N0.getOperand(1); 1354 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1355 1356 if (N2->isDivergent()) { 1357 if (N3->isDivergent()) { 1358 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 1359 // addr64, and construct the resource from a 0 address. 1360 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1361 VAddr = N0; 1362 } else { 1363 // N2 is divergent, N3 is not. 1364 Ptr = N3; 1365 VAddr = N2; 1366 } 1367 } else { 1368 // N2 is not divergent. 1369 Ptr = N2; 1370 VAddr = N3; 1371 } 1372 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1373 } else if (N0->isDivergent()) { 1374 // N0 is divergent. Use it as the addr64, and construct the resource from a 1375 // 0 address. 1376 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1377 VAddr = N0; 1378 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1379 } else { 1380 // N0 -> offset, or 1381 // (N0 + C1) -> offset 1382 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1383 Ptr = N0; 1384 } 1385 1386 if (!C1) { 1387 // No offset. 1388 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1389 return true; 1390 } 1391 1392 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1393 // Legal offset for instruction. 1394 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1395 return true; 1396 } 1397 1398 // Illegal offset, store it in soffset. 1399 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1400 SOffset = 1401 SDValue(CurDAG->getMachineNode( 1402 AMDGPU::S_MOV_B32, DL, MVT::i32, 1403 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1404 0); 1405 return true; 1406 } 1407 1408 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1409 SDValue &VAddr, SDValue &SOffset, 1410 SDValue &Offset, SDValue &GLC, 1411 SDValue &SLC, SDValue &TFE, 1412 SDValue &DLC, SDValue &SWZ) const { 1413 SDValue Ptr, Offen, Idxen, Addr64; 1414 1415 // addr64 bit was removed for volcanic islands. 1416 // FIXME: This should be a pattern predicate and not reach here 1417 if (!Subtarget->hasAddr64()) 1418 return false; 1419 1420 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1421 GLC, SLC, TFE, DLC, SWZ)) 1422 return false; 1423 1424 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1425 if (C->getSExtValue()) { 1426 SDLoc DL(Addr); 1427 1428 const SITargetLowering& Lowering = 1429 *static_cast<const SITargetLowering*>(getTargetLowering()); 1430 1431 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1432 return true; 1433 } 1434 1435 return false; 1436 } 1437 1438 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1439 SDValue &VAddr, SDValue &SOffset, 1440 SDValue &Offset, 1441 SDValue &SLC) const { 1442 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1); 1443 SDValue GLC, TFE, DLC, SWZ; 1444 1445 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1446 } 1447 1448 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 1449 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 1450 return PSV && PSV->isStack(); 1451 } 1452 1453 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1454 SDLoc DL(N); 1455 const MachineFunction &MF = CurDAG->getMachineFunction(); 1456 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1457 1458 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) { 1459 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1460 FI->getValueType(0)); 1461 1462 // If we can resolve this to a frame index access, this will be relative to 1463 // either the stack or frame pointer SGPR. 1464 return std::make_pair( 1465 TFI, CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)); 1466 } 1467 1468 // If we don't know this private access is a local stack object, it needs to 1469 // be relative to the entry point's scratch wave offset. 1470 return std::make_pair(N, CurDAG->getTargetConstant(0, DL, MVT::i32)); 1471 } 1472 1473 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1474 SDValue Addr, SDValue &Rsrc, 1475 SDValue &VAddr, SDValue &SOffset, 1476 SDValue &ImmOffset) const { 1477 1478 SDLoc DL(Addr); 1479 MachineFunction &MF = CurDAG->getMachineFunction(); 1480 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1481 1482 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1483 1484 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1485 unsigned Imm = CAddr->getZExtValue(); 1486 1487 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1488 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1489 DL, MVT::i32, HighBits); 1490 VAddr = SDValue(MovHighBits, 0); 1491 1492 // In a call sequence, stores to the argument stack area are relative to the 1493 // stack pointer. 1494 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1495 1496 SOffset = isStackPtrRelative(PtrInfo) 1497 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1498 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1499 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1500 return true; 1501 } 1502 1503 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1504 // (add n0, c1) 1505 1506 SDValue N0 = Addr.getOperand(0); 1507 SDValue N1 = Addr.getOperand(1); 1508 1509 // Offsets in vaddr must be positive if range checking is enabled. 1510 // 1511 // The total computation of vaddr + soffset + offset must not overflow. If 1512 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1513 // overflowing. 1514 // 1515 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1516 // always perform a range check. If a negative vaddr base index was used, 1517 // this would fail the range check. The overall address computation would 1518 // compute a valid address, but this doesn't happen due to the range 1519 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1520 // 1521 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1522 // MUBUF vaddr, but not on older subtargets which can only do this if the 1523 // sign bit is known 0. 1524 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1525 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1526 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1527 CurDAG->SignBitIsZero(N0))) { 1528 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1529 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1530 return true; 1531 } 1532 } 1533 1534 // (node) 1535 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1536 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1537 return true; 1538 } 1539 1540 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1541 SDValue Addr, 1542 SDValue &SRsrc, 1543 SDValue &SOffset, 1544 SDValue &Offset) const { 1545 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr); 1546 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1547 return false; 1548 1549 SDLoc DL(Addr); 1550 MachineFunction &MF = CurDAG->getMachineFunction(); 1551 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1552 1553 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1554 1555 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1556 1557 // FIXME: Get from MachinePointerInfo? We should only be using the frame 1558 // offset if we know this is in a call sequence. 1559 SOffset = isStackPtrRelative(PtrInfo) 1560 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1561 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1562 1563 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1564 return true; 1565 } 1566 1567 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1568 SDValue &SOffset, SDValue &Offset, 1569 SDValue &GLC, SDValue &SLC, 1570 SDValue &TFE, SDValue &DLC, 1571 SDValue &SWZ) const { 1572 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1573 const SIInstrInfo *TII = 1574 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1575 1576 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1577 GLC, SLC, TFE, DLC, SWZ)) 1578 return false; 1579 1580 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1581 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1582 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1583 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1584 APInt::getAllOnesValue(32).getZExtValue(); // Size 1585 SDLoc DL(Addr); 1586 1587 const SITargetLowering& Lowering = 1588 *static_cast<const SITargetLowering*>(getTargetLowering()); 1589 1590 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1591 return true; 1592 } 1593 return false; 1594 } 1595 1596 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1597 SDValue &Soffset, SDValue &Offset 1598 ) const { 1599 SDValue GLC, SLC, TFE, DLC, SWZ; 1600 1601 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1602 } 1603 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1604 SDValue &Soffset, SDValue &Offset, 1605 SDValue &SLC) const { 1606 SDValue GLC, TFE, DLC, SWZ; 1607 1608 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1609 } 1610 1611 // Find a load or store from corresponding pattern root. 1612 // Roots may be build_vector, bitconvert or their combinations. 1613 static MemSDNode* findMemSDNode(SDNode *N) { 1614 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode(); 1615 if (MemSDNode *MN = dyn_cast<MemSDNode>(N)) 1616 return MN; 1617 assert(isa<BuildVectorSDNode>(N)); 1618 for (SDValue V : N->op_values()) 1619 if (MemSDNode *MN = 1620 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V))) 1621 return MN; 1622 llvm_unreachable("cannot find MemSDNode in the pattern!"); 1623 } 1624 1625 template <bool IsSigned> 1626 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, 1627 SDValue Addr, 1628 SDValue &VAddr, 1629 SDValue &Offset, 1630 SDValue &SLC) const { 1631 int64_t OffsetVal = 0; 1632 1633 if (Subtarget->hasFlatInstOffsets() && 1634 (!Subtarget->hasFlatSegmentOffsetBug() || 1635 findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) && 1636 CurDAG->isBaseWithConstantOffset(Addr)) { 1637 SDValue N0 = Addr.getOperand(0); 1638 SDValue N1 = Addr.getOperand(1); 1639 uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1640 1641 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1642 unsigned AS = findMemSDNode(N)->getAddressSpace(); 1643 if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) { 1644 Addr = N0; 1645 OffsetVal = COffsetVal; 1646 } else { 1647 // If the offset doesn't fit, put the low bits into the offset field and 1648 // add the rest. 1649 1650 SDLoc DL(N); 1651 uint64_t ImmField; 1652 const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned); 1653 if (IsSigned) { 1654 ImmField = SignExtend64(COffsetVal, NumBits); 1655 1656 // Don't use a negative offset field if the base offset is positive. 1657 // Since the scheduler currently relies on the offset field, doing so 1658 // could result in strange scheduling decisions. 1659 1660 // TODO: Should we not do this in the opposite direction as well? 1661 if (static_cast<int64_t>(COffsetVal) > 0) { 1662 if (static_cast<int64_t>(ImmField) < 0) { 1663 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits - 1); 1664 ImmField = COffsetVal & OffsetMask; 1665 } 1666 } 1667 } else { 1668 // TODO: Should we do this for a negative offset? 1669 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits); 1670 ImmField = COffsetVal & OffsetMask; 1671 } 1672 1673 uint64_t RemainderOffset = COffsetVal - ImmField; 1674 1675 assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned)); 1676 assert(RemainderOffset + ImmField == COffsetVal); 1677 1678 OffsetVal = ImmField; 1679 1680 // TODO: Should this try to use a scalar add pseudo if the base address is 1681 // uniform and saddr is usable? 1682 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1683 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1684 1685 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1686 DL, MVT::i32, N0, Sub0); 1687 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1688 DL, MVT::i32, N0, Sub1); 1689 1690 SDValue AddOffsetLo 1691 = getMaterializedScalarImm32(Lo_32(RemainderOffset), DL); 1692 SDValue AddOffsetHi 1693 = getMaterializedScalarImm32(Hi_32(RemainderOffset), DL); 1694 1695 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1); 1696 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1697 1698 SDNode *Add = CurDAG->getMachineNode( 1699 AMDGPU::V_ADD_I32_e64, DL, VTs, 1700 {AddOffsetLo, SDValue(N0Lo, 0), Clamp}); 1701 1702 SDNode *Addc = CurDAG->getMachineNode( 1703 AMDGPU::V_ADDC_U32_e64, DL, VTs, 1704 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp}); 1705 1706 SDValue RegSequenceArgs[] = { 1707 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32), 1708 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1 1709 }; 1710 1711 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1712 MVT::i64, RegSequenceArgs), 0); 1713 } 1714 } 1715 1716 VAddr = Addr; 1717 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1718 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); 1719 return true; 1720 } 1721 1722 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDNode *N, 1723 SDValue Addr, 1724 SDValue &VAddr, 1725 SDValue &Offset, 1726 SDValue &SLC) const { 1727 return SelectFlatOffset<false>(N, Addr, VAddr, Offset, SLC); 1728 } 1729 1730 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N, 1731 SDValue Addr, 1732 SDValue &VAddr, 1733 SDValue &Offset, 1734 SDValue &SLC) const { 1735 return SelectFlatOffset<true>(N, Addr, VAddr, Offset, SLC); 1736 } 1737 1738 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 1739 SDValue &Offset, bool &Imm) const { 1740 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 1741 if (!C) { 1742 if (ByteOffsetNode.getValueType().isScalarInteger() && 1743 ByteOffsetNode.getValueType().getSizeInBits() == 32) { 1744 Offset = ByteOffsetNode; 1745 Imm = false; 1746 return true; 1747 } 1748 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) { 1749 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) { 1750 Offset = ByteOffsetNode.getOperand(0); 1751 Imm = false; 1752 return true; 1753 } 1754 } 1755 return false; 1756 } 1757 1758 SDLoc SL(ByteOffsetNode); 1759 // GFX9 and GFX10 have signed byte immediate offsets. 1760 int64_t ByteOffset = C->getSExtValue(); 1761 Optional<int64_t> EncodedOffset = 1762 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false); 1763 if (EncodedOffset) { 1764 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1765 Imm = true; 1766 return true; 1767 } 1768 1769 // SGPR and literal offsets are unsigned. 1770 if (ByteOffset < 0) 1771 return false; 1772 1773 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset); 1774 if (EncodedOffset) { 1775 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1776 return true; 1777 } 1778 1779 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset)) 1780 return false; 1781 1782 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 1783 Offset = SDValue( 1784 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0); 1785 1786 return true; 1787 } 1788 1789 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 1790 if (Addr.getValueType() != MVT::i32) 1791 return Addr; 1792 1793 // Zero-extend a 32-bit address. 1794 SDLoc SL(Addr); 1795 1796 const MachineFunction &MF = CurDAG->getMachineFunction(); 1797 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1798 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 1799 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 1800 1801 const SDValue Ops[] = { 1802 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 1803 Addr, 1804 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 1805 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 1806 0), 1807 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 1808 }; 1809 1810 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 1811 Ops), 0); 1812 } 1813 1814 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 1815 SDValue &Offset, bool &Imm) const { 1816 SDLoc SL(Addr); 1817 1818 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 1819 // wraparound, because s_load instructions perform the addition in 64 bits. 1820 if ((Addr.getValueType() != MVT::i32 || 1821 Addr->getFlags().hasNoUnsignedWrap()) && 1822 (CurDAG->isBaseWithConstantOffset(Addr) || 1823 Addr.getOpcode() == ISD::ADD)) { 1824 SDValue N0 = Addr.getOperand(0); 1825 SDValue N1 = Addr.getOperand(1); 1826 1827 if (SelectSMRDOffset(N1, Offset, Imm)) { 1828 SBase = Expand32BitAddress(N0); 1829 return true; 1830 } 1831 } 1832 SBase = Expand32BitAddress(Addr); 1833 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 1834 Imm = true; 1835 return true; 1836 } 1837 1838 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 1839 SDValue &Offset) const { 1840 bool Imm = false; 1841 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 1842 } 1843 1844 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 1845 SDValue &Offset) const { 1846 1847 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1848 1849 bool Imm = false; 1850 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 1851 return false; 1852 1853 return !Imm && isa<ConstantSDNode>(Offset); 1854 } 1855 1856 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 1857 SDValue &Offset) const { 1858 bool Imm = false; 1859 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 1860 !isa<ConstantSDNode>(Offset); 1861 } 1862 1863 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 1864 SDValue &Offset) const { 1865 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1866 // The immediate offset for S_BUFFER instructions is unsigned. 1867 if (auto Imm = 1868 AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) { 1869 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1870 return true; 1871 } 1872 } 1873 1874 return false; 1875 } 1876 1877 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 1878 SDValue &Offset) const { 1879 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1880 1881 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1882 if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, 1883 C->getZExtValue())) { 1884 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1885 return true; 1886 } 1887 } 1888 1889 return false; 1890 } 1891 1892 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 1893 SDValue &Base, 1894 SDValue &Offset) const { 1895 SDLoc DL(Index); 1896 1897 if (CurDAG->isBaseWithConstantOffset(Index)) { 1898 SDValue N0 = Index.getOperand(0); 1899 SDValue N1 = Index.getOperand(1); 1900 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1901 1902 // (add n0, c0) 1903 // Don't peel off the offset (c0) if doing so could possibly lead 1904 // the base (n0) to be negative. 1905 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) { 1906 Base = N0; 1907 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 1908 return true; 1909 } 1910 } 1911 1912 if (isa<ConstantSDNode>(Index)) 1913 return false; 1914 1915 Base = Index; 1916 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1917 return true; 1918 } 1919 1920 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 1921 SDValue Val, uint32_t Offset, 1922 uint32_t Width) { 1923 // Transformation function, pack the offset and width of a BFE into 1924 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1925 // source, bits [5:0] contain the offset and bits [22:16] the width. 1926 uint32_t PackedVal = Offset | (Width << 16); 1927 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 1928 1929 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 1930 } 1931 1932 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 1933 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 1934 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 1935 // Predicate: 0 < b <= c < 32 1936 1937 const SDValue &Shl = N->getOperand(0); 1938 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 1939 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1940 1941 if (B && C) { 1942 uint32_t BVal = B->getZExtValue(); 1943 uint32_t CVal = C->getZExtValue(); 1944 1945 if (0 < BVal && BVal <= CVal && CVal < 32) { 1946 bool Signed = N->getOpcode() == ISD::SRA; 1947 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1948 1949 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 1950 32 - CVal)); 1951 return; 1952 } 1953 } 1954 SelectCode(N); 1955 } 1956 1957 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 1958 switch (N->getOpcode()) { 1959 case ISD::AND: 1960 if (N->getOperand(0).getOpcode() == ISD::SRL) { 1961 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 1962 // Predicate: isMask(mask) 1963 const SDValue &Srl = N->getOperand(0); 1964 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 1965 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1966 1967 if (Shift && Mask) { 1968 uint32_t ShiftVal = Shift->getZExtValue(); 1969 uint32_t MaskVal = Mask->getZExtValue(); 1970 1971 if (isMask_32(MaskVal)) { 1972 uint32_t WidthVal = countPopulation(MaskVal); 1973 1974 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1975 Srl.getOperand(0), ShiftVal, WidthVal)); 1976 return; 1977 } 1978 } 1979 } 1980 break; 1981 case ISD::SRL: 1982 if (N->getOperand(0).getOpcode() == ISD::AND) { 1983 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 1984 // Predicate: isMask(mask >> b) 1985 const SDValue &And = N->getOperand(0); 1986 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1987 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 1988 1989 if (Shift && Mask) { 1990 uint32_t ShiftVal = Shift->getZExtValue(); 1991 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 1992 1993 if (isMask_32(MaskVal)) { 1994 uint32_t WidthVal = countPopulation(MaskVal); 1995 1996 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1997 And.getOperand(0), ShiftVal, WidthVal)); 1998 return; 1999 } 2000 } 2001 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 2002 SelectS_BFEFromShifts(N); 2003 return; 2004 } 2005 break; 2006 case ISD::SRA: 2007 if (N->getOperand(0).getOpcode() == ISD::SHL) { 2008 SelectS_BFEFromShifts(N); 2009 return; 2010 } 2011 break; 2012 2013 case ISD::SIGN_EXTEND_INREG: { 2014 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 2015 SDValue Src = N->getOperand(0); 2016 if (Src.getOpcode() != ISD::SRL) 2017 break; 2018 2019 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 2020 if (!Amt) 2021 break; 2022 2023 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2024 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 2025 Amt->getZExtValue(), Width)); 2026 return; 2027 } 2028 } 2029 2030 SelectCode(N); 2031 } 2032 2033 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 2034 assert(N->getOpcode() == ISD::BRCOND); 2035 if (!N->hasOneUse()) 2036 return false; 2037 2038 SDValue Cond = N->getOperand(1); 2039 if (Cond.getOpcode() == ISD::CopyToReg) 2040 Cond = Cond.getOperand(2); 2041 2042 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 2043 return false; 2044 2045 MVT VT = Cond.getOperand(0).getSimpleValueType(); 2046 if (VT == MVT::i32) 2047 return true; 2048 2049 if (VT == MVT::i64) { 2050 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 2051 2052 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 2053 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 2054 } 2055 2056 return false; 2057 } 2058 2059 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 2060 SDValue Cond = N->getOperand(1); 2061 2062 if (Cond.isUndef()) { 2063 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 2064 N->getOperand(2), N->getOperand(0)); 2065 return; 2066 } 2067 2068 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 2069 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 2070 2071 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 2072 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 2073 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC(); 2074 SDLoc SL(N); 2075 2076 if (!UseSCCBr) { 2077 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 2078 // analyzed what generates the vcc value, so we do not know whether vcc 2079 // bits for disabled lanes are 0. Thus we need to mask out bits for 2080 // disabled lanes. 2081 // 2082 // For the case that we select S_CBRANCH_SCC1 and it gets 2083 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 2084 // SIInstrInfo::moveToVALU which inserts the S_AND). 2085 // 2086 // We could add an analysis of what generates the vcc value here and omit 2087 // the S_AND when is unnecessary. But it would be better to add a separate 2088 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 2089 // catches both cases. 2090 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32 2091 : AMDGPU::S_AND_B64, 2092 SL, MVT::i1, 2093 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO 2094 : AMDGPU::EXEC, 2095 MVT::i1), 2096 Cond), 2097 0); 2098 } 2099 2100 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 2101 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 2102 N->getOperand(2), // Basic Block 2103 VCC.getValue(0)); 2104 } 2105 2106 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 2107 MVT VT = N->getSimpleValueType(0); 2108 bool IsFMA = N->getOpcode() == ISD::FMA; 2109 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 2110 !Subtarget->hasFmaMixInsts()) || 2111 ((IsFMA && Subtarget->hasMadMixInsts()) || 2112 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 2113 SelectCode(N); 2114 return; 2115 } 2116 2117 SDValue Src0 = N->getOperand(0); 2118 SDValue Src1 = N->getOperand(1); 2119 SDValue Src2 = N->getOperand(2); 2120 unsigned Src0Mods, Src1Mods, Src2Mods; 2121 2122 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 2123 // using the conversion from f16. 2124 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 2125 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 2126 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 2127 2128 assert((IsFMA || !Mode.allFP32Denormals()) && 2129 "fmad selected with denormals enabled"); 2130 // TODO: We can select this with f32 denormals enabled if all the sources are 2131 // converted from f16 (in which case fmad isn't legal). 2132 2133 if (Sel0 || Sel1 || Sel2) { 2134 // For dummy operands. 2135 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2136 SDValue Ops[] = { 2137 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 2138 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 2139 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 2140 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 2141 Zero, Zero 2142 }; 2143 2144 CurDAG->SelectNodeTo(N, 2145 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 2146 MVT::f32, Ops); 2147 } else { 2148 SelectCode(N); 2149 } 2150 } 2151 2152 // This is here because there isn't a way to use the generated sub0_sub1 as the 2153 // subreg index to EXTRACT_SUBREG in tablegen. 2154 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 2155 MemSDNode *Mem = cast<MemSDNode>(N); 2156 unsigned AS = Mem->getAddressSpace(); 2157 if (AS == AMDGPUAS::FLAT_ADDRESS) { 2158 SelectCode(N); 2159 return; 2160 } 2161 2162 MVT VT = N->getSimpleValueType(0); 2163 bool Is32 = (VT == MVT::i32); 2164 SDLoc SL(N); 2165 2166 MachineSDNode *CmpSwap = nullptr; 2167 if (Subtarget->hasAddr64()) { 2168 SDValue SRsrc, VAddr, SOffset, Offset, SLC; 2169 2170 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) { 2171 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 2172 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 2173 SDValue CmpVal = Mem->getOperand(2); 2174 2175 // XXX - Do we care about glue operands? 2176 2177 SDValue Ops[] = { 2178 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2179 }; 2180 2181 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2182 } 2183 } 2184 2185 if (!CmpSwap) { 2186 SDValue SRsrc, SOffset, Offset, SLC; 2187 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) { 2188 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 2189 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 2190 2191 SDValue CmpVal = Mem->getOperand(2); 2192 SDValue Ops[] = { 2193 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2194 }; 2195 2196 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2197 } 2198 } 2199 2200 if (!CmpSwap) { 2201 SelectCode(N); 2202 return; 2203 } 2204 2205 MachineMemOperand *MMO = Mem->getMemOperand(); 2206 CurDAG->setNodeMemRefs(CmpSwap, {MMO}); 2207 2208 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 2209 SDValue Extract 2210 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 2211 2212 ReplaceUses(SDValue(N, 0), Extract); 2213 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 2214 CurDAG->RemoveDeadNode(N); 2215 } 2216 2217 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { 2218 // The address is assumed to be uniform, so if it ends up in a VGPR, it will 2219 // be copied to an SGPR with readfirstlane. 2220 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ? 2221 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 2222 2223 SDValue Chain = N->getOperand(0); 2224 SDValue Ptr = N->getOperand(2); 2225 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2226 MachineMemOperand *MMO = M->getMemOperand(); 2227 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 2228 2229 SDValue Offset; 2230 if (CurDAG->isBaseWithConstantOffset(Ptr)) { 2231 SDValue PtrBase = Ptr.getOperand(0); 2232 SDValue PtrOffset = Ptr.getOperand(1); 2233 2234 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue(); 2235 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) { 2236 N = glueCopyToM0(N, PtrBase); 2237 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32); 2238 } 2239 } 2240 2241 if (!Offset) { 2242 N = glueCopyToM0(N, Ptr); 2243 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2244 } 2245 2246 SDValue Ops[] = { 2247 Offset, 2248 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32), 2249 Chain, 2250 N->getOperand(N->getNumOperands() - 1) // New glue 2251 }; 2252 2253 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2254 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2255 } 2256 2257 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 2258 switch (IntrID) { 2259 case Intrinsic::amdgcn_ds_gws_init: 2260 return AMDGPU::DS_GWS_INIT; 2261 case Intrinsic::amdgcn_ds_gws_barrier: 2262 return AMDGPU::DS_GWS_BARRIER; 2263 case Intrinsic::amdgcn_ds_gws_sema_v: 2264 return AMDGPU::DS_GWS_SEMA_V; 2265 case Intrinsic::amdgcn_ds_gws_sema_br: 2266 return AMDGPU::DS_GWS_SEMA_BR; 2267 case Intrinsic::amdgcn_ds_gws_sema_p: 2268 return AMDGPU::DS_GWS_SEMA_P; 2269 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2270 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 2271 default: 2272 llvm_unreachable("not a gws intrinsic"); 2273 } 2274 } 2275 2276 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { 2277 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all && 2278 !Subtarget->hasGWSSemaReleaseAll()) { 2279 // Let this error. 2280 SelectCode(N); 2281 return; 2282 } 2283 2284 // Chain, intrinsic ID, vsrc, offset 2285 const bool HasVSrc = N->getNumOperands() == 4; 2286 assert(HasVSrc || N->getNumOperands() == 3); 2287 2288 SDLoc SL(N); 2289 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2); 2290 int ImmOffset = 0; 2291 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2292 MachineMemOperand *MMO = M->getMemOperand(); 2293 2294 // Don't worry if the offset ends up in a VGPR. Only one lane will have 2295 // effect, so SIFixSGPRCopies will validly insert readfirstlane. 2296 2297 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 2298 // offset field) % 64. Some versions of the programming guide omit the m0 2299 // part, or claim it's from offset 0. 2300 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) { 2301 // If we have a constant offset, try to use the 0 in m0 as the base. 2302 // TODO: Look into changing the default m0 initialization value. If the 2303 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 2304 // the immediate offset. 2305 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); 2306 ImmOffset = ConstOffset->getZExtValue(); 2307 } else { 2308 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { 2309 ImmOffset = BaseOffset.getConstantOperandVal(1); 2310 BaseOffset = BaseOffset.getOperand(0); 2311 } 2312 2313 // Prefer to do the shift in an SGPR since it should be possible to use m0 2314 // as the result directly. If it's already an SGPR, it will be eliminated 2315 // later. 2316 SDNode *SGPROffset 2317 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, 2318 BaseOffset); 2319 // Shift to offset in m0 2320 SDNode *M0Base 2321 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32, 2322 SDValue(SGPROffset, 0), 2323 CurDAG->getTargetConstant(16, SL, MVT::i32)); 2324 glueCopyToM0(N, SDValue(M0Base, 0)); 2325 } 2326 2327 SDValue Chain = N->getOperand(0); 2328 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32); 2329 2330 // TODO: Can this just be removed from the instruction? 2331 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1); 2332 2333 const unsigned Opc = gwsIntrinToOpcode(IntrID); 2334 SmallVector<SDValue, 5> Ops; 2335 if (HasVSrc) 2336 Ops.push_back(N->getOperand(2)); 2337 Ops.push_back(OffsetField); 2338 Ops.push_back(GDS); 2339 Ops.push_back(Chain); 2340 2341 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2342 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2343 } 2344 2345 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) { 2346 if (Subtarget->getLDSBankCount() != 16) { 2347 // This is a single instruction with a pattern. 2348 SelectCode(N); 2349 return; 2350 } 2351 2352 SDLoc DL(N); 2353 2354 // This requires 2 instructions. It is possible to write a pattern to support 2355 // this, but the generated isel emitter doesn't correctly deal with multiple 2356 // output instructions using the same physical register input. The copy to m0 2357 // is incorrectly placed before the second instruction. 2358 // 2359 // TODO: Match source modifiers. 2360 // 2361 // def : Pat < 2362 // (int_amdgcn_interp_p1_f16 2363 // (VOP3Mods f32:$src0, i32:$src0_modifiers), 2364 // (i32 timm:$attrchan), (i32 timm:$attr), 2365 // (i1 timm:$high), M0), 2366 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr, 2367 // timm:$attrchan, 0, 2368 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> { 2369 // let Predicates = [has16BankLDS]; 2370 // } 2371 2372 // 16 bank LDS 2373 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0, 2374 N->getOperand(5), SDValue()); 2375 2376 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other); 2377 2378 SDNode *InterpMov = 2379 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, { 2380 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0 2381 N->getOperand(3), // Attr 2382 N->getOperand(2), // Attrchan 2383 ToM0.getValue(1) // In glue 2384 }); 2385 2386 SDNode *InterpP1LV = 2387 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, { 2388 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers 2389 N->getOperand(1), // Src0 2390 N->getOperand(3), // Attr 2391 N->getOperand(2), // Attrchan 2392 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers 2393 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high 2394 N->getOperand(4), // high 2395 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp 2396 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod 2397 SDValue(InterpMov, 1) 2398 }); 2399 2400 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0)); 2401 } 2402 2403 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { 2404 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2405 switch (IntrID) { 2406 case Intrinsic::amdgcn_ds_append: 2407 case Intrinsic::amdgcn_ds_consume: { 2408 if (N->getValueType(0) != MVT::i32) 2409 break; 2410 SelectDSAppendConsume(N, IntrID); 2411 return; 2412 } 2413 } 2414 2415 SelectCode(N); 2416 } 2417 2418 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { 2419 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 2420 unsigned Opcode; 2421 switch (IntrID) { 2422 case Intrinsic::amdgcn_wqm: 2423 Opcode = AMDGPU::WQM; 2424 break; 2425 case Intrinsic::amdgcn_softwqm: 2426 Opcode = AMDGPU::SOFT_WQM; 2427 break; 2428 case Intrinsic::amdgcn_wwm: 2429 Opcode = AMDGPU::WWM; 2430 break; 2431 case Intrinsic::amdgcn_interp_p1_f16: 2432 SelectInterpP1F16(N); 2433 return; 2434 default: 2435 SelectCode(N); 2436 return; 2437 } 2438 2439 SDValue Src = N->getOperand(1); 2440 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); 2441 } 2442 2443 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { 2444 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2445 switch (IntrID) { 2446 case Intrinsic::amdgcn_ds_gws_init: 2447 case Intrinsic::amdgcn_ds_gws_barrier: 2448 case Intrinsic::amdgcn_ds_gws_sema_v: 2449 case Intrinsic::amdgcn_ds_gws_sema_br: 2450 case Intrinsic::amdgcn_ds_gws_sema_p: 2451 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2452 SelectDS_GWS(N, IntrID); 2453 return; 2454 default: 2455 break; 2456 } 2457 2458 SelectCode(N); 2459 } 2460 2461 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 2462 unsigned &Mods) const { 2463 Mods = 0; 2464 Src = In; 2465 2466 if (Src.getOpcode() == ISD::FNEG) { 2467 Mods |= SISrcMods::NEG; 2468 Src = Src.getOperand(0); 2469 } 2470 2471 if (Src.getOpcode() == ISD::FABS) { 2472 Mods |= SISrcMods::ABS; 2473 Src = Src.getOperand(0); 2474 } 2475 2476 return true; 2477 } 2478 2479 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 2480 SDValue &SrcMods) const { 2481 unsigned Mods; 2482 if (SelectVOP3ModsImpl(In, Src, Mods)) { 2483 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2484 return true; 2485 } 2486 2487 return false; 2488 } 2489 2490 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 2491 SDValue &SrcMods) const { 2492 SelectVOP3Mods(In, Src, SrcMods); 2493 return isNoNanSrc(Src); 2494 } 2495 2496 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 2497 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 2498 return false; 2499 2500 Src = In; 2501 return true; 2502 } 2503 2504 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 2505 SDValue &SrcMods, SDValue &Clamp, 2506 SDValue &Omod) const { 2507 SDLoc DL(In); 2508 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2509 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2510 2511 return SelectVOP3Mods(In, Src, SrcMods); 2512 } 2513 2514 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 2515 SDValue &Clamp, SDValue &Omod) const { 2516 Src = In; 2517 2518 SDLoc DL(In); 2519 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2520 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2521 2522 return true; 2523 } 2524 2525 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 2526 SDValue &SrcMods) const { 2527 unsigned Mods = 0; 2528 Src = In; 2529 2530 if (Src.getOpcode() == ISD::FNEG) { 2531 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 2532 Src = Src.getOperand(0); 2533 } 2534 2535 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 2536 unsigned VecMods = Mods; 2537 2538 SDValue Lo = stripBitcast(Src.getOperand(0)); 2539 SDValue Hi = stripBitcast(Src.getOperand(1)); 2540 2541 if (Lo.getOpcode() == ISD::FNEG) { 2542 Lo = stripBitcast(Lo.getOperand(0)); 2543 Mods ^= SISrcMods::NEG; 2544 } 2545 2546 if (Hi.getOpcode() == ISD::FNEG) { 2547 Hi = stripBitcast(Hi.getOperand(0)); 2548 Mods ^= SISrcMods::NEG_HI; 2549 } 2550 2551 if (isExtractHiElt(Lo, Lo)) 2552 Mods |= SISrcMods::OP_SEL_0; 2553 2554 if (isExtractHiElt(Hi, Hi)) 2555 Mods |= SISrcMods::OP_SEL_1; 2556 2557 Lo = stripExtractLoElt(Lo); 2558 Hi = stripExtractLoElt(Hi); 2559 2560 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 2561 // Really a scalar input. Just select from the low half of the register to 2562 // avoid packing. 2563 2564 Src = Lo; 2565 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2566 return true; 2567 } 2568 2569 Mods = VecMods; 2570 } 2571 2572 // Packed instructions do not have abs modifiers. 2573 Mods |= SISrcMods::OP_SEL_1; 2574 2575 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2576 return true; 2577 } 2578 2579 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2580 SDValue &SrcMods) const { 2581 Src = In; 2582 // FIXME: Handle op_sel 2583 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2584 return true; 2585 } 2586 2587 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2588 SDValue &SrcMods) const { 2589 // FIXME: Handle op_sel 2590 return SelectVOP3Mods(In, Src, SrcMods); 2591 } 2592 2593 // The return value is not whether the match is possible (which it always is), 2594 // but whether or not it a conversion is really used. 2595 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2596 unsigned &Mods) const { 2597 Mods = 0; 2598 SelectVOP3ModsImpl(In, Src, Mods); 2599 2600 if (Src.getOpcode() == ISD::FP_EXTEND) { 2601 Src = Src.getOperand(0); 2602 assert(Src.getValueType() == MVT::f16); 2603 Src = stripBitcast(Src); 2604 2605 // Be careful about folding modifiers if we already have an abs. fneg is 2606 // applied last, so we don't want to apply an earlier fneg. 2607 if ((Mods & SISrcMods::ABS) == 0) { 2608 unsigned ModsTmp; 2609 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2610 2611 if ((ModsTmp & SISrcMods::NEG) != 0) 2612 Mods ^= SISrcMods::NEG; 2613 2614 if ((ModsTmp & SISrcMods::ABS) != 0) 2615 Mods |= SISrcMods::ABS; 2616 } 2617 2618 // op_sel/op_sel_hi decide the source type and source. 2619 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2620 // If the sources's op_sel is set, it picks the high half of the source 2621 // register. 2622 2623 Mods |= SISrcMods::OP_SEL_1; 2624 if (isExtractHiElt(Src, Src)) { 2625 Mods |= SISrcMods::OP_SEL_0; 2626 2627 // TODO: Should we try to look for neg/abs here? 2628 } 2629 2630 return true; 2631 } 2632 2633 return false; 2634 } 2635 2636 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2637 SDValue &SrcMods) const { 2638 unsigned Mods = 0; 2639 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2640 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2641 return true; 2642 } 2643 2644 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { 2645 if (In.isUndef()) 2646 return CurDAG->getUNDEF(MVT::i32); 2647 2648 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2649 SDLoc SL(In); 2650 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32); 2651 } 2652 2653 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 2654 SDLoc SL(In); 2655 return CurDAG->getConstant( 2656 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 2657 } 2658 2659 SDValue Src; 2660 if (isExtractHiElt(In, Src)) 2661 return Src; 2662 2663 return SDValue(); 2664 } 2665 2666 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const { 2667 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn); 2668 2669 const SIRegisterInfo *SIRI = 2670 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 2671 const SIInstrInfo * SII = 2672 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 2673 2674 unsigned Limit = 0; 2675 bool AllUsesAcceptSReg = true; 2676 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); 2677 Limit < 10 && U != E; ++U, ++Limit) { 2678 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); 2679 2680 // If the register class is unknown, it could be an unknown 2681 // register class that needs to be an SGPR, e.g. an inline asm 2682 // constraint 2683 if (!RC || SIRI->isSGPRClass(RC)) 2684 return false; 2685 2686 if (RC != &AMDGPU::VS_32RegClass) { 2687 AllUsesAcceptSReg = false; 2688 SDNode * User = *U; 2689 if (User->isMachineOpcode()) { 2690 unsigned Opc = User->getMachineOpcode(); 2691 MCInstrDesc Desc = SII->get(Opc); 2692 if (Desc.isCommutable()) { 2693 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo(); 2694 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 2695 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) { 2696 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs(); 2697 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo); 2698 if (CommutedRC == &AMDGPU::VS_32RegClass) 2699 AllUsesAcceptSReg = true; 2700 } 2701 } 2702 } 2703 // If "AllUsesAcceptSReg == false" so far we haven't suceeded 2704 // commuting current user. This means have at least one use 2705 // that strictly require VGPR. Thus, we will not attempt to commute 2706 // other user instructions. 2707 if (!AllUsesAcceptSReg) 2708 break; 2709 } 2710 } 2711 return !AllUsesAcceptSReg && (Limit < 10); 2712 } 2713 2714 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const { 2715 auto Ld = cast<LoadSDNode>(N); 2716 2717 return Ld->getAlignment() >= 4 && 2718 ( 2719 ( 2720 ( 2721 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2722 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT 2723 ) 2724 && 2725 !N->isDivergent() 2726 ) 2727 || 2728 ( 2729 Subtarget->getScalarizeGlobalBehavior() && 2730 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && 2731 !Ld->isVolatile() && 2732 !N->isDivergent() && 2733 static_cast<const SITargetLowering *>( 2734 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N) 2735 ) 2736 ); 2737 } 2738 2739 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 2740 const AMDGPUTargetLowering& Lowering = 2741 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 2742 bool IsModified = false; 2743 do { 2744 IsModified = false; 2745 2746 // Go over all selected nodes and try to fold them a bit more 2747 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 2748 while (Position != CurDAG->allnodes_end()) { 2749 SDNode *Node = &*Position++; 2750 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 2751 if (!MachineNode) 2752 continue; 2753 2754 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 2755 if (ResNode != Node) { 2756 if (ResNode) 2757 ReplaceUses(Node, ResNode); 2758 IsModified = true; 2759 } 2760 } 2761 CurDAG->RemoveDeadNodes(); 2762 } while (IsModified); 2763 } 2764 2765 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 2766 Subtarget = &MF.getSubtarget<R600Subtarget>(); 2767 return SelectionDAGISel::runOnMachineFunction(MF); 2768 } 2769 2770 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 2771 if (!N->readMem()) 2772 return false; 2773 if (CbId == -1) 2774 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2775 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 2776 2777 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; 2778 } 2779 2780 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 2781 SDValue& IntPtr) { 2782 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 2783 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 2784 true); 2785 return true; 2786 } 2787 return false; 2788 } 2789 2790 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 2791 SDValue& BaseReg, SDValue &Offset) { 2792 if (!isa<ConstantSDNode>(Addr)) { 2793 BaseReg = Addr; 2794 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 2795 return true; 2796 } 2797 return false; 2798 } 2799 2800 void R600DAGToDAGISel::Select(SDNode *N) { 2801 unsigned int Opc = N->getOpcode(); 2802 if (N->isMachineOpcode()) { 2803 N->setNodeId(-1); 2804 return; // Already selected. 2805 } 2806 2807 switch (Opc) { 2808 default: break; 2809 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 2810 case ISD::SCALAR_TO_VECTOR: 2811 case ISD::BUILD_VECTOR: { 2812 EVT VT = N->getValueType(0); 2813 unsigned NumVectorElts = VT.getVectorNumElements(); 2814 unsigned RegClassID; 2815 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 2816 // that adds a 128 bits reg copy when going through TwoAddressInstructions 2817 // pass. We want to avoid 128 bits copies as much as possible because they 2818 // can't be bundled by our scheduler. 2819 switch(NumVectorElts) { 2820 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 2821 case 4: 2822 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 2823 RegClassID = R600::R600_Reg128VerticalRegClassID; 2824 else 2825 RegClassID = R600::R600_Reg128RegClassID; 2826 break; 2827 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 2828 } 2829 SelectBuildVector(N, RegClassID); 2830 return; 2831 } 2832 } 2833 2834 SelectCode(N); 2835 } 2836 2837 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 2838 SDValue &Offset) { 2839 ConstantSDNode *C; 2840 SDLoc DL(Addr); 2841 2842 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 2843 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2844 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2845 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 2846 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 2847 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2848 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2849 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 2850 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 2851 Base = Addr.getOperand(0); 2852 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2853 } else { 2854 Base = Addr; 2855 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2856 } 2857 2858 return true; 2859 } 2860 2861 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 2862 SDValue &Offset) { 2863 ConstantSDNode *IMMOffset; 2864 2865 if (Addr.getOpcode() == ISD::ADD 2866 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 2867 && isInt<16>(IMMOffset->getZExtValue())) { 2868 2869 Base = Addr.getOperand(0); 2870 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2871 MVT::i32); 2872 return true; 2873 // If the pointer address is constant, we can move it to the offset field. 2874 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 2875 && isInt<16>(IMMOffset->getZExtValue())) { 2876 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 2877 SDLoc(CurDAG->getEntryNode()), 2878 R600::ZERO, MVT::i32); 2879 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2880 MVT::i32); 2881 return true; 2882 } 2883 2884 // Default case, no offset 2885 Base = Addr; 2886 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 2887 return true; 2888 } 2889