1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// \brief Custom DAG lowering for R600 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "R600ISelLowering.h" 16 #include "AMDGPUFrameLowering.h" 17 #include "AMDGPUIntrinsicInfo.h" 18 #include "AMDGPUSubtarget.h" 19 #include "R600Defines.h" 20 #include "R600InstrInfo.h" 21 #include "R600MachineFunctionInfo.h" 22 #include "llvm/Analysis/ValueTracking.h" 23 #include "llvm/CodeGen/CallingConvLower.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/MachineInstrBuilder.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/SelectionDAG.h" 28 #include "llvm/IR/Argument.h" 29 #include "llvm/IR/Function.h" 30 31 using namespace llvm; 32 33 R600TargetLowering::R600TargetLowering(const TargetMachine &TM, 34 const R600Subtarget &STI) 35 : AMDGPUTargetLowering(TM, STI), Gen(STI.getGeneration()) { 36 addRegisterClass(MVT::f32, &AMDGPU::R600_Reg32RegClass); 37 addRegisterClass(MVT::i32, &AMDGPU::R600_Reg32RegClass); 38 addRegisterClass(MVT::v2f32, &AMDGPU::R600_Reg64RegClass); 39 addRegisterClass(MVT::v2i32, &AMDGPU::R600_Reg64RegClass); 40 addRegisterClass(MVT::v4f32, &AMDGPU::R600_Reg128RegClass); 41 addRegisterClass(MVT::v4i32, &AMDGPU::R600_Reg128RegClass); 42 43 computeRegisterProperties(STI.getRegisterInfo()); 44 45 // Legalize loads and stores to the private address space. 46 setOperationAction(ISD::LOAD, MVT::i32, Custom); 47 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 48 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 49 50 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address 51 // spaces, so it is custom lowered to handle those where it isn't. 52 for (MVT VT : MVT::integer_valuetypes()) { 53 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 54 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Custom); 55 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Custom); 56 57 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 58 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Custom); 59 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Custom); 60 61 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 62 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Custom); 63 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Custom); 64 } 65 66 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads. 67 setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 68 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 69 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 70 71 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 72 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 73 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 74 75 76 setOperationAction(ISD::STORE, MVT::i8, Custom); 77 setOperationAction(ISD::STORE, MVT::i32, Custom); 78 setOperationAction(ISD::STORE, MVT::v2i32, Custom); 79 setOperationAction(ISD::STORE, MVT::v4i32, Custom); 80 81 setTruncStoreAction(MVT::i32, MVT::i8, Custom); 82 setTruncStoreAction(MVT::i32, MVT::i16, Custom); 83 84 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores. 85 setTruncStoreAction(MVT::v2i32, MVT::v2i1, Expand); 86 setTruncStoreAction(MVT::v4i32, MVT::v4i1, Expand); 87 88 // Set condition code actions 89 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 90 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 91 setCondCodeAction(ISD::SETLT, MVT::f32, Expand); 92 setCondCodeAction(ISD::SETLE, MVT::f32, Expand); 93 setCondCodeAction(ISD::SETOLT, MVT::f32, Expand); 94 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 95 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 96 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 97 setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); 98 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 99 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 100 setCondCodeAction(ISD::SETULE, MVT::f32, Expand); 101 102 setCondCodeAction(ISD::SETLE, MVT::i32, Expand); 103 setCondCodeAction(ISD::SETLT, MVT::i32, Expand); 104 setCondCodeAction(ISD::SETULE, MVT::i32, Expand); 105 setCondCodeAction(ISD::SETULT, MVT::i32, Expand); 106 107 setOperationAction(ISD::FCOS, MVT::f32, Custom); 108 setOperationAction(ISD::FSIN, MVT::f32, Custom); 109 110 setOperationAction(ISD::SETCC, MVT::v4i32, Expand); 111 setOperationAction(ISD::SETCC, MVT::v2i32, Expand); 112 113 setOperationAction(ISD::BR_CC, MVT::i32, Expand); 114 setOperationAction(ISD::BR_CC, MVT::f32, Expand); 115 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 116 117 setOperationAction(ISD::FSUB, MVT::f32, Expand); 118 119 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 120 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 121 122 setOperationAction(ISD::SETCC, MVT::i32, Expand); 123 setOperationAction(ISD::SETCC, MVT::f32, Expand); 124 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom); 125 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 126 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 127 128 setOperationAction(ISD::SELECT, MVT::i32, Expand); 129 setOperationAction(ISD::SELECT, MVT::f32, Expand); 130 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 131 setOperationAction(ISD::SELECT, MVT::v4i32, Expand); 132 133 // ADD, SUB overflow. 134 // TODO: turn these into Legal? 135 if (Subtarget->hasCARRY()) 136 setOperationAction(ISD::UADDO, MVT::i32, Custom); 137 138 if (Subtarget->hasBORROW()) 139 setOperationAction(ISD::USUBO, MVT::i32, Custom); 140 141 // Expand sign extension of vectors 142 if (!Subtarget->hasBFE()) 143 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 144 145 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand); 146 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand); 147 148 if (!Subtarget->hasBFE()) 149 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 150 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand); 151 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand); 152 153 if (!Subtarget->hasBFE()) 154 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 155 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand); 156 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand); 157 158 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 159 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand); 160 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand); 161 162 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand); 163 164 setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 165 166 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom); 167 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom); 168 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 169 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 170 171 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom); 172 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom); 173 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 174 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 175 176 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 177 // to be Legal/Custom in order to avoid library calls. 178 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 179 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 180 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 181 182 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 183 184 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 185 for (MVT VT : ScalarIntVTs) { 186 setOperationAction(ISD::ADDC, VT, Expand); 187 setOperationAction(ISD::SUBC, VT, Expand); 188 setOperationAction(ISD::ADDE, VT, Expand); 189 setOperationAction(ISD::SUBE, VT, Expand); 190 } 191 192 setSchedulingPreference(Sched::Source); 193 194 195 setTargetDAGCombine(ISD::FP_ROUND); 196 setTargetDAGCombine(ISD::FP_TO_SINT); 197 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 198 setTargetDAGCombine(ISD::SELECT_CC); 199 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 200 } 201 202 const R600Subtarget *R600TargetLowering::getSubtarget() const { 203 return static_cast<const R600Subtarget *>(Subtarget); 204 } 205 206 static inline bool isEOP(MachineBasicBlock::iterator I) { 207 return std::next(I)->getOpcode() == AMDGPU::RETURN; 208 } 209 210 MachineBasicBlock * R600TargetLowering::EmitInstrWithCustomInserter( 211 MachineInstr * MI, MachineBasicBlock * BB) const { 212 MachineFunction * MF = BB->getParent(); 213 MachineRegisterInfo &MRI = MF->getRegInfo(); 214 MachineBasicBlock::iterator I = *MI; 215 const R600InstrInfo *TII = getSubtarget()->getInstrInfo(); 216 217 switch (MI->getOpcode()) { 218 default: 219 // Replace LDS_*_RET instruction that don't have any uses with the 220 // equivalent LDS_*_NORET instruction. 221 if (TII->isLDSRetInstr(MI->getOpcode())) { 222 int DstIdx = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst); 223 assert(DstIdx != -1); 224 MachineInstrBuilder NewMI; 225 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add 226 // LDS_1A2D support and remove this special case. 227 if (!MRI.use_empty(MI->getOperand(DstIdx).getReg()) || 228 MI->getOpcode() == AMDGPU::LDS_CMPST_RET) 229 return BB; 230 231 NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), 232 TII->get(AMDGPU::getLDSNoRetOp(MI->getOpcode()))); 233 for (unsigned i = 1, e = MI->getNumOperands(); i < e; ++i) { 234 NewMI.addOperand(MI->getOperand(i)); 235 } 236 } else { 237 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 238 } 239 break; 240 case AMDGPU::CLAMP_R600: { 241 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 242 AMDGPU::MOV, 243 MI->getOperand(0).getReg(), 244 MI->getOperand(1).getReg()); 245 TII->addFlag(NewMI, 0, MO_FLAG_CLAMP); 246 break; 247 } 248 249 case AMDGPU::FABS_R600: { 250 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 251 AMDGPU::MOV, 252 MI->getOperand(0).getReg(), 253 MI->getOperand(1).getReg()); 254 TII->addFlag(NewMI, 0, MO_FLAG_ABS); 255 break; 256 } 257 258 case AMDGPU::FNEG_R600: { 259 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 260 AMDGPU::MOV, 261 MI->getOperand(0).getReg(), 262 MI->getOperand(1).getReg()); 263 TII->addFlag(NewMI, 0, MO_FLAG_NEG); 264 break; 265 } 266 267 case AMDGPU::MASK_WRITE: { 268 unsigned maskedRegister = MI->getOperand(0).getReg(); 269 assert(TargetRegisterInfo::isVirtualRegister(maskedRegister)); 270 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister); 271 TII->addFlag(defInstr, 0, MO_FLAG_MASK); 272 break; 273 } 274 275 case AMDGPU::MOV_IMM_F32: 276 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 277 MI->getOperand(1).getFPImm()->getValueAPF() 278 .bitcastToAPInt().getZExtValue()); 279 break; 280 case AMDGPU::MOV_IMM_I32: 281 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 282 MI->getOperand(1).getImm()); 283 break; 284 case AMDGPU::MOV_IMM_GLOBAL_ADDR: { 285 //TODO: Perhaps combine this instruction with the next if possible 286 auto MIB = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 287 MI->getOperand(0).getReg(), 288 AMDGPU::ALU_LITERAL_X); 289 int Idx = TII->getOperandIdx(*MIB, AMDGPU::OpName::literal); 290 //TODO: Ugh this is rather ugly 291 MIB->getOperand(Idx) = MI->getOperand(1); 292 break; 293 } 294 case AMDGPU::CONST_COPY: { 295 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 296 MI->getOperand(0).getReg(), AMDGPU::ALU_CONST); 297 TII->setImmOperand(NewMI, AMDGPU::OpName::src0_sel, 298 MI->getOperand(1).getImm()); 299 break; 300 } 301 302 case AMDGPU::RAT_WRITE_CACHELESS_32_eg: 303 case AMDGPU::RAT_WRITE_CACHELESS_64_eg: 304 case AMDGPU::RAT_WRITE_CACHELESS_128_eg: { 305 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 306 .addOperand(MI->getOperand(0)) 307 .addOperand(MI->getOperand(1)) 308 .addImm(isEOP(I)); // Set End of program bit 309 break; 310 } 311 case AMDGPU::RAT_STORE_TYPED_eg: { 312 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 313 .addOperand(MI->getOperand(0)) 314 .addOperand(MI->getOperand(1)) 315 .addOperand(MI->getOperand(2)) 316 .addImm(isEOP(I)); // Set End of program bit 317 break; 318 } 319 320 case AMDGPU::TXD: { 321 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 322 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 323 MachineOperand &RID = MI->getOperand(4); 324 MachineOperand &SID = MI->getOperand(5); 325 unsigned TextureId = MI->getOperand(6).getImm(); 326 unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 327 unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 328 329 switch (TextureId) { 330 case 5: // Rect 331 CTX = CTY = 0; 332 break; 333 case 6: // Shadow1D 334 SrcW = SrcZ; 335 break; 336 case 7: // Shadow2D 337 SrcW = SrcZ; 338 break; 339 case 8: // ShadowRect 340 CTX = CTY = 0; 341 SrcW = SrcZ; 342 break; 343 case 9: // 1DArray 344 SrcZ = SrcY; 345 CTZ = 0; 346 break; 347 case 10: // 2DArray 348 CTZ = 0; 349 break; 350 case 11: // Shadow1DArray 351 SrcZ = SrcY; 352 CTZ = 0; 353 break; 354 case 12: // Shadow2DArray 355 CTZ = 0; 356 break; 357 } 358 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 359 .addOperand(MI->getOperand(3)) 360 .addImm(SrcX) 361 .addImm(SrcY) 362 .addImm(SrcZ) 363 .addImm(SrcW) 364 .addImm(0) 365 .addImm(0) 366 .addImm(0) 367 .addImm(0) 368 .addImm(1) 369 .addImm(2) 370 .addImm(3) 371 .addOperand(RID) 372 .addOperand(SID) 373 .addImm(CTX) 374 .addImm(CTY) 375 .addImm(CTZ) 376 .addImm(CTW); 377 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 378 .addOperand(MI->getOperand(2)) 379 .addImm(SrcX) 380 .addImm(SrcY) 381 .addImm(SrcZ) 382 .addImm(SrcW) 383 .addImm(0) 384 .addImm(0) 385 .addImm(0) 386 .addImm(0) 387 .addImm(1) 388 .addImm(2) 389 .addImm(3) 390 .addOperand(RID) 391 .addOperand(SID) 392 .addImm(CTX) 393 .addImm(CTY) 394 .addImm(CTZ) 395 .addImm(CTW); 396 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_G)) 397 .addOperand(MI->getOperand(0)) 398 .addOperand(MI->getOperand(1)) 399 .addImm(SrcX) 400 .addImm(SrcY) 401 .addImm(SrcZ) 402 .addImm(SrcW) 403 .addImm(0) 404 .addImm(0) 405 .addImm(0) 406 .addImm(0) 407 .addImm(1) 408 .addImm(2) 409 .addImm(3) 410 .addOperand(RID) 411 .addOperand(SID) 412 .addImm(CTX) 413 .addImm(CTY) 414 .addImm(CTZ) 415 .addImm(CTW) 416 .addReg(T0, RegState::Implicit) 417 .addReg(T1, RegState::Implicit); 418 break; 419 } 420 421 case AMDGPU::TXD_SHADOW: { 422 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 423 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 424 MachineOperand &RID = MI->getOperand(4); 425 MachineOperand &SID = MI->getOperand(5); 426 unsigned TextureId = MI->getOperand(6).getImm(); 427 unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 428 unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 429 430 switch (TextureId) { 431 case 5: // Rect 432 CTX = CTY = 0; 433 break; 434 case 6: // Shadow1D 435 SrcW = SrcZ; 436 break; 437 case 7: // Shadow2D 438 SrcW = SrcZ; 439 break; 440 case 8: // ShadowRect 441 CTX = CTY = 0; 442 SrcW = SrcZ; 443 break; 444 case 9: // 1DArray 445 SrcZ = SrcY; 446 CTZ = 0; 447 break; 448 case 10: // 2DArray 449 CTZ = 0; 450 break; 451 case 11: // Shadow1DArray 452 SrcZ = SrcY; 453 CTZ = 0; 454 break; 455 case 12: // Shadow2DArray 456 CTZ = 0; 457 break; 458 } 459 460 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 461 .addOperand(MI->getOperand(3)) 462 .addImm(SrcX) 463 .addImm(SrcY) 464 .addImm(SrcZ) 465 .addImm(SrcW) 466 .addImm(0) 467 .addImm(0) 468 .addImm(0) 469 .addImm(0) 470 .addImm(1) 471 .addImm(2) 472 .addImm(3) 473 .addOperand(RID) 474 .addOperand(SID) 475 .addImm(CTX) 476 .addImm(CTY) 477 .addImm(CTZ) 478 .addImm(CTW); 479 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 480 .addOperand(MI->getOperand(2)) 481 .addImm(SrcX) 482 .addImm(SrcY) 483 .addImm(SrcZ) 484 .addImm(SrcW) 485 .addImm(0) 486 .addImm(0) 487 .addImm(0) 488 .addImm(0) 489 .addImm(1) 490 .addImm(2) 491 .addImm(3) 492 .addOperand(RID) 493 .addOperand(SID) 494 .addImm(CTX) 495 .addImm(CTY) 496 .addImm(CTZ) 497 .addImm(CTW); 498 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_C_G)) 499 .addOperand(MI->getOperand(0)) 500 .addOperand(MI->getOperand(1)) 501 .addImm(SrcX) 502 .addImm(SrcY) 503 .addImm(SrcZ) 504 .addImm(SrcW) 505 .addImm(0) 506 .addImm(0) 507 .addImm(0) 508 .addImm(0) 509 .addImm(1) 510 .addImm(2) 511 .addImm(3) 512 .addOperand(RID) 513 .addOperand(SID) 514 .addImm(CTX) 515 .addImm(CTY) 516 .addImm(CTZ) 517 .addImm(CTW) 518 .addReg(T0, RegState::Implicit) 519 .addReg(T1, RegState::Implicit); 520 break; 521 } 522 523 case AMDGPU::BRANCH: 524 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP)) 525 .addOperand(MI->getOperand(0)); 526 break; 527 528 case AMDGPU::BRANCH_COND_f32: { 529 MachineInstr *NewMI = 530 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 531 AMDGPU::PREDICATE_BIT) 532 .addOperand(MI->getOperand(1)) 533 .addImm(OPCODE_IS_NOT_ZERO) 534 .addImm(0); // Flags 535 TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 536 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 537 .addOperand(MI->getOperand(0)) 538 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 539 break; 540 } 541 542 case AMDGPU::BRANCH_COND_i32: { 543 MachineInstr *NewMI = 544 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 545 AMDGPU::PREDICATE_BIT) 546 .addOperand(MI->getOperand(1)) 547 .addImm(OPCODE_IS_NOT_ZERO_INT) 548 .addImm(0); // Flags 549 TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 550 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 551 .addOperand(MI->getOperand(0)) 552 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 553 break; 554 } 555 556 case AMDGPU::EG_ExportSwz: 557 case AMDGPU::R600_ExportSwz: { 558 // Instruction is left unmodified if its not the last one of its type 559 bool isLastInstructionOfItsType = true; 560 unsigned InstExportType = MI->getOperand(1).getImm(); 561 for (MachineBasicBlock::iterator NextExportInst = std::next(I), 562 EndBlock = BB->end(); NextExportInst != EndBlock; 563 NextExportInst = std::next(NextExportInst)) { 564 if (NextExportInst->getOpcode() == AMDGPU::EG_ExportSwz || 565 NextExportInst->getOpcode() == AMDGPU::R600_ExportSwz) { 566 unsigned CurrentInstExportType = NextExportInst->getOperand(1) 567 .getImm(); 568 if (CurrentInstExportType == InstExportType) { 569 isLastInstructionOfItsType = false; 570 break; 571 } 572 } 573 } 574 bool EOP = isEOP(I); 575 if (!EOP && !isLastInstructionOfItsType) 576 return BB; 577 unsigned CfInst = (MI->getOpcode() == AMDGPU::EG_ExportSwz)? 84 : 40; 578 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 579 .addOperand(MI->getOperand(0)) 580 .addOperand(MI->getOperand(1)) 581 .addOperand(MI->getOperand(2)) 582 .addOperand(MI->getOperand(3)) 583 .addOperand(MI->getOperand(4)) 584 .addOperand(MI->getOperand(5)) 585 .addOperand(MI->getOperand(6)) 586 .addImm(CfInst) 587 .addImm(EOP); 588 break; 589 } 590 case AMDGPU::RETURN: { 591 // RETURN instructions must have the live-out registers as implicit uses, 592 // otherwise they appear dead. 593 R600MachineFunctionInfo *MFI = MF->getInfo<R600MachineFunctionInfo>(); 594 MachineInstrBuilder MIB(*MF, MI); 595 for (unsigned i = 0, e = MFI->LiveOuts.size(); i != e; ++i) 596 MIB.addReg(MFI->LiveOuts[i], RegState::Implicit); 597 return BB; 598 } 599 } 600 601 MI->eraseFromParent(); 602 return BB; 603 } 604 605 //===----------------------------------------------------------------------===// 606 // Custom DAG Lowering Operations 607 //===----------------------------------------------------------------------===// 608 609 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 610 MachineFunction &MF = DAG.getMachineFunction(); 611 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 612 switch (Op.getOpcode()) { 613 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 614 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 615 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 616 case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG); 617 case ISD::SRA_PARTS: 618 case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG); 619 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY); 620 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW); 621 case ISD::FCOS: 622 case ISD::FSIN: return LowerTrig(Op, DAG); 623 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 624 case ISD::STORE: return LowerSTORE(Op, DAG); 625 case ISD::LOAD: { 626 SDValue Result = LowerLOAD(Op, DAG); 627 assert((!Result.getNode() || 628 Result.getNode()->getNumValues() == 2) && 629 "Load should return a value and a chain"); 630 return Result; 631 } 632 633 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 634 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG); 635 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG); 636 case ISD::INTRINSIC_VOID: { 637 SDValue Chain = Op.getOperand(0); 638 unsigned IntrinsicID = 639 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 640 switch (IntrinsicID) { 641 case AMDGPUIntrinsic::R600_store_swizzle: { 642 SDLoc DL(Op); 643 const SDValue Args[8] = { 644 Chain, 645 Op.getOperand(2), // Export Value 646 Op.getOperand(3), // ArrayBase 647 Op.getOperand(4), // Type 648 DAG.getConstant(0, DL, MVT::i32), // SWZ_X 649 DAG.getConstant(1, DL, MVT::i32), // SWZ_Y 650 DAG.getConstant(2, DL, MVT::i32), // SWZ_Z 651 DAG.getConstant(3, DL, MVT::i32) // SWZ_W 652 }; 653 return DAG.getNode(AMDGPUISD::EXPORT, DL, Op.getValueType(), Args); 654 } 655 656 // default for switch(IntrinsicID) 657 default: break; 658 } 659 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode()) 660 break; 661 } 662 case ISD::INTRINSIC_WO_CHAIN: { 663 unsigned IntrinsicID = 664 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 665 EVT VT = Op.getValueType(); 666 SDLoc DL(Op); 667 switch(IntrinsicID) { 668 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 669 case AMDGPUIntrinsic::R600_interp_xy: 670 case AMDGPUIntrinsic::R600_interp_zw: { 671 int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 672 MachineSDNode *interp; 673 SDValue RegisterINode = Op.getOperand(2); 674 SDValue RegisterJNode = Op.getOperand(3); 675 676 if (IntrinsicID == AMDGPUIntrinsic::R600_interp_xy) 677 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 678 MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 679 RegisterJNode, RegisterINode); 680 else 681 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 682 MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 683 RegisterJNode, RegisterINode); 684 return DAG.getBuildVector(MVT::v2f32, DL, 685 {SDValue(interp, 0), SDValue(interp, 1)}); 686 } 687 case AMDGPUIntrinsic::r600_tex: 688 case AMDGPUIntrinsic::r600_texc: 689 case AMDGPUIntrinsic::r600_txl: 690 case AMDGPUIntrinsic::r600_txlc: 691 case AMDGPUIntrinsic::r600_txb: 692 case AMDGPUIntrinsic::r600_txbc: 693 case AMDGPUIntrinsic::r600_txf: 694 case AMDGPUIntrinsic::r600_txq: 695 case AMDGPUIntrinsic::r600_ddx: 696 case AMDGPUIntrinsic::r600_ddy: 697 case AMDGPUIntrinsic::R600_ldptr: { 698 unsigned TextureOp; 699 switch (IntrinsicID) { 700 case AMDGPUIntrinsic::r600_tex: 701 TextureOp = 0; 702 break; 703 case AMDGPUIntrinsic::r600_texc: 704 TextureOp = 1; 705 break; 706 case AMDGPUIntrinsic::r600_txl: 707 TextureOp = 2; 708 break; 709 case AMDGPUIntrinsic::r600_txlc: 710 TextureOp = 3; 711 break; 712 case AMDGPUIntrinsic::r600_txb: 713 TextureOp = 4; 714 break; 715 case AMDGPUIntrinsic::r600_txbc: 716 TextureOp = 5; 717 break; 718 case AMDGPUIntrinsic::r600_txf: 719 TextureOp = 6; 720 break; 721 case AMDGPUIntrinsic::r600_txq: 722 TextureOp = 7; 723 break; 724 case AMDGPUIntrinsic::r600_ddx: 725 TextureOp = 8; 726 break; 727 case AMDGPUIntrinsic::r600_ddy: 728 TextureOp = 9; 729 break; 730 case AMDGPUIntrinsic::R600_ldptr: 731 TextureOp = 10; 732 break; 733 default: 734 llvm_unreachable("Unknow Texture Operation"); 735 } 736 737 SDValue TexArgs[19] = { 738 DAG.getConstant(TextureOp, DL, MVT::i32), 739 Op.getOperand(1), 740 DAG.getConstant(0, DL, MVT::i32), 741 DAG.getConstant(1, DL, MVT::i32), 742 DAG.getConstant(2, DL, MVT::i32), 743 DAG.getConstant(3, DL, MVT::i32), 744 Op.getOperand(2), 745 Op.getOperand(3), 746 Op.getOperand(4), 747 DAG.getConstant(0, DL, MVT::i32), 748 DAG.getConstant(1, DL, MVT::i32), 749 DAG.getConstant(2, DL, MVT::i32), 750 DAG.getConstant(3, DL, MVT::i32), 751 Op.getOperand(5), 752 Op.getOperand(6), 753 Op.getOperand(7), 754 Op.getOperand(8), 755 Op.getOperand(9), 756 Op.getOperand(10) 757 }; 758 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs); 759 } 760 case AMDGPUIntrinsic::AMDGPU_dp4: { 761 SDValue Args[8] = { 762 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 763 DAG.getConstant(0, DL, MVT::i32)), 764 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 765 DAG.getConstant(0, DL, MVT::i32)), 766 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 767 DAG.getConstant(1, DL, MVT::i32)), 768 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 769 DAG.getConstant(1, DL, MVT::i32)), 770 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 771 DAG.getConstant(2, DL, MVT::i32)), 772 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 773 DAG.getConstant(2, DL, MVT::i32)), 774 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 775 DAG.getConstant(3, DL, MVT::i32)), 776 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 777 DAG.getConstant(3, DL, MVT::i32)) 778 }; 779 return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args); 780 } 781 782 case Intrinsic::r600_read_ngroups_x: 783 return LowerImplicitParameter(DAG, VT, DL, 0); 784 case Intrinsic::r600_read_ngroups_y: 785 return LowerImplicitParameter(DAG, VT, DL, 1); 786 case Intrinsic::r600_read_ngroups_z: 787 return LowerImplicitParameter(DAG, VT, DL, 2); 788 case Intrinsic::r600_read_global_size_x: 789 return LowerImplicitParameter(DAG, VT, DL, 3); 790 case Intrinsic::r600_read_global_size_y: 791 return LowerImplicitParameter(DAG, VT, DL, 4); 792 case Intrinsic::r600_read_global_size_z: 793 return LowerImplicitParameter(DAG, VT, DL, 5); 794 case Intrinsic::r600_read_local_size_x: 795 return LowerImplicitParameter(DAG, VT, DL, 6); 796 case Intrinsic::r600_read_local_size_y: 797 return LowerImplicitParameter(DAG, VT, DL, 7); 798 case Intrinsic::r600_read_local_size_z: 799 return LowerImplicitParameter(DAG, VT, DL, 8); 800 801 case Intrinsic::r600_read_workdim: 802 case AMDGPUIntrinsic::AMDGPU_read_workdim: { // Legacy name. 803 uint32_t ByteOffset = getImplicitParameterOffset(MFI, GRID_DIM); 804 return LowerImplicitParameter(DAG, VT, DL, ByteOffset / 4); 805 } 806 807 case Intrinsic::r600_read_tgid_x: 808 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 809 AMDGPU::T1_X, VT); 810 case Intrinsic::r600_read_tgid_y: 811 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 812 AMDGPU::T1_Y, VT); 813 case Intrinsic::r600_read_tgid_z: 814 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 815 AMDGPU::T1_Z, VT); 816 case Intrinsic::r600_read_tidig_x: 817 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 818 AMDGPU::T0_X, VT); 819 case Intrinsic::r600_read_tidig_y: 820 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 821 AMDGPU::T0_Y, VT); 822 case Intrinsic::r600_read_tidig_z: 823 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 824 AMDGPU::T0_Z, VT); 825 826 // FIXME: Should be renamed to r600 prefix 827 case AMDGPUIntrinsic::AMDGPU_rsq_clamped: 828 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1)); 829 830 case Intrinsic::r600_rsq: 831 case AMDGPUIntrinsic::AMDGPU_rsq: // Legacy name 832 // XXX - I'm assuming SI's RSQ_LEGACY matches R600's behavior. 833 return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 834 } 835 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode()) 836 break; 837 } 838 } // end switch(Op.getOpcode()) 839 return SDValue(); 840 } 841 842 void R600TargetLowering::ReplaceNodeResults(SDNode *N, 843 SmallVectorImpl<SDValue> &Results, 844 SelectionDAG &DAG) const { 845 switch (N->getOpcode()) { 846 default: 847 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG); 848 return; 849 case ISD::FP_TO_UINT: 850 if (N->getValueType(0) == MVT::i1) { 851 Results.push_back(LowerFPTOUINT(N->getOperand(0), DAG)); 852 return; 853 } 854 // Fall-through. Since we don't care about out of bounds values 855 // we can use FP_TO_SINT for uints too. The DAGLegalizer code for uint 856 // considers some extra cases which are not necessary here. 857 case ISD::FP_TO_SINT: { 858 SDValue Result; 859 if (expandFP_TO_SINT(N, Result, DAG)) 860 Results.push_back(Result); 861 return; 862 } 863 case ISD::SDIVREM: { 864 SDValue Op = SDValue(N, 1); 865 SDValue RES = LowerSDIVREM(Op, DAG); 866 Results.push_back(RES); 867 Results.push_back(RES.getValue(1)); 868 break; 869 } 870 case ISD::UDIVREM: { 871 SDValue Op = SDValue(N, 0); 872 LowerUDIVREM64(Op, DAG, Results); 873 break; 874 } 875 } 876 } 877 878 SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG, 879 SDValue Vector) const { 880 881 SDLoc DL(Vector); 882 EVT VecVT = Vector.getValueType(); 883 EVT EltVT = VecVT.getVectorElementType(); 884 SmallVector<SDValue, 8> Args; 885 886 for (unsigned i = 0, e = VecVT.getVectorNumElements(); 887 i != e; ++i) { 888 Args.push_back(DAG.getNode( 889 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector, 890 DAG.getConstant(i, DL, getVectorIdxTy(DAG.getDataLayout())))); 891 } 892 893 return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args); 894 } 895 896 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 897 SelectionDAG &DAG) const { 898 899 SDLoc DL(Op); 900 SDValue Vector = Op.getOperand(0); 901 SDValue Index = Op.getOperand(1); 902 903 if (isa<ConstantSDNode>(Index) || 904 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 905 return Op; 906 907 Vector = vectorToVerticalVector(DAG, Vector); 908 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), 909 Vector, Index); 910 } 911 912 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 913 SelectionDAG &DAG) const { 914 SDLoc DL(Op); 915 SDValue Vector = Op.getOperand(0); 916 SDValue Value = Op.getOperand(1); 917 SDValue Index = Op.getOperand(2); 918 919 if (isa<ConstantSDNode>(Index) || 920 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 921 return Op; 922 923 Vector = vectorToVerticalVector(DAG, Vector); 924 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(), 925 Vector, Value, Index); 926 return vectorToVerticalVector(DAG, Insert); 927 } 928 929 SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, 930 SDValue Op, 931 SelectionDAG &DAG) const { 932 933 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op); 934 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS) 935 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG); 936 937 const DataLayout &DL = DAG.getDataLayout(); 938 const GlobalValue *GV = GSD->getGlobal(); 939 MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); 940 941 SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT); 942 return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA); 943 } 944 945 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 946 // On hw >= R700, COS/SIN input must be between -1. and 1. 947 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) 948 EVT VT = Op.getValueType(); 949 SDValue Arg = Op.getOperand(0); 950 SDLoc DL(Op); 951 952 // TODO: Should this propagate fast-math-flags? 953 SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT, 954 DAG.getNode(ISD::FADD, DL, VT, 955 DAG.getNode(ISD::FMUL, DL, VT, Arg, 956 DAG.getConstantFP(0.15915494309, DL, MVT::f32)), 957 DAG.getConstantFP(0.5, DL, MVT::f32))); 958 unsigned TrigNode; 959 switch (Op.getOpcode()) { 960 case ISD::FCOS: 961 TrigNode = AMDGPUISD::COS_HW; 962 break; 963 case ISD::FSIN: 964 TrigNode = AMDGPUISD::SIN_HW; 965 break; 966 default: 967 llvm_unreachable("Wrong trig opcode"); 968 } 969 SDValue TrigVal = DAG.getNode(TrigNode, DL, VT, 970 DAG.getNode(ISD::FADD, DL, VT, FractPart, 971 DAG.getConstantFP(-0.5, DL, MVT::f32))); 972 if (Gen >= R600Subtarget::R700) 973 return TrigVal; 974 // On R600 hw, COS/SIN input must be between -Pi and Pi. 975 return DAG.getNode(ISD::FMUL, DL, VT, TrigVal, 976 DAG.getConstantFP(3.14159265359, DL, MVT::f32)); 977 } 978 979 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const { 980 SDLoc DL(Op); 981 EVT VT = Op.getValueType(); 982 983 SDValue Lo = Op.getOperand(0); 984 SDValue Hi = Op.getOperand(1); 985 SDValue Shift = Op.getOperand(2); 986 SDValue Zero = DAG.getConstant(0, DL, VT); 987 SDValue One = DAG.getConstant(1, DL, VT); 988 989 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 990 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 991 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 992 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 993 994 // The dance around Width1 is necessary for 0 special case. 995 // Without it the CompShift might be 32, producing incorrect results in 996 // Overflow. So we do the shift in two steps, the alternative is to 997 // add a conditional to filter the special case. 998 999 SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift); 1000 Overflow = DAG.getNode(ISD::SRL, DL, VT, Overflow, One); 1001 1002 SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift); 1003 HiSmall = DAG.getNode(ISD::OR, DL, VT, HiSmall, Overflow); 1004 SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift); 1005 1006 SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift); 1007 SDValue LoBig = Zero; 1008 1009 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1010 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1011 1012 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1013 } 1014 1015 SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const { 1016 SDLoc DL(Op); 1017 EVT VT = Op.getValueType(); 1018 1019 SDValue Lo = Op.getOperand(0); 1020 SDValue Hi = Op.getOperand(1); 1021 SDValue Shift = Op.getOperand(2); 1022 SDValue Zero = DAG.getConstant(0, DL, VT); 1023 SDValue One = DAG.getConstant(1, DL, VT); 1024 1025 const bool SRA = Op.getOpcode() == ISD::SRA_PARTS; 1026 1027 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 1028 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 1029 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1030 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1031 1032 // The dance around Width1 is necessary for 0 special case. 1033 // Without it the CompShift might be 32, producing incorrect results in 1034 // Overflow. So we do the shift in two steps, the alternative is to 1035 // add a conditional to filter the special case. 1036 1037 SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift); 1038 Overflow = DAG.getNode(ISD::SHL, DL, VT, Overflow, One); 1039 1040 SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift); 1041 SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift); 1042 LoSmall = DAG.getNode(ISD::OR, DL, VT, LoSmall, Overflow); 1043 1044 SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift); 1045 SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero; 1046 1047 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1048 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1049 1050 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1051 } 1052 1053 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG, 1054 unsigned mainop, unsigned ovf) const { 1055 SDLoc DL(Op); 1056 EVT VT = Op.getValueType(); 1057 1058 SDValue Lo = Op.getOperand(0); 1059 SDValue Hi = Op.getOperand(1); 1060 1061 SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi); 1062 // Extend sign. 1063 OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF, 1064 DAG.getValueType(MVT::i1)); 1065 1066 SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi); 1067 1068 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF); 1069 } 1070 1071 SDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const { 1072 SDLoc DL(Op); 1073 return DAG.getNode( 1074 ISD::SETCC, 1075 DL, 1076 MVT::i1, 1077 Op, DAG.getConstantFP(0.0f, DL, MVT::f32), 1078 DAG.getCondCode(ISD::SETNE) 1079 ); 1080 } 1081 1082 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT, 1083 const SDLoc &DL, 1084 unsigned DwordOffset) const { 1085 unsigned ByteOffset = DwordOffset * 4; 1086 PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1087 AMDGPUAS::CONSTANT_BUFFER_0); 1088 1089 // We shouldn't be using an offset wider than 16-bits for implicit parameters. 1090 assert(isInt<16>(ByteOffset)); 1091 1092 return DAG.getLoad(VT, DL, DAG.getEntryNode(), 1093 DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR 1094 MachinePointerInfo(ConstantPointerNull::get(PtrType)), 1095 false, false, false, 0); 1096 } 1097 1098 bool R600TargetLowering::isZero(SDValue Op) const { 1099 if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) { 1100 return Cst->isNullValue(); 1101 } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){ 1102 return CstFP->isZero(); 1103 } else { 1104 return false; 1105 } 1106 } 1107 1108 bool R600TargetLowering::isHWTrueValue(SDValue Op) const { 1109 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 1110 return CFP->isExactlyValue(1.0); 1111 } 1112 return isAllOnesConstant(Op); 1113 } 1114 1115 bool R600TargetLowering::isHWFalseValue(SDValue Op) const { 1116 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 1117 return CFP->getValueAPF().isZero(); 1118 } 1119 return isNullConstant(Op); 1120 } 1121 1122 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 1123 SDLoc DL(Op); 1124 EVT VT = Op.getValueType(); 1125 1126 SDValue LHS = Op.getOperand(0); 1127 SDValue RHS = Op.getOperand(1); 1128 SDValue True = Op.getOperand(2); 1129 SDValue False = Op.getOperand(3); 1130 SDValue CC = Op.getOperand(4); 1131 SDValue Temp; 1132 1133 if (VT == MVT::f32) { 1134 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); 1135 SDValue MinMax = CombineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI); 1136 if (MinMax) 1137 return MinMax; 1138 } 1139 1140 // LHS and RHS are guaranteed to be the same value type 1141 EVT CompareVT = LHS.getValueType(); 1142 1143 // Check if we can lower this to a native operation. 1144 1145 // Try to lower to a SET* instruction: 1146 // 1147 // SET* can match the following patterns: 1148 // 1149 // select_cc f32, f32, -1, 0, cc_supported 1150 // select_cc f32, f32, 1.0f, 0.0f, cc_supported 1151 // select_cc i32, i32, -1, 0, cc_supported 1152 // 1153 1154 // Move hardware True/False values to the correct operand. 1155 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1156 ISD::CondCode InverseCC = 1157 ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1158 if (isHWTrueValue(False) && isHWFalseValue(True)) { 1159 if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { 1160 std::swap(False, True); 1161 CC = DAG.getCondCode(InverseCC); 1162 } else { 1163 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC); 1164 if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) { 1165 std::swap(False, True); 1166 std::swap(LHS, RHS); 1167 CC = DAG.getCondCode(SwapInvCC); 1168 } 1169 } 1170 } 1171 1172 if (isHWTrueValue(True) && isHWFalseValue(False) && 1173 (CompareVT == VT || VT == MVT::i32)) { 1174 // This can be matched by a SET* instruction. 1175 return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC); 1176 } 1177 1178 // Try to lower to a CND* instruction: 1179 // 1180 // CND* can match the following patterns: 1181 // 1182 // select_cc f32, 0.0, f32, f32, cc_supported 1183 // select_cc f32, 0.0, i32, i32, cc_supported 1184 // select_cc i32, 0, f32, f32, cc_supported 1185 // select_cc i32, 0, i32, i32, cc_supported 1186 // 1187 1188 // Try to move the zero value to the RHS 1189 if (isZero(LHS)) { 1190 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1191 // Try swapping the operands 1192 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode); 1193 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1194 std::swap(LHS, RHS); 1195 CC = DAG.getCondCode(CCSwapped); 1196 } else { 1197 // Try inverting the conditon and then swapping the operands 1198 ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT.isInteger()); 1199 CCSwapped = ISD::getSetCCSwappedOperands(CCInv); 1200 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1201 std::swap(True, False); 1202 std::swap(LHS, RHS); 1203 CC = DAG.getCondCode(CCSwapped); 1204 } 1205 } 1206 } 1207 if (isZero(RHS)) { 1208 SDValue Cond = LHS; 1209 SDValue Zero = RHS; 1210 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1211 if (CompareVT != VT) { 1212 // Bitcast True / False to the correct types. This will end up being 1213 // a nop, but it allows us to define only a single pattern in the 1214 // .TD files for each CND* instruction rather than having to have 1215 // one pattern for integer True/False and one for fp True/False 1216 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True); 1217 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False); 1218 } 1219 1220 switch (CCOpcode) { 1221 case ISD::SETONE: 1222 case ISD::SETUNE: 1223 case ISD::SETNE: 1224 CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1225 Temp = True; 1226 True = False; 1227 False = Temp; 1228 break; 1229 default: 1230 break; 1231 } 1232 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, 1233 Cond, Zero, 1234 True, False, 1235 DAG.getCondCode(CCOpcode)); 1236 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode); 1237 } 1238 1239 // If we make it this for it means we have no native instructions to handle 1240 // this SELECT_CC, so we must lower it. 1241 SDValue HWTrue, HWFalse; 1242 1243 if (CompareVT == MVT::f32) { 1244 HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT); 1245 HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT); 1246 } else if (CompareVT == MVT::i32) { 1247 HWTrue = DAG.getConstant(-1, DL, CompareVT); 1248 HWFalse = DAG.getConstant(0, DL, CompareVT); 1249 } 1250 else { 1251 llvm_unreachable("Unhandled value type in LowerSELECT_CC"); 1252 } 1253 1254 // Lower this unsupported SELECT_CC into a combination of two supported 1255 // SELECT_CC operations. 1256 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC); 1257 1258 return DAG.getNode(ISD::SELECT_CC, DL, VT, 1259 Cond, HWFalse, 1260 True, False, 1261 DAG.getCondCode(ISD::SETNE)); 1262 } 1263 1264 /// LLVM generates byte-addressed pointers. For indirect addressing, we need to 1265 /// convert these pointers to a register index. Each register holds 1266 /// 16 bytes, (4 x 32bit sub-register), but we need to take into account the 1267 /// \p StackWidth, which tells us how many of the 4 sub-registrers will be used 1268 /// for indirect addressing. 1269 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr, 1270 unsigned StackWidth, 1271 SelectionDAG &DAG) const { 1272 unsigned SRLPad; 1273 switch(StackWidth) { 1274 case 1: 1275 SRLPad = 2; 1276 break; 1277 case 2: 1278 SRLPad = 3; 1279 break; 1280 case 4: 1281 SRLPad = 4; 1282 break; 1283 default: llvm_unreachable("Invalid stack width"); 1284 } 1285 1286 SDLoc DL(Ptr); 1287 return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr, 1288 DAG.getConstant(SRLPad, DL, MVT::i32)); 1289 } 1290 1291 void R600TargetLowering::getStackAddress(unsigned StackWidth, 1292 unsigned ElemIdx, 1293 unsigned &Channel, 1294 unsigned &PtrIncr) const { 1295 switch (StackWidth) { 1296 default: 1297 case 1: 1298 Channel = 0; 1299 if (ElemIdx > 0) { 1300 PtrIncr = 1; 1301 } else { 1302 PtrIncr = 0; 1303 } 1304 break; 1305 case 2: 1306 Channel = ElemIdx % 2; 1307 if (ElemIdx == 2) { 1308 PtrIncr = 1; 1309 } else { 1310 PtrIncr = 0; 1311 } 1312 break; 1313 case 4: 1314 Channel = ElemIdx; 1315 PtrIncr = 0; 1316 break; 1317 } 1318 } 1319 1320 SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store, 1321 SelectionDAG &DAG) const { 1322 SDLoc DL(Store); 1323 1324 unsigned Mask = 0; 1325 if (Store->getMemoryVT() == MVT::i8) { 1326 Mask = 0xff; 1327 } else if (Store->getMemoryVT() == MVT::i16) { 1328 Mask = 0xffff; 1329 } 1330 1331 SDValue Chain = Store->getChain(); 1332 SDValue BasePtr = Store->getBasePtr(); 1333 EVT MemVT = Store->getMemoryVT(); 1334 1335 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr, 1336 DAG.getConstant(2, DL, MVT::i32)); 1337 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32, 1338 Chain, Ptr, 1339 DAG.getTargetConstant(0, DL, MVT::i32)); 1340 1341 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr, 1342 DAG.getConstant(0x3, DL, MVT::i32)); 1343 1344 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1345 DAG.getConstant(3, DL, MVT::i32)); 1346 1347 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, 1348 Store->getValue()); 1349 1350 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT); 1351 1352 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, 1353 MaskedValue, ShiftAmt); 1354 1355 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, 1356 DAG.getConstant(Mask, DL, MVT::i32), 1357 ShiftAmt); 1358 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask, 1359 DAG.getConstant(0xffffffff, DL, MVT::i32)); 1360 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); 1361 1362 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); 1363 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1364 Chain, Value, Ptr, 1365 DAG.getTargetConstant(0, DL, MVT::i32)); 1366 } 1367 1368 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1369 if (SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG)) 1370 return Result; 1371 1372 StoreSDNode *StoreNode = cast<StoreSDNode>(Op); 1373 unsigned AS = StoreNode->getAddressSpace(); 1374 SDValue Value = StoreNode->getValue(); 1375 EVT ValueVT = Value.getValueType(); 1376 1377 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) && 1378 ValueVT.isVector()) { 1379 return SplitVectorStore(Op, DAG); 1380 } 1381 1382 SDLoc DL(Op); 1383 SDValue Chain = StoreNode->getChain(); 1384 SDValue Ptr = StoreNode->getBasePtr(); 1385 1386 if (AS == AMDGPUAS::GLOBAL_ADDRESS) { 1387 if (StoreNode->isTruncatingStore()) { 1388 EVT VT = Value.getValueType(); 1389 assert(VT.bitsLE(MVT::i32)); 1390 EVT MemVT = StoreNode->getMemoryVT(); 1391 SDValue MaskConstant; 1392 if (MemVT == MVT::i8) { 1393 MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32); 1394 } else { 1395 assert(MemVT == MVT::i16); 1396 MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32); 1397 } 1398 SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr, 1399 DAG.getConstant(2, DL, MVT::i32)); 1400 SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr, 1401 DAG.getConstant(0x00000003, DL, VT)); 1402 SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant); 1403 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex, 1404 DAG.getConstant(3, DL, VT)); 1405 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, Shift); 1406 SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, Shift); 1407 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 1408 // vector instead. 1409 SDValue Src[4] = { 1410 ShiftedValue, 1411 DAG.getConstant(0, DL, MVT::i32), 1412 DAG.getConstant(0, DL, MVT::i32), 1413 Mask 1414 }; 1415 SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src); 1416 SDValue Args[3] = { Chain, Input, DWordAddr }; 1417 return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL, 1418 Op->getVTList(), Args, MemVT, 1419 StoreNode->getMemOperand()); 1420 } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && 1421 ValueVT.bitsGE(MVT::i32)) { 1422 // Convert pointer from byte address to dword address. 1423 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(), 1424 DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), 1425 Ptr, DAG.getConstant(2, DL, MVT::i32))); 1426 1427 if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) { 1428 llvm_unreachable("Truncated and indexed stores not supported yet"); 1429 } else { 1430 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1431 } 1432 return Chain; 1433 } 1434 } 1435 1436 if (AS != AMDGPUAS::PRIVATE_ADDRESS) 1437 return SDValue(); 1438 1439 EVT MemVT = StoreNode->getMemoryVT(); 1440 if (MemVT.bitsLT(MVT::i32)) 1441 return lowerPrivateTruncStore(StoreNode, DAG); 1442 1443 // Lowering for indirect addressing 1444 const MachineFunction &MF = DAG.getMachineFunction(); 1445 const R600FrameLowering *TFL = getSubtarget()->getFrameLowering(); 1446 unsigned StackWidth = TFL->getStackWidth(MF); 1447 1448 Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1449 1450 if (ValueVT.isVector()) { 1451 unsigned NumElemVT = ValueVT.getVectorNumElements(); 1452 EVT ElemVT = ValueVT.getVectorElementType(); 1453 SmallVector<SDValue, 4> Stores(NumElemVT); 1454 1455 assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1456 "vector width in load"); 1457 1458 for (unsigned i = 0; i < NumElemVT; ++i) { 1459 unsigned Channel, PtrIncr; 1460 getStackAddress(StackWidth, i, Channel, PtrIncr); 1461 Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1462 DAG.getConstant(PtrIncr, DL, MVT::i32)); 1463 SDValue Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, 1464 Value, DAG.getConstant(i, DL, MVT::i32)); 1465 1466 Stores[i] = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1467 Chain, Elem, Ptr, 1468 DAG.getTargetConstant(Channel, DL, MVT::i32)); 1469 } 1470 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); 1471 } else { 1472 if (ValueVT == MVT::i8) { 1473 Value = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Value); 1474 } 1475 Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, Chain, Value, Ptr, 1476 DAG.getTargetConstant(0, DL, MVT::i32)); // Channel 1477 } 1478 1479 return Chain; 1480 } 1481 1482 // return (512 + (kc_bank << 12) 1483 static int 1484 ConstantAddressBlock(unsigned AddressSpace) { 1485 switch (AddressSpace) { 1486 case AMDGPUAS::CONSTANT_BUFFER_0: 1487 return 512; 1488 case AMDGPUAS::CONSTANT_BUFFER_1: 1489 return 512 + 4096; 1490 case AMDGPUAS::CONSTANT_BUFFER_2: 1491 return 512 + 4096 * 2; 1492 case AMDGPUAS::CONSTANT_BUFFER_3: 1493 return 512 + 4096 * 3; 1494 case AMDGPUAS::CONSTANT_BUFFER_4: 1495 return 512 + 4096 * 4; 1496 case AMDGPUAS::CONSTANT_BUFFER_5: 1497 return 512 + 4096 * 5; 1498 case AMDGPUAS::CONSTANT_BUFFER_6: 1499 return 512 + 4096 * 6; 1500 case AMDGPUAS::CONSTANT_BUFFER_7: 1501 return 512 + 4096 * 7; 1502 case AMDGPUAS::CONSTANT_BUFFER_8: 1503 return 512 + 4096 * 8; 1504 case AMDGPUAS::CONSTANT_BUFFER_9: 1505 return 512 + 4096 * 9; 1506 case AMDGPUAS::CONSTANT_BUFFER_10: 1507 return 512 + 4096 * 10; 1508 case AMDGPUAS::CONSTANT_BUFFER_11: 1509 return 512 + 4096 * 11; 1510 case AMDGPUAS::CONSTANT_BUFFER_12: 1511 return 512 + 4096 * 12; 1512 case AMDGPUAS::CONSTANT_BUFFER_13: 1513 return 512 + 4096 * 13; 1514 case AMDGPUAS::CONSTANT_BUFFER_14: 1515 return 512 + 4096 * 14; 1516 case AMDGPUAS::CONSTANT_BUFFER_15: 1517 return 512 + 4096 * 15; 1518 default: 1519 return -1; 1520 } 1521 } 1522 1523 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op, 1524 SelectionDAG &DAG) const { 1525 SDLoc DL(Op); 1526 LoadSDNode *Load = cast<LoadSDNode>(Op); 1527 ISD::LoadExtType ExtType = Load->getExtensionType(); 1528 EVT MemVT = Load->getMemoryVT(); 1529 1530 // <SI && AS=PRIVATE && EXTLOAD && size < 32bit, 1531 // register (2-)byte extract. 1532 1533 // Get Register holding the target. 1534 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(), 1535 DAG.getConstant(2, DL, MVT::i32)); 1536 // Load the Register. 1537 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(), 1538 Load->getChain(), 1539 Ptr, 1540 DAG.getTargetConstant(0, DL, MVT::i32), 1541 Op.getOperand(2)); 1542 1543 // Get offset within the register. 1544 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, 1545 Load->getBasePtr(), 1546 DAG.getConstant(0x3, DL, MVT::i32)); 1547 1548 // Bit offset of target byte (byteIdx * 8). 1549 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1550 DAG.getConstant(3, DL, MVT::i32)); 1551 1552 // Shift to the right. 1553 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt); 1554 1555 // Eliminate the upper bits by setting them to ... 1556 EVT MemEltVT = MemVT.getScalarType(); 1557 1558 // ... ones. 1559 if (ExtType == ISD::SEXTLOAD) { 1560 SDValue MemEltVTNode = DAG.getValueType(MemEltVT); 1561 1562 SDValue Ops[] = { 1563 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode), 1564 Load->getChain() 1565 }; 1566 1567 return DAG.getMergeValues(Ops, DL); 1568 } 1569 1570 // ... or zeros. 1571 SDValue Ops[] = { 1572 DAG.getZeroExtendInReg(Ret, DL, MemEltVT), 1573 Load->getChain() 1574 }; 1575 1576 return DAG.getMergeValues(Ops, DL); 1577 } 1578 1579 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 1580 LoadSDNode *LoadNode = cast<LoadSDNode>(Op); 1581 unsigned AS = LoadNode->getAddressSpace(); 1582 EVT MemVT = LoadNode->getMemoryVT(); 1583 ISD::LoadExtType ExtType = LoadNode->getExtensionType(); 1584 1585 if (AS == AMDGPUAS::PRIVATE_ADDRESS && 1586 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(MVT::i32)) { 1587 return lowerPrivateExtLoad(Op, DAG); 1588 } 1589 1590 SDLoc DL(Op); 1591 EVT VT = Op.getValueType(); 1592 SDValue Chain = LoadNode->getChain(); 1593 SDValue Ptr = LoadNode->getBasePtr(); 1594 1595 if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) { 1596 SDValue MergedValues[2] = { 1597 scalarizeVectorLoad(LoadNode, DAG), 1598 Chain 1599 }; 1600 return DAG.getMergeValues(MergedValues, DL); 1601 } 1602 1603 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); 1604 if (ConstantBlock > -1 && 1605 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || 1606 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { 1607 SDValue Result; 1608 if (isa<ConstantExpr>(LoadNode->getMemOperand()->getValue()) || 1609 isa<Constant>(LoadNode->getMemOperand()->getValue()) || 1610 isa<ConstantSDNode>(Ptr)) { 1611 SDValue Slots[4]; 1612 for (unsigned i = 0; i < 4; i++) { 1613 // We want Const position encoded with the following formula : 1614 // (((512 + (kc_bank << 12) + const_index) << 2) + chan) 1615 // const_index is Ptr computed by llvm using an alignment of 16. 1616 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and 1617 // then div by 4 at the ISel step 1618 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1619 DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32)); 1620 Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr); 1621 } 1622 EVT NewVT = MVT::v4i32; 1623 unsigned NumElements = 4; 1624 if (VT.isVector()) { 1625 NewVT = VT; 1626 NumElements = VT.getVectorNumElements(); 1627 } 1628 Result = DAG.getBuildVector(NewVT, DL, makeArrayRef(Slots, NumElements)); 1629 } else { 1630 // non-constant ptr can't be folded, keeps it as a v4f32 load 1631 Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32, 1632 DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 1633 DAG.getConstant(4, DL, MVT::i32)), 1634 DAG.getConstant(LoadNode->getAddressSpace() - 1635 AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32) 1636 ); 1637 } 1638 1639 if (!VT.isVector()) { 1640 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 1641 DAG.getConstant(0, DL, MVT::i32)); 1642 } 1643 1644 SDValue MergedValues[2] = { 1645 Result, 1646 Chain 1647 }; 1648 return DAG.getMergeValues(MergedValues, DL); 1649 } 1650 1651 SDValue LoweredLoad; 1652 1653 // For most operations returning SDValue() will result in the node being 1654 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we 1655 // need to manually expand loads that may be legal in some address spaces and 1656 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for 1657 // compute shaders, since the data is sign extended when it is uploaded to the 1658 // buffer. However SEXT loads from other address spaces are not supported, so 1659 // we need to expand them here. 1660 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { 1661 EVT MemVT = LoadNode->getMemoryVT(); 1662 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); 1663 SDValue NewLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, Chain, Ptr, 1664 LoadNode->getPointerInfo(), MemVT, 1665 LoadNode->isVolatile(), 1666 LoadNode->isNonTemporal(), 1667 LoadNode->isInvariant(), 1668 LoadNode->getAlignment()); 1669 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad, 1670 DAG.getValueType(MemVT)); 1671 1672 SDValue MergedValues[2] = { Res, Chain }; 1673 return DAG.getMergeValues(MergedValues, DL); 1674 } 1675 1676 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1677 return SDValue(); 1678 } 1679 1680 // Lowering for indirect addressing 1681 const MachineFunction &MF = DAG.getMachineFunction(); 1682 const R600FrameLowering *TFL = getSubtarget()->getFrameLowering(); 1683 unsigned StackWidth = TFL->getStackWidth(MF); 1684 1685 Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1686 1687 if (VT.isVector()) { 1688 unsigned NumElemVT = VT.getVectorNumElements(); 1689 EVT ElemVT = VT.getVectorElementType(); 1690 SDValue Loads[4]; 1691 1692 assert(NumElemVT <= 4); 1693 assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1694 "vector width in load"); 1695 1696 for (unsigned i = 0; i < NumElemVT; ++i) { 1697 unsigned Channel, PtrIncr; 1698 getStackAddress(StackWidth, i, Channel, PtrIncr); 1699 Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1700 DAG.getConstant(PtrIncr, DL, MVT::i32)); 1701 Loads[i] = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, ElemVT, 1702 Chain, Ptr, 1703 DAG.getTargetConstant(Channel, DL, MVT::i32), 1704 Op.getOperand(2)); 1705 } 1706 EVT TargetVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElemVT); 1707 LoweredLoad = DAG.getBuildVector(TargetVT, DL, makeArrayRef(Loads, NumElemVT)); 1708 } else { 1709 LoweredLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, VT, 1710 Chain, Ptr, 1711 DAG.getTargetConstant(0, DL, MVT::i32), // Channel 1712 Op.getOperand(2)); 1713 } 1714 1715 SDValue Ops[2] = { 1716 LoweredLoad, 1717 Chain 1718 }; 1719 1720 return DAG.getMergeValues(Ops, DL); 1721 } 1722 1723 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 1724 SDValue Chain = Op.getOperand(0); 1725 SDValue Cond = Op.getOperand(1); 1726 SDValue Jump = Op.getOperand(2); 1727 1728 return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(), 1729 Chain, Jump, Cond); 1730 } 1731 1732 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op, 1733 SelectionDAG &DAG) const { 1734 MachineFunction &MF = DAG.getMachineFunction(); 1735 const R600FrameLowering *TFL = getSubtarget()->getFrameLowering(); 1736 1737 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op); 1738 1739 unsigned FrameIndex = FIN->getIndex(); 1740 unsigned IgnoredFrameReg; 1741 unsigned Offset = 1742 TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg); 1743 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), SDLoc(Op), 1744 Op.getValueType()); 1745 } 1746 1747 /// XXX Only kernel functions are supported, so we can assume for now that 1748 /// every function is a kernel function, but in the future we should use 1749 /// separate calling conventions for kernel and non-kernel functions. 1750 SDValue R600TargetLowering::LowerFormalArguments( 1751 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 1752 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1753 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1754 SmallVector<CCValAssign, 16> ArgLocs; 1755 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 1756 *DAG.getContext()); 1757 MachineFunction &MF = DAG.getMachineFunction(); 1758 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 1759 1760 SmallVector<ISD::InputArg, 8> LocalIns; 1761 1762 getOriginalFunctionArgs(DAG, MF.getFunction(), Ins, LocalIns); 1763 1764 AnalyzeFormalArguments(CCInfo, LocalIns); 1765 1766 for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 1767 CCValAssign &VA = ArgLocs[i]; 1768 const ISD::InputArg &In = Ins[i]; 1769 EVT VT = In.VT; 1770 EVT MemVT = VA.getLocVT(); 1771 if (!VT.isVector() && MemVT.isVector()) { 1772 // Get load source type if scalarized. 1773 MemVT = MemVT.getVectorElementType(); 1774 } 1775 1776 if (AMDGPU::isShader(CallConv)) { 1777 unsigned Reg = MF.addLiveIn(VA.getLocReg(), &AMDGPU::R600_Reg128RegClass); 1778 SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT); 1779 InVals.push_back(Register); 1780 continue; 1781 } 1782 1783 PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1784 AMDGPUAS::CONSTANT_BUFFER_0); 1785 1786 // i64 isn't a legal type, so the register type used ends up as i32, which 1787 // isn't expected here. It attempts to create this sextload, but it ends up 1788 // being invalid. Somehow this seems to work with i64 arguments, but breaks 1789 // for <1 x i64>. 1790 1791 // The first 36 bytes of the input buffer contains information about 1792 // thread group and global sizes. 1793 ISD::LoadExtType Ext = ISD::NON_EXTLOAD; 1794 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) { 1795 // FIXME: This should really check the extload type, but the handling of 1796 // extload vector parameters seems to be broken. 1797 1798 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD; 1799 Ext = ISD::SEXTLOAD; 1800 } 1801 1802 // Compute the offset from the value. 1803 // XXX - I think PartOffset should give you this, but it seems to give the 1804 // size of the register which isn't useful. 1805 1806 unsigned ValBase = ArgLocs[In.getOrigArgIndex()].getLocMemOffset(); 1807 unsigned PartOffset = VA.getLocMemOffset(); 1808 unsigned Offset = 36 + VA.getLocMemOffset(); 1809 1810 MachinePointerInfo PtrInfo(UndefValue::get(PtrTy), PartOffset - ValBase); 1811 SDValue Arg = DAG.getLoad(ISD::UNINDEXED, Ext, VT, DL, Chain, 1812 DAG.getConstant(Offset, DL, MVT::i32), 1813 DAG.getUNDEF(MVT::i32), 1814 PtrInfo, 1815 MemVT, false, true, true, 4); 1816 1817 // 4 is the preferred alignment for the CONSTANT memory space. 1818 InVals.push_back(Arg); 1819 MFI->ABIArgOffset = Offset + MemVT.getStoreSize(); 1820 } 1821 return Chain; 1822 } 1823 1824 EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1825 EVT VT) const { 1826 if (!VT.isVector()) 1827 return MVT::i32; 1828 return VT.changeVectorElementTypeToInteger(); 1829 } 1830 1831 bool R600TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 1832 unsigned AddrSpace, 1833 unsigned Align, 1834 bool *IsFast) const { 1835 if (IsFast) 1836 *IsFast = false; 1837 1838 if (!VT.isSimple() || VT == MVT::Other) 1839 return false; 1840 1841 if (VT.bitsLT(MVT::i32)) 1842 return false; 1843 1844 // TODO: This is a rough estimate. 1845 if (IsFast) 1846 *IsFast = true; 1847 1848 return VT.bitsGT(MVT::i32) && Align % 4 == 0; 1849 } 1850 1851 static SDValue CompactSwizzlableVector( 1852 SelectionDAG &DAG, SDValue VectorEntry, 1853 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1854 assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 1855 assert(RemapSwizzle.empty()); 1856 SDValue NewBldVec[4] = { 1857 VectorEntry.getOperand(0), 1858 VectorEntry.getOperand(1), 1859 VectorEntry.getOperand(2), 1860 VectorEntry.getOperand(3) 1861 }; 1862 1863 for (unsigned i = 0; i < 4; i++) { 1864 if (NewBldVec[i].isUndef()) 1865 // We mask write here to teach later passes that the ith element of this 1866 // vector is undef. Thus we can use it to reduce 128 bits reg usage, 1867 // break false dependencies and additionnaly make assembly easier to read. 1868 RemapSwizzle[i] = 7; // SEL_MASK_WRITE 1869 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) { 1870 if (C->isZero()) { 1871 RemapSwizzle[i] = 4; // SEL_0 1872 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1873 } else if (C->isExactlyValue(1.0)) { 1874 RemapSwizzle[i] = 5; // SEL_1 1875 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1876 } 1877 } 1878 1879 if (NewBldVec[i].isUndef()) 1880 continue; 1881 for (unsigned j = 0; j < i; j++) { 1882 if (NewBldVec[i] == NewBldVec[j]) { 1883 NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); 1884 RemapSwizzle[i] = j; 1885 break; 1886 } 1887 } 1888 } 1889 1890 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1891 NewBldVec); 1892 } 1893 1894 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, 1895 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1896 assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 1897 assert(RemapSwizzle.empty()); 1898 SDValue NewBldVec[4] = { 1899 VectorEntry.getOperand(0), 1900 VectorEntry.getOperand(1), 1901 VectorEntry.getOperand(2), 1902 VectorEntry.getOperand(3) 1903 }; 1904 bool isUnmovable[4] = { false, false, false, false }; 1905 for (unsigned i = 0; i < 4; i++) { 1906 RemapSwizzle[i] = i; 1907 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1908 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1909 ->getZExtValue(); 1910 if (i == Idx) 1911 isUnmovable[Idx] = true; 1912 } 1913 } 1914 1915 for (unsigned i = 0; i < 4; i++) { 1916 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1917 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1918 ->getZExtValue(); 1919 if (isUnmovable[Idx]) 1920 continue; 1921 // Swap i and Idx 1922 std::swap(NewBldVec[Idx], NewBldVec[i]); 1923 std::swap(RemapSwizzle[i], RemapSwizzle[Idx]); 1924 break; 1925 } 1926 } 1927 1928 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1929 NewBldVec); 1930 } 1931 1932 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4], 1933 SelectionDAG &DAG, 1934 const SDLoc &DL) const { 1935 assert(BuildVector.getOpcode() == ISD::BUILD_VECTOR); 1936 // Old -> New swizzle values 1937 DenseMap<unsigned, unsigned> SwizzleRemap; 1938 1939 BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap); 1940 for (unsigned i = 0; i < 4; i++) { 1941 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1942 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1943 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1944 } 1945 1946 SwizzleRemap.clear(); 1947 BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap); 1948 for (unsigned i = 0; i < 4; i++) { 1949 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1950 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1951 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1952 } 1953 1954 return BuildVector; 1955 } 1956 1957 1958 //===----------------------------------------------------------------------===// 1959 // Custom DAG Optimizations 1960 //===----------------------------------------------------------------------===// 1961 1962 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N, 1963 DAGCombinerInfo &DCI) const { 1964 SelectionDAG &DAG = DCI.DAG; 1965 1966 switch (N->getOpcode()) { 1967 default: return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1968 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) 1969 case ISD::FP_ROUND: { 1970 SDValue Arg = N->getOperand(0); 1971 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) { 1972 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), N->getValueType(0), 1973 Arg.getOperand(0)); 1974 } 1975 break; 1976 } 1977 1978 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> 1979 // (i32 select_cc f32, f32, -1, 0 cc) 1980 // 1981 // Mesa's GLSL frontend generates the above pattern a lot and we can lower 1982 // this to one of the SET*_DX10 instructions. 1983 case ISD::FP_TO_SINT: { 1984 SDValue FNeg = N->getOperand(0); 1985 if (FNeg.getOpcode() != ISD::FNEG) { 1986 return SDValue(); 1987 } 1988 SDValue SelectCC = FNeg.getOperand(0); 1989 if (SelectCC.getOpcode() != ISD::SELECT_CC || 1990 SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS 1991 SelectCC.getOperand(2).getValueType() != MVT::f32 || // True 1992 !isHWTrueValue(SelectCC.getOperand(2)) || 1993 !isHWFalseValue(SelectCC.getOperand(3))) { 1994 return SDValue(); 1995 } 1996 1997 SDLoc dl(N); 1998 return DAG.getNode(ISD::SELECT_CC, dl, N->getValueType(0), 1999 SelectCC.getOperand(0), // LHS 2000 SelectCC.getOperand(1), // RHS 2001 DAG.getConstant(-1, dl, MVT::i32), // True 2002 DAG.getConstant(0, dl, MVT::i32), // False 2003 SelectCC.getOperand(4)); // CC 2004 2005 break; 2006 } 2007 2008 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx 2009 // => build_vector elt0, ... , NewEltIdx, ... , eltN 2010 case ISD::INSERT_VECTOR_ELT: { 2011 SDValue InVec = N->getOperand(0); 2012 SDValue InVal = N->getOperand(1); 2013 SDValue EltNo = N->getOperand(2); 2014 SDLoc dl(N); 2015 2016 // If the inserted element is an UNDEF, just use the input vector. 2017 if (InVal.isUndef()) 2018 return InVec; 2019 2020 EVT VT = InVec.getValueType(); 2021 2022 // If we can't generate a legal BUILD_VECTOR, exit 2023 if (!isOperationLegal(ISD::BUILD_VECTOR, VT)) 2024 return SDValue(); 2025 2026 // Check that we know which element is being inserted 2027 if (!isa<ConstantSDNode>(EltNo)) 2028 return SDValue(); 2029 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 2030 2031 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 2032 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 2033 // vector elements. 2034 SmallVector<SDValue, 8> Ops; 2035 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 2036 Ops.append(InVec.getNode()->op_begin(), 2037 InVec.getNode()->op_end()); 2038 } else if (InVec.isUndef()) { 2039 unsigned NElts = VT.getVectorNumElements(); 2040 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 2041 } else { 2042 return SDValue(); 2043 } 2044 2045 // Insert the element 2046 if (Elt < Ops.size()) { 2047 // All the operands of BUILD_VECTOR must have the same type; 2048 // we enforce that here. 2049 EVT OpVT = Ops[0].getValueType(); 2050 if (InVal.getValueType() != OpVT) 2051 InVal = OpVT.bitsGT(InVal.getValueType()) ? 2052 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : 2053 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); 2054 Ops[Elt] = InVal; 2055 } 2056 2057 // Return the new vector 2058 return DAG.getBuildVector(VT, dl, Ops); 2059 } 2060 2061 // Extract_vec (Build_vector) generated by custom lowering 2062 // also needs to be customly combined 2063 case ISD::EXTRACT_VECTOR_ELT: { 2064 SDValue Arg = N->getOperand(0); 2065 if (Arg.getOpcode() == ISD::BUILD_VECTOR) { 2066 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2067 unsigned Element = Const->getZExtValue(); 2068 return Arg->getOperand(Element); 2069 } 2070 } 2071 if (Arg.getOpcode() == ISD::BITCAST && 2072 Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { 2073 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2074 unsigned Element = Const->getZExtValue(); 2075 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getVTList(), 2076 Arg->getOperand(0).getOperand(Element)); 2077 } 2078 } 2079 break; 2080 } 2081 2082 case ISD::SELECT_CC: { 2083 // Try common optimizations 2084 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI)) 2085 return Ret; 2086 2087 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> 2088 // selectcc x, y, a, b, inv(cc) 2089 // 2090 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> 2091 // selectcc x, y, a, b, cc 2092 SDValue LHS = N->getOperand(0); 2093 if (LHS.getOpcode() != ISD::SELECT_CC) { 2094 return SDValue(); 2095 } 2096 2097 SDValue RHS = N->getOperand(1); 2098 SDValue True = N->getOperand(2); 2099 SDValue False = N->getOperand(3); 2100 ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2101 2102 if (LHS.getOperand(2).getNode() != True.getNode() || 2103 LHS.getOperand(3).getNode() != False.getNode() || 2104 RHS.getNode() != False.getNode()) { 2105 return SDValue(); 2106 } 2107 2108 switch (NCC) { 2109 default: return SDValue(); 2110 case ISD::SETNE: return LHS; 2111 case ISD::SETEQ: { 2112 ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); 2113 LHSCC = ISD::getSetCCInverse(LHSCC, 2114 LHS.getOperand(0).getValueType().isInteger()); 2115 if (DCI.isBeforeLegalizeOps() || 2116 isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) 2117 return DAG.getSelectCC(SDLoc(N), 2118 LHS.getOperand(0), 2119 LHS.getOperand(1), 2120 LHS.getOperand(2), 2121 LHS.getOperand(3), 2122 LHSCC); 2123 break; 2124 } 2125 } 2126 return SDValue(); 2127 } 2128 2129 case AMDGPUISD::EXPORT: { 2130 SDValue Arg = N->getOperand(1); 2131 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2132 break; 2133 2134 SDValue NewArgs[8] = { 2135 N->getOperand(0), // Chain 2136 SDValue(), 2137 N->getOperand(2), // ArrayBase 2138 N->getOperand(3), // Type 2139 N->getOperand(4), // SWZ_X 2140 N->getOperand(5), // SWZ_Y 2141 N->getOperand(6), // SWZ_Z 2142 N->getOperand(7) // SWZ_W 2143 }; 2144 SDLoc DL(N); 2145 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL); 2146 return DAG.getNode(AMDGPUISD::EXPORT, DL, N->getVTList(), NewArgs); 2147 } 2148 case AMDGPUISD::TEXTURE_FETCH: { 2149 SDValue Arg = N->getOperand(1); 2150 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2151 break; 2152 2153 SDValue NewArgs[19] = { 2154 N->getOperand(0), 2155 N->getOperand(1), 2156 N->getOperand(2), 2157 N->getOperand(3), 2158 N->getOperand(4), 2159 N->getOperand(5), 2160 N->getOperand(6), 2161 N->getOperand(7), 2162 N->getOperand(8), 2163 N->getOperand(9), 2164 N->getOperand(10), 2165 N->getOperand(11), 2166 N->getOperand(12), 2167 N->getOperand(13), 2168 N->getOperand(14), 2169 N->getOperand(15), 2170 N->getOperand(16), 2171 N->getOperand(17), 2172 N->getOperand(18), 2173 }; 2174 SDLoc DL(N); 2175 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL); 2176 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs); 2177 } 2178 } 2179 2180 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 2181 } 2182 2183 bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx, 2184 SDValue &Src, SDValue &Neg, SDValue &Abs, 2185 SDValue &Sel, SDValue &Imm, 2186 SelectionDAG &DAG) const { 2187 const R600InstrInfo *TII = getSubtarget()->getInstrInfo(); 2188 if (!Src.isMachineOpcode()) 2189 return false; 2190 2191 switch (Src.getMachineOpcode()) { 2192 case AMDGPU::FNEG_R600: 2193 if (!Neg.getNode()) 2194 return false; 2195 Src = Src.getOperand(0); 2196 Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2197 return true; 2198 case AMDGPU::FABS_R600: 2199 if (!Abs.getNode()) 2200 return false; 2201 Src = Src.getOperand(0); 2202 Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2203 return true; 2204 case AMDGPU::CONST_COPY: { 2205 unsigned Opcode = ParentNode->getMachineOpcode(); 2206 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2207 2208 if (!Sel.getNode()) 2209 return false; 2210 2211 SDValue CstOffset = Src.getOperand(0); 2212 if (ParentNode->getValueType(0).isVector()) 2213 return false; 2214 2215 // Gather constants values 2216 int SrcIndices[] = { 2217 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2218 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2219 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2), 2220 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2221 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2222 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2223 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2224 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2225 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2226 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2227 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2228 }; 2229 std::vector<unsigned> Consts; 2230 for (int OtherSrcIdx : SrcIndices) { 2231 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); 2232 if (OtherSrcIdx < 0 || OtherSelIdx < 0) 2233 continue; 2234 if (HasDst) { 2235 OtherSrcIdx--; 2236 OtherSelIdx--; 2237 } 2238 if (RegisterSDNode *Reg = 2239 dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { 2240 if (Reg->getReg() == AMDGPU::ALU_CONST) { 2241 ConstantSDNode *Cst 2242 = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx)); 2243 Consts.push_back(Cst->getZExtValue()); 2244 } 2245 } 2246 } 2247 2248 ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset); 2249 Consts.push_back(Cst->getZExtValue()); 2250 if (!TII->fitsConstReadLimitations(Consts)) { 2251 return false; 2252 } 2253 2254 Sel = CstOffset; 2255 Src = DAG.getRegister(AMDGPU::ALU_CONST, MVT::f32); 2256 return true; 2257 } 2258 case AMDGPU::MOV_IMM_GLOBAL_ADDR: 2259 // Check if the Imm slot is used. Taken from below. 2260 if (cast<ConstantSDNode>(Imm)->getZExtValue()) 2261 return false; 2262 Imm = Src.getOperand(0); 2263 Src = DAG.getRegister(AMDGPU::ALU_LITERAL_X, MVT::i32); 2264 return true; 2265 case AMDGPU::MOV_IMM_I32: 2266 case AMDGPU::MOV_IMM_F32: { 2267 unsigned ImmReg = AMDGPU::ALU_LITERAL_X; 2268 uint64_t ImmValue = 0; 2269 2270 2271 if (Src.getMachineOpcode() == AMDGPU::MOV_IMM_F32) { 2272 ConstantFPSDNode *FPC = dyn_cast<ConstantFPSDNode>(Src.getOperand(0)); 2273 float FloatValue = FPC->getValueAPF().convertToFloat(); 2274 if (FloatValue == 0.0) { 2275 ImmReg = AMDGPU::ZERO; 2276 } else if (FloatValue == 0.5) { 2277 ImmReg = AMDGPU::HALF; 2278 } else if (FloatValue == 1.0) { 2279 ImmReg = AMDGPU::ONE; 2280 } else { 2281 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); 2282 } 2283 } else { 2284 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src.getOperand(0)); 2285 uint64_t Value = C->getZExtValue(); 2286 if (Value == 0) { 2287 ImmReg = AMDGPU::ZERO; 2288 } else if (Value == 1) { 2289 ImmReg = AMDGPU::ONE_INT; 2290 } else { 2291 ImmValue = Value; 2292 } 2293 } 2294 2295 // Check that we aren't already using an immediate. 2296 // XXX: It's possible for an instruction to have more than one 2297 // immediate operand, but this is not supported yet. 2298 if (ImmReg == AMDGPU::ALU_LITERAL_X) { 2299 if (!Imm.getNode()) 2300 return false; 2301 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Imm); 2302 assert(C); 2303 if (C->getZExtValue()) 2304 return false; 2305 Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32); 2306 } 2307 Src = DAG.getRegister(ImmReg, MVT::i32); 2308 return true; 2309 } 2310 default: 2311 return false; 2312 } 2313 } 2314 2315 /// \brief Fold the instructions after selecting them 2316 SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node, 2317 SelectionDAG &DAG) const { 2318 const R600InstrInfo *TII = getSubtarget()->getInstrInfo(); 2319 if (!Node->isMachineOpcode()) 2320 return Node; 2321 2322 unsigned Opcode = Node->getMachineOpcode(); 2323 SDValue FakeOp; 2324 2325 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end()); 2326 2327 if (Opcode == AMDGPU::DOT_4) { 2328 int OperandIdx[] = { 2329 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2330 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2331 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2332 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2333 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2334 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2335 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2336 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2337 }; 2338 int NegIdx[] = { 2339 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_X), 2340 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Y), 2341 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Z), 2342 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_W), 2343 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_X), 2344 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Y), 2345 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Z), 2346 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_W) 2347 }; 2348 int AbsIdx[] = { 2349 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_X), 2350 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Y), 2351 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Z), 2352 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_W), 2353 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_X), 2354 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Y), 2355 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Z), 2356 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_W) 2357 }; 2358 for (unsigned i = 0; i < 8; i++) { 2359 if (OperandIdx[i] < 0) 2360 return Node; 2361 SDValue &Src = Ops[OperandIdx[i] - 1]; 2362 SDValue &Neg = Ops[NegIdx[i] - 1]; 2363 SDValue &Abs = Ops[AbsIdx[i] - 1]; 2364 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2365 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2366 if (HasDst) 2367 SelIdx--; 2368 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2369 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG)) 2370 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2371 } 2372 } else if (Opcode == AMDGPU::REG_SEQUENCE) { 2373 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { 2374 SDValue &Src = Ops[i]; 2375 if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG)) 2376 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2377 } 2378 } else if (Opcode == AMDGPU::CLAMP_R600) { 2379 SDValue Src = Node->getOperand(0); 2380 if (!Src.isMachineOpcode() || 2381 !TII->hasInstrModifiers(Src.getMachineOpcode())) 2382 return Node; 2383 int ClampIdx = TII->getOperandIdx(Src.getMachineOpcode(), 2384 AMDGPU::OpName::clamp); 2385 if (ClampIdx < 0) 2386 return Node; 2387 SDLoc DL(Node); 2388 std::vector<SDValue> Ops(Src->op_begin(), Src->op_end()); 2389 Ops[ClampIdx - 1] = DAG.getTargetConstant(1, DL, MVT::i32); 2390 return DAG.getMachineNode(Src.getMachineOpcode(), DL, 2391 Node->getVTList(), Ops); 2392 } else { 2393 if (!TII->hasInstrModifiers(Opcode)) 2394 return Node; 2395 int OperandIdx[] = { 2396 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2397 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2398 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2) 2399 }; 2400 int NegIdx[] = { 2401 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg), 2402 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg), 2403 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2_neg) 2404 }; 2405 int AbsIdx[] = { 2406 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs), 2407 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs), 2408 -1 2409 }; 2410 for (unsigned i = 0; i < 3; i++) { 2411 if (OperandIdx[i] < 0) 2412 return Node; 2413 SDValue &Src = Ops[OperandIdx[i] - 1]; 2414 SDValue &Neg = Ops[NegIdx[i] - 1]; 2415 SDValue FakeAbs; 2416 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; 2417 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2418 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2419 int ImmIdx = TII->getOperandIdx(Opcode, AMDGPU::OpName::literal); 2420 if (HasDst) { 2421 SelIdx--; 2422 ImmIdx--; 2423 } 2424 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2425 SDValue &Imm = Ops[ImmIdx]; 2426 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG)) 2427 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2428 } 2429 } 2430 2431 return Node; 2432 } 2433