1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// \brief Custom DAG lowering for R600 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "R600ISelLowering.h" 16 #include "AMDGPUFrameLowering.h" 17 #include "AMDGPUIntrinsicInfo.h" 18 #include "AMDGPUSubtarget.h" 19 #include "R600Defines.h" 20 #include "R600InstrInfo.h" 21 #include "R600MachineFunctionInfo.h" 22 #include "llvm/Analysis/ValueTracking.h" 23 #include "llvm/CodeGen/CallingConvLower.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/MachineInstrBuilder.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/SelectionDAG.h" 28 #include "llvm/IR/Argument.h" 29 #include "llvm/IR/Function.h" 30 31 using namespace llvm; 32 33 R600TargetLowering::R600TargetLowering(TargetMachine &TM, 34 const AMDGPUSubtarget &STI) 35 : AMDGPUTargetLowering(TM, STI), Gen(STI.getGeneration()) { 36 addRegisterClass(MVT::f32, &AMDGPU::R600_Reg32RegClass); 37 addRegisterClass(MVT::i32, &AMDGPU::R600_Reg32RegClass); 38 addRegisterClass(MVT::v2f32, &AMDGPU::R600_Reg64RegClass); 39 addRegisterClass(MVT::v2i32, &AMDGPU::R600_Reg64RegClass); 40 addRegisterClass(MVT::v4f32, &AMDGPU::R600_Reg128RegClass); 41 addRegisterClass(MVT::v4i32, &AMDGPU::R600_Reg128RegClass); 42 43 computeRegisterProperties(STI.getRegisterInfo()); 44 45 // Legalize loads and stores to the private address space. 46 setOperationAction(ISD::LOAD, MVT::i32, Custom); 47 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 48 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 49 50 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address 51 // spaces, so it is custom lowered to handle those where it isn't. 52 for (MVT VT : MVT::integer_valuetypes()) { 53 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 54 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Custom); 55 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Custom); 56 57 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 58 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Custom); 59 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Custom); 60 61 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 62 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Custom); 63 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Custom); 64 } 65 66 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads. 67 setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 68 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 69 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 70 71 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 72 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 73 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 74 75 76 setOperationAction(ISD::STORE, MVT::i8, Custom); 77 setOperationAction(ISD::STORE, MVT::i32, Custom); 78 setOperationAction(ISD::STORE, MVT::v2i32, Custom); 79 setOperationAction(ISD::STORE, MVT::v4i32, Custom); 80 81 setTruncStoreAction(MVT::i32, MVT::i8, Custom); 82 setTruncStoreAction(MVT::i32, MVT::i16, Custom); 83 84 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores. 85 setTruncStoreAction(MVT::v2i32, MVT::v2i1, Expand); 86 setTruncStoreAction(MVT::v4i32, MVT::v4i1, Expand); 87 88 // Set condition code actions 89 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 90 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 91 setCondCodeAction(ISD::SETLT, MVT::f32, Expand); 92 setCondCodeAction(ISD::SETLE, MVT::f32, Expand); 93 setCondCodeAction(ISD::SETOLT, MVT::f32, Expand); 94 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 95 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 96 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 97 setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); 98 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 99 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 100 setCondCodeAction(ISD::SETULE, MVT::f32, Expand); 101 102 setCondCodeAction(ISD::SETLE, MVT::i32, Expand); 103 setCondCodeAction(ISD::SETLT, MVT::i32, Expand); 104 setCondCodeAction(ISD::SETULE, MVT::i32, Expand); 105 setCondCodeAction(ISD::SETULT, MVT::i32, Expand); 106 107 setOperationAction(ISD::FCOS, MVT::f32, Custom); 108 setOperationAction(ISD::FSIN, MVT::f32, Custom); 109 110 setOperationAction(ISD::SETCC, MVT::v4i32, Expand); 111 setOperationAction(ISD::SETCC, MVT::v2i32, Expand); 112 113 setOperationAction(ISD::BR_CC, MVT::i32, Expand); 114 setOperationAction(ISD::BR_CC, MVT::f32, Expand); 115 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 116 117 setOperationAction(ISD::FSUB, MVT::f32, Expand); 118 119 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 120 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 121 122 setOperationAction(ISD::SETCC, MVT::i32, Expand); 123 setOperationAction(ISD::SETCC, MVT::f32, Expand); 124 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom); 125 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 126 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 127 128 setOperationAction(ISD::SELECT, MVT::i32, Expand); 129 setOperationAction(ISD::SELECT, MVT::f32, Expand); 130 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 131 setOperationAction(ISD::SELECT, MVT::v4i32, Expand); 132 133 // ADD, SUB overflow. 134 // TODO: turn these into Legal? 135 if (Subtarget->hasCARRY()) 136 setOperationAction(ISD::UADDO, MVT::i32, Custom); 137 138 if (Subtarget->hasBORROW()) 139 setOperationAction(ISD::USUBO, MVT::i32, Custom); 140 141 // Expand sign extension of vectors 142 if (!Subtarget->hasBFE()) 143 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 144 145 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand); 146 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand); 147 148 if (!Subtarget->hasBFE()) 149 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 150 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand); 151 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand); 152 153 if (!Subtarget->hasBFE()) 154 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 155 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand); 156 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand); 157 158 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 159 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand); 160 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand); 161 162 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand); 163 164 setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 165 166 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom); 167 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom); 168 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 169 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 170 171 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom); 172 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom); 173 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 174 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 175 176 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 177 // to be Legal/Custom in order to avoid library calls. 178 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 179 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 180 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 181 182 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 183 184 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 185 for (MVT VT : ScalarIntVTs) { 186 setOperationAction(ISD::ADDC, VT, Expand); 187 setOperationAction(ISD::SUBC, VT, Expand); 188 setOperationAction(ISD::ADDE, VT, Expand); 189 setOperationAction(ISD::SUBE, VT, Expand); 190 } 191 192 setSchedulingPreference(Sched::Source); 193 194 195 setTargetDAGCombine(ISD::FP_ROUND); 196 setTargetDAGCombine(ISD::FP_TO_SINT); 197 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 198 setTargetDAGCombine(ISD::SELECT_CC); 199 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 200 } 201 202 static inline bool isEOP(MachineBasicBlock::iterator I) { 203 return std::next(I)->getOpcode() == AMDGPU::RETURN; 204 } 205 206 MachineBasicBlock * R600TargetLowering::EmitInstrWithCustomInserter( 207 MachineInstr * MI, MachineBasicBlock * BB) const { 208 MachineFunction * MF = BB->getParent(); 209 MachineRegisterInfo &MRI = MF->getRegInfo(); 210 MachineBasicBlock::iterator I = *MI; 211 const R600InstrInfo *TII = 212 static_cast<const R600InstrInfo *>(Subtarget->getInstrInfo()); 213 214 switch (MI->getOpcode()) { 215 default: 216 // Replace LDS_*_RET instruction that don't have any uses with the 217 // equivalent LDS_*_NORET instruction. 218 if (TII->isLDSRetInstr(MI->getOpcode())) { 219 int DstIdx = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst); 220 assert(DstIdx != -1); 221 MachineInstrBuilder NewMI; 222 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add 223 // LDS_1A2D support and remove this special case. 224 if (!MRI.use_empty(MI->getOperand(DstIdx).getReg()) || 225 MI->getOpcode() == AMDGPU::LDS_CMPST_RET) 226 return BB; 227 228 NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), 229 TII->get(AMDGPU::getLDSNoRetOp(MI->getOpcode()))); 230 for (unsigned i = 1, e = MI->getNumOperands(); i < e; ++i) { 231 NewMI.addOperand(MI->getOperand(i)); 232 } 233 } else { 234 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 235 } 236 break; 237 case AMDGPU::CLAMP_R600: { 238 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 239 AMDGPU::MOV, 240 MI->getOperand(0).getReg(), 241 MI->getOperand(1).getReg()); 242 TII->addFlag(NewMI, 0, MO_FLAG_CLAMP); 243 break; 244 } 245 246 case AMDGPU::FABS_R600: { 247 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 248 AMDGPU::MOV, 249 MI->getOperand(0).getReg(), 250 MI->getOperand(1).getReg()); 251 TII->addFlag(NewMI, 0, MO_FLAG_ABS); 252 break; 253 } 254 255 case AMDGPU::FNEG_R600: { 256 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 257 AMDGPU::MOV, 258 MI->getOperand(0).getReg(), 259 MI->getOperand(1).getReg()); 260 TII->addFlag(NewMI, 0, MO_FLAG_NEG); 261 break; 262 } 263 264 case AMDGPU::MASK_WRITE: { 265 unsigned maskedRegister = MI->getOperand(0).getReg(); 266 assert(TargetRegisterInfo::isVirtualRegister(maskedRegister)); 267 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister); 268 TII->addFlag(defInstr, 0, MO_FLAG_MASK); 269 break; 270 } 271 272 case AMDGPU::MOV_IMM_F32: 273 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 274 MI->getOperand(1).getFPImm()->getValueAPF() 275 .bitcastToAPInt().getZExtValue()); 276 break; 277 case AMDGPU::MOV_IMM_I32: 278 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 279 MI->getOperand(1).getImm()); 280 break; 281 case AMDGPU::MOV_IMM_GLOBAL_ADDR: { 282 //TODO: Perhaps combine this instruction with the next if possible 283 auto MIB = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 284 MI->getOperand(0).getReg(), 285 AMDGPU::ALU_LITERAL_X); 286 int Idx = TII->getOperandIdx(*MIB, AMDGPU::OpName::literal); 287 //TODO: Ugh this is rather ugly 288 MIB->getOperand(Idx) = MI->getOperand(1); 289 break; 290 } 291 case AMDGPU::CONST_COPY: { 292 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 293 MI->getOperand(0).getReg(), AMDGPU::ALU_CONST); 294 TII->setImmOperand(NewMI, AMDGPU::OpName::src0_sel, 295 MI->getOperand(1).getImm()); 296 break; 297 } 298 299 case AMDGPU::RAT_WRITE_CACHELESS_32_eg: 300 case AMDGPU::RAT_WRITE_CACHELESS_64_eg: 301 case AMDGPU::RAT_WRITE_CACHELESS_128_eg: { 302 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 303 .addOperand(MI->getOperand(0)) 304 .addOperand(MI->getOperand(1)) 305 .addImm(isEOP(I)); // Set End of program bit 306 break; 307 } 308 case AMDGPU::RAT_STORE_TYPED_eg: { 309 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 310 .addOperand(MI->getOperand(0)) 311 .addOperand(MI->getOperand(1)) 312 .addOperand(MI->getOperand(2)) 313 .addImm(isEOP(I)); // Set End of program bit 314 break; 315 } 316 317 case AMDGPU::TXD: { 318 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 319 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 320 MachineOperand &RID = MI->getOperand(4); 321 MachineOperand &SID = MI->getOperand(5); 322 unsigned TextureId = MI->getOperand(6).getImm(); 323 unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 324 unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 325 326 switch (TextureId) { 327 case 5: // Rect 328 CTX = CTY = 0; 329 break; 330 case 6: // Shadow1D 331 SrcW = SrcZ; 332 break; 333 case 7: // Shadow2D 334 SrcW = SrcZ; 335 break; 336 case 8: // ShadowRect 337 CTX = CTY = 0; 338 SrcW = SrcZ; 339 break; 340 case 9: // 1DArray 341 SrcZ = SrcY; 342 CTZ = 0; 343 break; 344 case 10: // 2DArray 345 CTZ = 0; 346 break; 347 case 11: // Shadow1DArray 348 SrcZ = SrcY; 349 CTZ = 0; 350 break; 351 case 12: // Shadow2DArray 352 CTZ = 0; 353 break; 354 } 355 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 356 .addOperand(MI->getOperand(3)) 357 .addImm(SrcX) 358 .addImm(SrcY) 359 .addImm(SrcZ) 360 .addImm(SrcW) 361 .addImm(0) 362 .addImm(0) 363 .addImm(0) 364 .addImm(0) 365 .addImm(1) 366 .addImm(2) 367 .addImm(3) 368 .addOperand(RID) 369 .addOperand(SID) 370 .addImm(CTX) 371 .addImm(CTY) 372 .addImm(CTZ) 373 .addImm(CTW); 374 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 375 .addOperand(MI->getOperand(2)) 376 .addImm(SrcX) 377 .addImm(SrcY) 378 .addImm(SrcZ) 379 .addImm(SrcW) 380 .addImm(0) 381 .addImm(0) 382 .addImm(0) 383 .addImm(0) 384 .addImm(1) 385 .addImm(2) 386 .addImm(3) 387 .addOperand(RID) 388 .addOperand(SID) 389 .addImm(CTX) 390 .addImm(CTY) 391 .addImm(CTZ) 392 .addImm(CTW); 393 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_G)) 394 .addOperand(MI->getOperand(0)) 395 .addOperand(MI->getOperand(1)) 396 .addImm(SrcX) 397 .addImm(SrcY) 398 .addImm(SrcZ) 399 .addImm(SrcW) 400 .addImm(0) 401 .addImm(0) 402 .addImm(0) 403 .addImm(0) 404 .addImm(1) 405 .addImm(2) 406 .addImm(3) 407 .addOperand(RID) 408 .addOperand(SID) 409 .addImm(CTX) 410 .addImm(CTY) 411 .addImm(CTZ) 412 .addImm(CTW) 413 .addReg(T0, RegState::Implicit) 414 .addReg(T1, RegState::Implicit); 415 break; 416 } 417 418 case AMDGPU::TXD_SHADOW: { 419 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 420 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 421 MachineOperand &RID = MI->getOperand(4); 422 MachineOperand &SID = MI->getOperand(5); 423 unsigned TextureId = MI->getOperand(6).getImm(); 424 unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 425 unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 426 427 switch (TextureId) { 428 case 5: // Rect 429 CTX = CTY = 0; 430 break; 431 case 6: // Shadow1D 432 SrcW = SrcZ; 433 break; 434 case 7: // Shadow2D 435 SrcW = SrcZ; 436 break; 437 case 8: // ShadowRect 438 CTX = CTY = 0; 439 SrcW = SrcZ; 440 break; 441 case 9: // 1DArray 442 SrcZ = SrcY; 443 CTZ = 0; 444 break; 445 case 10: // 2DArray 446 CTZ = 0; 447 break; 448 case 11: // Shadow1DArray 449 SrcZ = SrcY; 450 CTZ = 0; 451 break; 452 case 12: // Shadow2DArray 453 CTZ = 0; 454 break; 455 } 456 457 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 458 .addOperand(MI->getOperand(3)) 459 .addImm(SrcX) 460 .addImm(SrcY) 461 .addImm(SrcZ) 462 .addImm(SrcW) 463 .addImm(0) 464 .addImm(0) 465 .addImm(0) 466 .addImm(0) 467 .addImm(1) 468 .addImm(2) 469 .addImm(3) 470 .addOperand(RID) 471 .addOperand(SID) 472 .addImm(CTX) 473 .addImm(CTY) 474 .addImm(CTZ) 475 .addImm(CTW); 476 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 477 .addOperand(MI->getOperand(2)) 478 .addImm(SrcX) 479 .addImm(SrcY) 480 .addImm(SrcZ) 481 .addImm(SrcW) 482 .addImm(0) 483 .addImm(0) 484 .addImm(0) 485 .addImm(0) 486 .addImm(1) 487 .addImm(2) 488 .addImm(3) 489 .addOperand(RID) 490 .addOperand(SID) 491 .addImm(CTX) 492 .addImm(CTY) 493 .addImm(CTZ) 494 .addImm(CTW); 495 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_C_G)) 496 .addOperand(MI->getOperand(0)) 497 .addOperand(MI->getOperand(1)) 498 .addImm(SrcX) 499 .addImm(SrcY) 500 .addImm(SrcZ) 501 .addImm(SrcW) 502 .addImm(0) 503 .addImm(0) 504 .addImm(0) 505 .addImm(0) 506 .addImm(1) 507 .addImm(2) 508 .addImm(3) 509 .addOperand(RID) 510 .addOperand(SID) 511 .addImm(CTX) 512 .addImm(CTY) 513 .addImm(CTZ) 514 .addImm(CTW) 515 .addReg(T0, RegState::Implicit) 516 .addReg(T1, RegState::Implicit); 517 break; 518 } 519 520 case AMDGPU::BRANCH: 521 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP)) 522 .addOperand(MI->getOperand(0)); 523 break; 524 525 case AMDGPU::BRANCH_COND_f32: { 526 MachineInstr *NewMI = 527 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 528 AMDGPU::PREDICATE_BIT) 529 .addOperand(MI->getOperand(1)) 530 .addImm(OPCODE_IS_NOT_ZERO) 531 .addImm(0); // Flags 532 TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 533 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 534 .addOperand(MI->getOperand(0)) 535 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 536 break; 537 } 538 539 case AMDGPU::BRANCH_COND_i32: { 540 MachineInstr *NewMI = 541 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 542 AMDGPU::PREDICATE_BIT) 543 .addOperand(MI->getOperand(1)) 544 .addImm(OPCODE_IS_NOT_ZERO_INT) 545 .addImm(0); // Flags 546 TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 547 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 548 .addOperand(MI->getOperand(0)) 549 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 550 break; 551 } 552 553 case AMDGPU::EG_ExportSwz: 554 case AMDGPU::R600_ExportSwz: { 555 // Instruction is left unmodified if its not the last one of its type 556 bool isLastInstructionOfItsType = true; 557 unsigned InstExportType = MI->getOperand(1).getImm(); 558 for (MachineBasicBlock::iterator NextExportInst = std::next(I), 559 EndBlock = BB->end(); NextExportInst != EndBlock; 560 NextExportInst = std::next(NextExportInst)) { 561 if (NextExportInst->getOpcode() == AMDGPU::EG_ExportSwz || 562 NextExportInst->getOpcode() == AMDGPU::R600_ExportSwz) { 563 unsigned CurrentInstExportType = NextExportInst->getOperand(1) 564 .getImm(); 565 if (CurrentInstExportType == InstExportType) { 566 isLastInstructionOfItsType = false; 567 break; 568 } 569 } 570 } 571 bool EOP = isEOP(I); 572 if (!EOP && !isLastInstructionOfItsType) 573 return BB; 574 unsigned CfInst = (MI->getOpcode() == AMDGPU::EG_ExportSwz)? 84 : 40; 575 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 576 .addOperand(MI->getOperand(0)) 577 .addOperand(MI->getOperand(1)) 578 .addOperand(MI->getOperand(2)) 579 .addOperand(MI->getOperand(3)) 580 .addOperand(MI->getOperand(4)) 581 .addOperand(MI->getOperand(5)) 582 .addOperand(MI->getOperand(6)) 583 .addImm(CfInst) 584 .addImm(EOP); 585 break; 586 } 587 case AMDGPU::RETURN: { 588 // RETURN instructions must have the live-out registers as implicit uses, 589 // otherwise they appear dead. 590 R600MachineFunctionInfo *MFI = MF->getInfo<R600MachineFunctionInfo>(); 591 MachineInstrBuilder MIB(*MF, MI); 592 for (unsigned i = 0, e = MFI->LiveOuts.size(); i != e; ++i) 593 MIB.addReg(MFI->LiveOuts[i], RegState::Implicit); 594 return BB; 595 } 596 } 597 598 MI->eraseFromParent(); 599 return BB; 600 } 601 602 //===----------------------------------------------------------------------===// 603 // Custom DAG Lowering Operations 604 //===----------------------------------------------------------------------===// 605 606 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 607 MachineFunction &MF = DAG.getMachineFunction(); 608 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 609 switch (Op.getOpcode()) { 610 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 611 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 612 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 613 case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG); 614 case ISD::SRA_PARTS: 615 case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG); 616 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY); 617 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW); 618 case ISD::FCOS: 619 case ISD::FSIN: return LowerTrig(Op, DAG); 620 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 621 case ISD::STORE: return LowerSTORE(Op, DAG); 622 case ISD::LOAD: { 623 SDValue Result = LowerLOAD(Op, DAG); 624 assert((!Result.getNode() || 625 Result.getNode()->getNumValues() == 2) && 626 "Load should return a value and a chain"); 627 return Result; 628 } 629 630 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 631 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG); 632 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG); 633 case ISD::INTRINSIC_VOID: { 634 SDValue Chain = Op.getOperand(0); 635 unsigned IntrinsicID = 636 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 637 switch (IntrinsicID) { 638 case AMDGPUIntrinsic::R600_store_swizzle: { 639 SDLoc DL(Op); 640 const SDValue Args[8] = { 641 Chain, 642 Op.getOperand(2), // Export Value 643 Op.getOperand(3), // ArrayBase 644 Op.getOperand(4), // Type 645 DAG.getConstant(0, DL, MVT::i32), // SWZ_X 646 DAG.getConstant(1, DL, MVT::i32), // SWZ_Y 647 DAG.getConstant(2, DL, MVT::i32), // SWZ_Z 648 DAG.getConstant(3, DL, MVT::i32) // SWZ_W 649 }; 650 return DAG.getNode(AMDGPUISD::EXPORT, DL, Op.getValueType(), Args); 651 } 652 653 // default for switch(IntrinsicID) 654 default: break; 655 } 656 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode()) 657 break; 658 } 659 case ISD::INTRINSIC_WO_CHAIN: { 660 unsigned IntrinsicID = 661 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 662 EVT VT = Op.getValueType(); 663 SDLoc DL(Op); 664 switch(IntrinsicID) { 665 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 666 case AMDGPUIntrinsic::R600_interp_xy: 667 case AMDGPUIntrinsic::R600_interp_zw: { 668 int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 669 MachineSDNode *interp; 670 SDValue RegisterINode = Op.getOperand(2); 671 SDValue RegisterJNode = Op.getOperand(3); 672 673 if (IntrinsicID == AMDGPUIntrinsic::R600_interp_xy) 674 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 675 MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 676 RegisterJNode, RegisterINode); 677 else 678 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 679 MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 680 RegisterJNode, RegisterINode); 681 return DAG.getBuildVector(MVT::v2f32, DL, 682 {SDValue(interp, 0), SDValue(interp, 1)}); 683 } 684 case AMDGPUIntrinsic::r600_tex: 685 case AMDGPUIntrinsic::r600_texc: 686 case AMDGPUIntrinsic::r600_txl: 687 case AMDGPUIntrinsic::r600_txlc: 688 case AMDGPUIntrinsic::r600_txb: 689 case AMDGPUIntrinsic::r600_txbc: 690 case AMDGPUIntrinsic::r600_txf: 691 case AMDGPUIntrinsic::r600_txq: 692 case AMDGPUIntrinsic::r600_ddx: 693 case AMDGPUIntrinsic::r600_ddy: 694 case AMDGPUIntrinsic::R600_ldptr: { 695 unsigned TextureOp; 696 switch (IntrinsicID) { 697 case AMDGPUIntrinsic::r600_tex: 698 TextureOp = 0; 699 break; 700 case AMDGPUIntrinsic::r600_texc: 701 TextureOp = 1; 702 break; 703 case AMDGPUIntrinsic::r600_txl: 704 TextureOp = 2; 705 break; 706 case AMDGPUIntrinsic::r600_txlc: 707 TextureOp = 3; 708 break; 709 case AMDGPUIntrinsic::r600_txb: 710 TextureOp = 4; 711 break; 712 case AMDGPUIntrinsic::r600_txbc: 713 TextureOp = 5; 714 break; 715 case AMDGPUIntrinsic::r600_txf: 716 TextureOp = 6; 717 break; 718 case AMDGPUIntrinsic::r600_txq: 719 TextureOp = 7; 720 break; 721 case AMDGPUIntrinsic::r600_ddx: 722 TextureOp = 8; 723 break; 724 case AMDGPUIntrinsic::r600_ddy: 725 TextureOp = 9; 726 break; 727 case AMDGPUIntrinsic::R600_ldptr: 728 TextureOp = 10; 729 break; 730 default: 731 llvm_unreachable("Unknow Texture Operation"); 732 } 733 734 SDValue TexArgs[19] = { 735 DAG.getConstant(TextureOp, DL, MVT::i32), 736 Op.getOperand(1), 737 DAG.getConstant(0, DL, MVT::i32), 738 DAG.getConstant(1, DL, MVT::i32), 739 DAG.getConstant(2, DL, MVT::i32), 740 DAG.getConstant(3, DL, MVT::i32), 741 Op.getOperand(2), 742 Op.getOperand(3), 743 Op.getOperand(4), 744 DAG.getConstant(0, DL, MVT::i32), 745 DAG.getConstant(1, DL, MVT::i32), 746 DAG.getConstant(2, DL, MVT::i32), 747 DAG.getConstant(3, DL, MVT::i32), 748 Op.getOperand(5), 749 Op.getOperand(6), 750 Op.getOperand(7), 751 Op.getOperand(8), 752 Op.getOperand(9), 753 Op.getOperand(10) 754 }; 755 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs); 756 } 757 case AMDGPUIntrinsic::AMDGPU_dp4: { 758 SDValue Args[8] = { 759 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 760 DAG.getConstant(0, DL, MVT::i32)), 761 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 762 DAG.getConstant(0, DL, MVT::i32)), 763 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 764 DAG.getConstant(1, DL, MVT::i32)), 765 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 766 DAG.getConstant(1, DL, MVT::i32)), 767 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 768 DAG.getConstant(2, DL, MVT::i32)), 769 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 770 DAG.getConstant(2, DL, MVT::i32)), 771 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 772 DAG.getConstant(3, DL, MVT::i32)), 773 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 774 DAG.getConstant(3, DL, MVT::i32)) 775 }; 776 return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args); 777 } 778 779 case Intrinsic::r600_read_ngroups_x: 780 return LowerImplicitParameter(DAG, VT, DL, 0); 781 case Intrinsic::r600_read_ngroups_y: 782 return LowerImplicitParameter(DAG, VT, DL, 1); 783 case Intrinsic::r600_read_ngroups_z: 784 return LowerImplicitParameter(DAG, VT, DL, 2); 785 case Intrinsic::r600_read_global_size_x: 786 return LowerImplicitParameter(DAG, VT, DL, 3); 787 case Intrinsic::r600_read_global_size_y: 788 return LowerImplicitParameter(DAG, VT, DL, 4); 789 case Intrinsic::r600_read_global_size_z: 790 return LowerImplicitParameter(DAG, VT, DL, 5); 791 case Intrinsic::r600_read_local_size_x: 792 return LowerImplicitParameter(DAG, VT, DL, 6); 793 case Intrinsic::r600_read_local_size_y: 794 return LowerImplicitParameter(DAG, VT, DL, 7); 795 case Intrinsic::r600_read_local_size_z: 796 return LowerImplicitParameter(DAG, VT, DL, 8); 797 798 case Intrinsic::r600_read_workdim: 799 case AMDGPUIntrinsic::AMDGPU_read_workdim: { // Legacy name. 800 uint32_t ByteOffset = getImplicitParameterOffset(MFI, GRID_DIM); 801 return LowerImplicitParameter(DAG, VT, DL, ByteOffset / 4); 802 } 803 804 case Intrinsic::r600_read_tgid_x: 805 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 806 AMDGPU::T1_X, VT); 807 case Intrinsic::r600_read_tgid_y: 808 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 809 AMDGPU::T1_Y, VT); 810 case Intrinsic::r600_read_tgid_z: 811 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 812 AMDGPU::T1_Z, VT); 813 case Intrinsic::r600_read_tidig_x: 814 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 815 AMDGPU::T0_X, VT); 816 case Intrinsic::r600_read_tidig_y: 817 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 818 AMDGPU::T0_Y, VT); 819 case Intrinsic::r600_read_tidig_z: 820 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 821 AMDGPU::T0_Z, VT); 822 823 // FIXME: Should be renamed to r600 prefix 824 case AMDGPUIntrinsic::AMDGPU_rsq_clamped: 825 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1)); 826 827 case Intrinsic::r600_rsq: 828 case AMDGPUIntrinsic::AMDGPU_rsq: // Legacy name 829 // XXX - I'm assuming SI's RSQ_LEGACY matches R600's behavior. 830 return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 831 } 832 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode()) 833 break; 834 } 835 } // end switch(Op.getOpcode()) 836 return SDValue(); 837 } 838 839 void R600TargetLowering::ReplaceNodeResults(SDNode *N, 840 SmallVectorImpl<SDValue> &Results, 841 SelectionDAG &DAG) const { 842 switch (N->getOpcode()) { 843 default: 844 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG); 845 return; 846 case ISD::FP_TO_UINT: 847 if (N->getValueType(0) == MVT::i1) { 848 Results.push_back(LowerFPTOUINT(N->getOperand(0), DAG)); 849 return; 850 } 851 // Fall-through. Since we don't care about out of bounds values 852 // we can use FP_TO_SINT for uints too. The DAGLegalizer code for uint 853 // considers some extra cases which are not necessary here. 854 case ISD::FP_TO_SINT: { 855 SDValue Result; 856 if (expandFP_TO_SINT(N, Result, DAG)) 857 Results.push_back(Result); 858 return; 859 } 860 case ISD::SDIVREM: { 861 SDValue Op = SDValue(N, 1); 862 SDValue RES = LowerSDIVREM(Op, DAG); 863 Results.push_back(RES); 864 Results.push_back(RES.getValue(1)); 865 break; 866 } 867 case ISD::UDIVREM: { 868 SDValue Op = SDValue(N, 0); 869 LowerUDIVREM64(Op, DAG, Results); 870 break; 871 } 872 } 873 } 874 875 SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG, 876 SDValue Vector) const { 877 878 SDLoc DL(Vector); 879 EVT VecVT = Vector.getValueType(); 880 EVT EltVT = VecVT.getVectorElementType(); 881 SmallVector<SDValue, 8> Args; 882 883 for (unsigned i = 0, e = VecVT.getVectorNumElements(); 884 i != e; ++i) { 885 Args.push_back(DAG.getNode( 886 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector, 887 DAG.getConstant(i, DL, getVectorIdxTy(DAG.getDataLayout())))); 888 } 889 890 return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args); 891 } 892 893 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 894 SelectionDAG &DAG) const { 895 896 SDLoc DL(Op); 897 SDValue Vector = Op.getOperand(0); 898 SDValue Index = Op.getOperand(1); 899 900 if (isa<ConstantSDNode>(Index) || 901 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 902 return Op; 903 904 Vector = vectorToVerticalVector(DAG, Vector); 905 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), 906 Vector, Index); 907 } 908 909 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 910 SelectionDAG &DAG) const { 911 SDLoc DL(Op); 912 SDValue Vector = Op.getOperand(0); 913 SDValue Value = Op.getOperand(1); 914 SDValue Index = Op.getOperand(2); 915 916 if (isa<ConstantSDNode>(Index) || 917 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 918 return Op; 919 920 Vector = vectorToVerticalVector(DAG, Vector); 921 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(), 922 Vector, Value, Index); 923 return vectorToVerticalVector(DAG, Insert); 924 } 925 926 SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, 927 SDValue Op, 928 SelectionDAG &DAG) const { 929 930 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op); 931 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS) 932 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG); 933 934 const DataLayout &DL = DAG.getDataLayout(); 935 const GlobalValue *GV = GSD->getGlobal(); 936 MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); 937 938 SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT); 939 return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA); 940 } 941 942 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 943 // On hw >= R700, COS/SIN input must be between -1. and 1. 944 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) 945 EVT VT = Op.getValueType(); 946 SDValue Arg = Op.getOperand(0); 947 SDLoc DL(Op); 948 949 // TODO: Should this propagate fast-math-flags? 950 SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT, 951 DAG.getNode(ISD::FADD, DL, VT, 952 DAG.getNode(ISD::FMUL, DL, VT, Arg, 953 DAG.getConstantFP(0.15915494309, DL, MVT::f32)), 954 DAG.getConstantFP(0.5, DL, MVT::f32))); 955 unsigned TrigNode; 956 switch (Op.getOpcode()) { 957 case ISD::FCOS: 958 TrigNode = AMDGPUISD::COS_HW; 959 break; 960 case ISD::FSIN: 961 TrigNode = AMDGPUISD::SIN_HW; 962 break; 963 default: 964 llvm_unreachable("Wrong trig opcode"); 965 } 966 SDValue TrigVal = DAG.getNode(TrigNode, DL, VT, 967 DAG.getNode(ISD::FADD, DL, VT, FractPart, 968 DAG.getConstantFP(-0.5, DL, MVT::f32))); 969 if (Gen >= AMDGPUSubtarget::R700) 970 return TrigVal; 971 // On R600 hw, COS/SIN input must be between -Pi and Pi. 972 return DAG.getNode(ISD::FMUL, DL, VT, TrigVal, 973 DAG.getConstantFP(3.14159265359, DL, MVT::f32)); 974 } 975 976 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const { 977 SDLoc DL(Op); 978 EVT VT = Op.getValueType(); 979 980 SDValue Lo = Op.getOperand(0); 981 SDValue Hi = Op.getOperand(1); 982 SDValue Shift = Op.getOperand(2); 983 SDValue Zero = DAG.getConstant(0, DL, VT); 984 SDValue One = DAG.getConstant(1, DL, VT); 985 986 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 987 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 988 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 989 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 990 991 // The dance around Width1 is necessary for 0 special case. 992 // Without it the CompShift might be 32, producing incorrect results in 993 // Overflow. So we do the shift in two steps, the alternative is to 994 // add a conditional to filter the special case. 995 996 SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift); 997 Overflow = DAG.getNode(ISD::SRL, DL, VT, Overflow, One); 998 999 SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift); 1000 HiSmall = DAG.getNode(ISD::OR, DL, VT, HiSmall, Overflow); 1001 SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift); 1002 1003 SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift); 1004 SDValue LoBig = Zero; 1005 1006 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1007 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1008 1009 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1010 } 1011 1012 SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const { 1013 SDLoc DL(Op); 1014 EVT VT = Op.getValueType(); 1015 1016 SDValue Lo = Op.getOperand(0); 1017 SDValue Hi = Op.getOperand(1); 1018 SDValue Shift = Op.getOperand(2); 1019 SDValue Zero = DAG.getConstant(0, DL, VT); 1020 SDValue One = DAG.getConstant(1, DL, VT); 1021 1022 const bool SRA = Op.getOpcode() == ISD::SRA_PARTS; 1023 1024 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 1025 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 1026 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1027 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1028 1029 // The dance around Width1 is necessary for 0 special case. 1030 // Without it the CompShift might be 32, producing incorrect results in 1031 // Overflow. So we do the shift in two steps, the alternative is to 1032 // add a conditional to filter the special case. 1033 1034 SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift); 1035 Overflow = DAG.getNode(ISD::SHL, DL, VT, Overflow, One); 1036 1037 SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift); 1038 SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift); 1039 LoSmall = DAG.getNode(ISD::OR, DL, VT, LoSmall, Overflow); 1040 1041 SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift); 1042 SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero; 1043 1044 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1045 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1046 1047 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1048 } 1049 1050 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG, 1051 unsigned mainop, unsigned ovf) const { 1052 SDLoc DL(Op); 1053 EVT VT = Op.getValueType(); 1054 1055 SDValue Lo = Op.getOperand(0); 1056 SDValue Hi = Op.getOperand(1); 1057 1058 SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi); 1059 // Extend sign. 1060 OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF, 1061 DAG.getValueType(MVT::i1)); 1062 1063 SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi); 1064 1065 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF); 1066 } 1067 1068 SDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const { 1069 SDLoc DL(Op); 1070 return DAG.getNode( 1071 ISD::SETCC, 1072 DL, 1073 MVT::i1, 1074 Op, DAG.getConstantFP(0.0f, DL, MVT::f32), 1075 DAG.getCondCode(ISD::SETNE) 1076 ); 1077 } 1078 1079 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT, 1080 const SDLoc &DL, 1081 unsigned DwordOffset) const { 1082 unsigned ByteOffset = DwordOffset * 4; 1083 PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1084 AMDGPUAS::CONSTANT_BUFFER_0); 1085 1086 // We shouldn't be using an offset wider than 16-bits for implicit parameters. 1087 assert(isInt<16>(ByteOffset)); 1088 1089 return DAG.getLoad(VT, DL, DAG.getEntryNode(), 1090 DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR 1091 MachinePointerInfo(ConstantPointerNull::get(PtrType)), 1092 false, false, false, 0); 1093 } 1094 1095 bool R600TargetLowering::isZero(SDValue Op) const { 1096 if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) { 1097 return Cst->isNullValue(); 1098 } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){ 1099 return CstFP->isZero(); 1100 } else { 1101 return false; 1102 } 1103 } 1104 1105 bool R600TargetLowering::isHWTrueValue(SDValue Op) const { 1106 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 1107 return CFP->isExactlyValue(1.0); 1108 } 1109 return isAllOnesConstant(Op); 1110 } 1111 1112 bool R600TargetLowering::isHWFalseValue(SDValue Op) const { 1113 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 1114 return CFP->getValueAPF().isZero(); 1115 } 1116 return isNullConstant(Op); 1117 } 1118 1119 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 1120 SDLoc DL(Op); 1121 EVT VT = Op.getValueType(); 1122 1123 SDValue LHS = Op.getOperand(0); 1124 SDValue RHS = Op.getOperand(1); 1125 SDValue True = Op.getOperand(2); 1126 SDValue False = Op.getOperand(3); 1127 SDValue CC = Op.getOperand(4); 1128 SDValue Temp; 1129 1130 if (VT == MVT::f32) { 1131 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); 1132 SDValue MinMax = CombineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI); 1133 if (MinMax) 1134 return MinMax; 1135 } 1136 1137 // LHS and RHS are guaranteed to be the same value type 1138 EVT CompareVT = LHS.getValueType(); 1139 1140 // Check if we can lower this to a native operation. 1141 1142 // Try to lower to a SET* instruction: 1143 // 1144 // SET* can match the following patterns: 1145 // 1146 // select_cc f32, f32, -1, 0, cc_supported 1147 // select_cc f32, f32, 1.0f, 0.0f, cc_supported 1148 // select_cc i32, i32, -1, 0, cc_supported 1149 // 1150 1151 // Move hardware True/False values to the correct operand. 1152 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1153 ISD::CondCode InverseCC = 1154 ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1155 if (isHWTrueValue(False) && isHWFalseValue(True)) { 1156 if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { 1157 std::swap(False, True); 1158 CC = DAG.getCondCode(InverseCC); 1159 } else { 1160 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC); 1161 if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) { 1162 std::swap(False, True); 1163 std::swap(LHS, RHS); 1164 CC = DAG.getCondCode(SwapInvCC); 1165 } 1166 } 1167 } 1168 1169 if (isHWTrueValue(True) && isHWFalseValue(False) && 1170 (CompareVT == VT || VT == MVT::i32)) { 1171 // This can be matched by a SET* instruction. 1172 return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC); 1173 } 1174 1175 // Try to lower to a CND* instruction: 1176 // 1177 // CND* can match the following patterns: 1178 // 1179 // select_cc f32, 0.0, f32, f32, cc_supported 1180 // select_cc f32, 0.0, i32, i32, cc_supported 1181 // select_cc i32, 0, f32, f32, cc_supported 1182 // select_cc i32, 0, i32, i32, cc_supported 1183 // 1184 1185 // Try to move the zero value to the RHS 1186 if (isZero(LHS)) { 1187 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1188 // Try swapping the operands 1189 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode); 1190 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1191 std::swap(LHS, RHS); 1192 CC = DAG.getCondCode(CCSwapped); 1193 } else { 1194 // Try inverting the conditon and then swapping the operands 1195 ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT.isInteger()); 1196 CCSwapped = ISD::getSetCCSwappedOperands(CCInv); 1197 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1198 std::swap(True, False); 1199 std::swap(LHS, RHS); 1200 CC = DAG.getCondCode(CCSwapped); 1201 } 1202 } 1203 } 1204 if (isZero(RHS)) { 1205 SDValue Cond = LHS; 1206 SDValue Zero = RHS; 1207 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1208 if (CompareVT != VT) { 1209 // Bitcast True / False to the correct types. This will end up being 1210 // a nop, but it allows us to define only a single pattern in the 1211 // .TD files for each CND* instruction rather than having to have 1212 // one pattern for integer True/False and one for fp True/False 1213 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True); 1214 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False); 1215 } 1216 1217 switch (CCOpcode) { 1218 case ISD::SETONE: 1219 case ISD::SETUNE: 1220 case ISD::SETNE: 1221 CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1222 Temp = True; 1223 True = False; 1224 False = Temp; 1225 break; 1226 default: 1227 break; 1228 } 1229 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, 1230 Cond, Zero, 1231 True, False, 1232 DAG.getCondCode(CCOpcode)); 1233 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode); 1234 } 1235 1236 // If we make it this for it means we have no native instructions to handle 1237 // this SELECT_CC, so we must lower it. 1238 SDValue HWTrue, HWFalse; 1239 1240 if (CompareVT == MVT::f32) { 1241 HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT); 1242 HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT); 1243 } else if (CompareVT == MVT::i32) { 1244 HWTrue = DAG.getConstant(-1, DL, CompareVT); 1245 HWFalse = DAG.getConstant(0, DL, CompareVT); 1246 } 1247 else { 1248 llvm_unreachable("Unhandled value type in LowerSELECT_CC"); 1249 } 1250 1251 // Lower this unsupported SELECT_CC into a combination of two supported 1252 // SELECT_CC operations. 1253 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC); 1254 1255 return DAG.getNode(ISD::SELECT_CC, DL, VT, 1256 Cond, HWFalse, 1257 True, False, 1258 DAG.getCondCode(ISD::SETNE)); 1259 } 1260 1261 /// LLVM generates byte-addressed pointers. For indirect addressing, we need to 1262 /// convert these pointers to a register index. Each register holds 1263 /// 16 bytes, (4 x 32bit sub-register), but we need to take into account the 1264 /// \p StackWidth, which tells us how many of the 4 sub-registrers will be used 1265 /// for indirect addressing. 1266 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr, 1267 unsigned StackWidth, 1268 SelectionDAG &DAG) const { 1269 unsigned SRLPad; 1270 switch(StackWidth) { 1271 case 1: 1272 SRLPad = 2; 1273 break; 1274 case 2: 1275 SRLPad = 3; 1276 break; 1277 case 4: 1278 SRLPad = 4; 1279 break; 1280 default: llvm_unreachable("Invalid stack width"); 1281 } 1282 1283 SDLoc DL(Ptr); 1284 return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr, 1285 DAG.getConstant(SRLPad, DL, MVT::i32)); 1286 } 1287 1288 void R600TargetLowering::getStackAddress(unsigned StackWidth, 1289 unsigned ElemIdx, 1290 unsigned &Channel, 1291 unsigned &PtrIncr) const { 1292 switch (StackWidth) { 1293 default: 1294 case 1: 1295 Channel = 0; 1296 if (ElemIdx > 0) { 1297 PtrIncr = 1; 1298 } else { 1299 PtrIncr = 0; 1300 } 1301 break; 1302 case 2: 1303 Channel = ElemIdx % 2; 1304 if (ElemIdx == 2) { 1305 PtrIncr = 1; 1306 } else { 1307 PtrIncr = 0; 1308 } 1309 break; 1310 case 4: 1311 Channel = ElemIdx; 1312 PtrIncr = 0; 1313 break; 1314 } 1315 } 1316 1317 SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store, 1318 SelectionDAG &DAG) const { 1319 SDLoc DL(Store); 1320 1321 unsigned Mask = 0; 1322 if (Store->getMemoryVT() == MVT::i8) { 1323 Mask = 0xff; 1324 } else if (Store->getMemoryVT() == MVT::i16) { 1325 Mask = 0xffff; 1326 } 1327 1328 SDValue Chain = Store->getChain(); 1329 SDValue BasePtr = Store->getBasePtr(); 1330 EVT MemVT = Store->getMemoryVT(); 1331 1332 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr, 1333 DAG.getConstant(2, DL, MVT::i32)); 1334 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32, 1335 Chain, Ptr, 1336 DAG.getTargetConstant(0, DL, MVT::i32)); 1337 1338 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr, 1339 DAG.getConstant(0x3, DL, MVT::i32)); 1340 1341 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1342 DAG.getConstant(3, DL, MVT::i32)); 1343 1344 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, 1345 Store->getValue()); 1346 1347 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT); 1348 1349 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, 1350 MaskedValue, ShiftAmt); 1351 1352 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, 1353 DAG.getConstant(Mask, DL, MVT::i32), 1354 ShiftAmt); 1355 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask, 1356 DAG.getConstant(0xffffffff, DL, MVT::i32)); 1357 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); 1358 1359 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); 1360 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1361 Chain, Value, Ptr, 1362 DAG.getTargetConstant(0, DL, MVT::i32)); 1363 } 1364 1365 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1366 if (SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG)) 1367 return Result; 1368 1369 StoreSDNode *StoreNode = cast<StoreSDNode>(Op); 1370 unsigned AS = StoreNode->getAddressSpace(); 1371 SDValue Value = StoreNode->getValue(); 1372 EVT ValueVT = Value.getValueType(); 1373 1374 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) && 1375 ValueVT.isVector()) { 1376 return SplitVectorStore(Op, DAG); 1377 } 1378 1379 SDLoc DL(Op); 1380 SDValue Chain = StoreNode->getChain(); 1381 SDValue Ptr = StoreNode->getBasePtr(); 1382 1383 if (AS == AMDGPUAS::GLOBAL_ADDRESS) { 1384 if (StoreNode->isTruncatingStore()) { 1385 EVT VT = Value.getValueType(); 1386 assert(VT.bitsLE(MVT::i32)); 1387 EVT MemVT = StoreNode->getMemoryVT(); 1388 SDValue MaskConstant; 1389 if (MemVT == MVT::i8) { 1390 MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32); 1391 } else { 1392 assert(MemVT == MVT::i16); 1393 MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32); 1394 } 1395 SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr, 1396 DAG.getConstant(2, DL, MVT::i32)); 1397 SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr, 1398 DAG.getConstant(0x00000003, DL, VT)); 1399 SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant); 1400 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex, 1401 DAG.getConstant(3, DL, VT)); 1402 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, Shift); 1403 SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, Shift); 1404 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 1405 // vector instead. 1406 SDValue Src[4] = { 1407 ShiftedValue, 1408 DAG.getConstant(0, DL, MVT::i32), 1409 DAG.getConstant(0, DL, MVT::i32), 1410 Mask 1411 }; 1412 SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src); 1413 SDValue Args[3] = { Chain, Input, DWordAddr }; 1414 return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL, 1415 Op->getVTList(), Args, MemVT, 1416 StoreNode->getMemOperand()); 1417 } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && 1418 ValueVT.bitsGE(MVT::i32)) { 1419 // Convert pointer from byte address to dword address. 1420 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(), 1421 DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), 1422 Ptr, DAG.getConstant(2, DL, MVT::i32))); 1423 1424 if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) { 1425 llvm_unreachable("Truncated and indexed stores not supported yet"); 1426 } else { 1427 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1428 } 1429 return Chain; 1430 } 1431 } 1432 1433 if (AS != AMDGPUAS::PRIVATE_ADDRESS) 1434 return SDValue(); 1435 1436 EVT MemVT = StoreNode->getMemoryVT(); 1437 if (MemVT.bitsLT(MVT::i32)) 1438 return lowerPrivateTruncStore(StoreNode, DAG); 1439 1440 // Lowering for indirect addressing 1441 const MachineFunction &MF = DAG.getMachineFunction(); 1442 const AMDGPUFrameLowering *TFL = 1443 static_cast<const AMDGPUFrameLowering *>(Subtarget->getFrameLowering()); 1444 unsigned StackWidth = TFL->getStackWidth(MF); 1445 1446 Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1447 1448 if (ValueVT.isVector()) { 1449 unsigned NumElemVT = ValueVT.getVectorNumElements(); 1450 EVT ElemVT = ValueVT.getVectorElementType(); 1451 SmallVector<SDValue, 4> Stores(NumElemVT); 1452 1453 assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1454 "vector width in load"); 1455 1456 for (unsigned i = 0; i < NumElemVT; ++i) { 1457 unsigned Channel, PtrIncr; 1458 getStackAddress(StackWidth, i, Channel, PtrIncr); 1459 Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1460 DAG.getConstant(PtrIncr, DL, MVT::i32)); 1461 SDValue Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, 1462 Value, DAG.getConstant(i, DL, MVT::i32)); 1463 1464 Stores[i] = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1465 Chain, Elem, Ptr, 1466 DAG.getTargetConstant(Channel, DL, MVT::i32)); 1467 } 1468 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); 1469 } else { 1470 if (ValueVT == MVT::i8) { 1471 Value = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Value); 1472 } 1473 Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, Chain, Value, Ptr, 1474 DAG.getTargetConstant(0, DL, MVT::i32)); // Channel 1475 } 1476 1477 return Chain; 1478 } 1479 1480 // return (512 + (kc_bank << 12) 1481 static int 1482 ConstantAddressBlock(unsigned AddressSpace) { 1483 switch (AddressSpace) { 1484 case AMDGPUAS::CONSTANT_BUFFER_0: 1485 return 512; 1486 case AMDGPUAS::CONSTANT_BUFFER_1: 1487 return 512 + 4096; 1488 case AMDGPUAS::CONSTANT_BUFFER_2: 1489 return 512 + 4096 * 2; 1490 case AMDGPUAS::CONSTANT_BUFFER_3: 1491 return 512 + 4096 * 3; 1492 case AMDGPUAS::CONSTANT_BUFFER_4: 1493 return 512 + 4096 * 4; 1494 case AMDGPUAS::CONSTANT_BUFFER_5: 1495 return 512 + 4096 * 5; 1496 case AMDGPUAS::CONSTANT_BUFFER_6: 1497 return 512 + 4096 * 6; 1498 case AMDGPUAS::CONSTANT_BUFFER_7: 1499 return 512 + 4096 * 7; 1500 case AMDGPUAS::CONSTANT_BUFFER_8: 1501 return 512 + 4096 * 8; 1502 case AMDGPUAS::CONSTANT_BUFFER_9: 1503 return 512 + 4096 * 9; 1504 case AMDGPUAS::CONSTANT_BUFFER_10: 1505 return 512 + 4096 * 10; 1506 case AMDGPUAS::CONSTANT_BUFFER_11: 1507 return 512 + 4096 * 11; 1508 case AMDGPUAS::CONSTANT_BUFFER_12: 1509 return 512 + 4096 * 12; 1510 case AMDGPUAS::CONSTANT_BUFFER_13: 1511 return 512 + 4096 * 13; 1512 case AMDGPUAS::CONSTANT_BUFFER_14: 1513 return 512 + 4096 * 14; 1514 case AMDGPUAS::CONSTANT_BUFFER_15: 1515 return 512 + 4096 * 15; 1516 default: 1517 return -1; 1518 } 1519 } 1520 1521 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op, 1522 SelectionDAG &DAG) const { 1523 SDLoc DL(Op); 1524 LoadSDNode *Load = cast<LoadSDNode>(Op); 1525 ISD::LoadExtType ExtType = Load->getExtensionType(); 1526 EVT MemVT = Load->getMemoryVT(); 1527 1528 // <SI && AS=PRIVATE && EXTLOAD && size < 32bit, 1529 // register (2-)byte extract. 1530 1531 // Get Register holding the target. 1532 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(), 1533 DAG.getConstant(2, DL, MVT::i32)); 1534 // Load the Register. 1535 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(), 1536 Load->getChain(), 1537 Ptr, 1538 DAG.getTargetConstant(0, DL, MVT::i32), 1539 Op.getOperand(2)); 1540 1541 // Get offset within the register. 1542 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, 1543 Load->getBasePtr(), 1544 DAG.getConstant(0x3, DL, MVT::i32)); 1545 1546 // Bit offset of target byte (byteIdx * 8). 1547 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1548 DAG.getConstant(3, DL, MVT::i32)); 1549 1550 // Shift to the right. 1551 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt); 1552 1553 // Eliminate the upper bits by setting them to ... 1554 EVT MemEltVT = MemVT.getScalarType(); 1555 1556 // ... ones. 1557 if (ExtType == ISD::SEXTLOAD) { 1558 SDValue MemEltVTNode = DAG.getValueType(MemEltVT); 1559 1560 SDValue Ops[] = { 1561 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode), 1562 Load->getChain() 1563 }; 1564 1565 return DAG.getMergeValues(Ops, DL); 1566 } 1567 1568 // ... or zeros. 1569 SDValue Ops[] = { 1570 DAG.getZeroExtendInReg(Ret, DL, MemEltVT), 1571 Load->getChain() 1572 }; 1573 1574 return DAG.getMergeValues(Ops, DL); 1575 } 1576 1577 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 1578 LoadSDNode *LoadNode = cast<LoadSDNode>(Op); 1579 unsigned AS = LoadNode->getAddressSpace(); 1580 EVT MemVT = LoadNode->getMemoryVT(); 1581 ISD::LoadExtType ExtType = LoadNode->getExtensionType(); 1582 1583 if (AS == AMDGPUAS::PRIVATE_ADDRESS && 1584 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(MVT::i32)) { 1585 return lowerPrivateExtLoad(Op, DAG); 1586 } 1587 1588 SDLoc DL(Op); 1589 EVT VT = Op.getValueType(); 1590 SDValue Chain = LoadNode->getChain(); 1591 SDValue Ptr = LoadNode->getBasePtr(); 1592 1593 if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) { 1594 SDValue MergedValues[2] = { 1595 scalarizeVectorLoad(LoadNode, DAG), 1596 Chain 1597 }; 1598 return DAG.getMergeValues(MergedValues, DL); 1599 } 1600 1601 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); 1602 if (ConstantBlock > -1 && 1603 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || 1604 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { 1605 SDValue Result; 1606 if (isa<ConstantExpr>(LoadNode->getMemOperand()->getValue()) || 1607 isa<Constant>(LoadNode->getMemOperand()->getValue()) || 1608 isa<ConstantSDNode>(Ptr)) { 1609 SDValue Slots[4]; 1610 for (unsigned i = 0; i < 4; i++) { 1611 // We want Const position encoded with the following formula : 1612 // (((512 + (kc_bank << 12) + const_index) << 2) + chan) 1613 // const_index is Ptr computed by llvm using an alignment of 16. 1614 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and 1615 // then div by 4 at the ISel step 1616 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1617 DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32)); 1618 Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr); 1619 } 1620 EVT NewVT = MVT::v4i32; 1621 unsigned NumElements = 4; 1622 if (VT.isVector()) { 1623 NewVT = VT; 1624 NumElements = VT.getVectorNumElements(); 1625 } 1626 Result = DAG.getBuildVector(NewVT, DL, makeArrayRef(Slots, NumElements)); 1627 } else { 1628 // non-constant ptr can't be folded, keeps it as a v4f32 load 1629 Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32, 1630 DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 1631 DAG.getConstant(4, DL, MVT::i32)), 1632 DAG.getConstant(LoadNode->getAddressSpace() - 1633 AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32) 1634 ); 1635 } 1636 1637 if (!VT.isVector()) { 1638 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 1639 DAG.getConstant(0, DL, MVT::i32)); 1640 } 1641 1642 SDValue MergedValues[2] = { 1643 Result, 1644 Chain 1645 }; 1646 return DAG.getMergeValues(MergedValues, DL); 1647 } 1648 1649 SDValue LoweredLoad; 1650 1651 // For most operations returning SDValue() will result in the node being 1652 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we 1653 // need to manually expand loads that may be legal in some address spaces and 1654 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for 1655 // compute shaders, since the data is sign extended when it is uploaded to the 1656 // buffer. However SEXT loads from other address spaces are not supported, so 1657 // we need to expand them here. 1658 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { 1659 EVT MemVT = LoadNode->getMemoryVT(); 1660 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); 1661 SDValue NewLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, Chain, Ptr, 1662 LoadNode->getPointerInfo(), MemVT, 1663 LoadNode->isVolatile(), 1664 LoadNode->isNonTemporal(), 1665 LoadNode->isInvariant(), 1666 LoadNode->getAlignment()); 1667 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad, 1668 DAG.getValueType(MemVT)); 1669 1670 SDValue MergedValues[2] = { Res, Chain }; 1671 return DAG.getMergeValues(MergedValues, DL); 1672 } 1673 1674 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1675 return SDValue(); 1676 } 1677 1678 // Lowering for indirect addressing 1679 const MachineFunction &MF = DAG.getMachineFunction(); 1680 const AMDGPUFrameLowering *TFL = 1681 static_cast<const AMDGPUFrameLowering *>(Subtarget->getFrameLowering()); 1682 unsigned StackWidth = TFL->getStackWidth(MF); 1683 1684 Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1685 1686 if (VT.isVector()) { 1687 unsigned NumElemVT = VT.getVectorNumElements(); 1688 EVT ElemVT = VT.getVectorElementType(); 1689 SDValue Loads[4]; 1690 1691 assert(NumElemVT <= 4); 1692 assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1693 "vector width in load"); 1694 1695 for (unsigned i = 0; i < NumElemVT; ++i) { 1696 unsigned Channel, PtrIncr; 1697 getStackAddress(StackWidth, i, Channel, PtrIncr); 1698 Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1699 DAG.getConstant(PtrIncr, DL, MVT::i32)); 1700 Loads[i] = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, ElemVT, 1701 Chain, Ptr, 1702 DAG.getTargetConstant(Channel, DL, MVT::i32), 1703 Op.getOperand(2)); 1704 } 1705 EVT TargetVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElemVT); 1706 LoweredLoad = DAG.getBuildVector(TargetVT, DL, makeArrayRef(Loads, NumElemVT)); 1707 } else { 1708 LoweredLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, VT, 1709 Chain, Ptr, 1710 DAG.getTargetConstant(0, DL, MVT::i32), // Channel 1711 Op.getOperand(2)); 1712 } 1713 1714 SDValue Ops[2] = { 1715 LoweredLoad, 1716 Chain 1717 }; 1718 1719 return DAG.getMergeValues(Ops, DL); 1720 } 1721 1722 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 1723 SDValue Chain = Op.getOperand(0); 1724 SDValue Cond = Op.getOperand(1); 1725 SDValue Jump = Op.getOperand(2); 1726 1727 return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(), 1728 Chain, Jump, Cond); 1729 } 1730 1731 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op, 1732 SelectionDAG &DAG) const { 1733 MachineFunction &MF = DAG.getMachineFunction(); 1734 const AMDGPUFrameLowering *TFL = Subtarget->getFrameLowering(); 1735 1736 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op); 1737 1738 unsigned FrameIndex = FIN->getIndex(); 1739 unsigned IgnoredFrameReg; 1740 unsigned Offset = 1741 TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg); 1742 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), SDLoc(Op), 1743 Op.getValueType()); 1744 } 1745 1746 /// XXX Only kernel functions are supported, so we can assume for now that 1747 /// every function is a kernel function, but in the future we should use 1748 /// separate calling conventions for kernel and non-kernel functions. 1749 SDValue R600TargetLowering::LowerFormalArguments( 1750 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 1751 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1752 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1753 SmallVector<CCValAssign, 16> ArgLocs; 1754 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 1755 *DAG.getContext()); 1756 MachineFunction &MF = DAG.getMachineFunction(); 1757 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 1758 1759 SmallVector<ISD::InputArg, 8> LocalIns; 1760 1761 getOriginalFunctionArgs(DAG, MF.getFunction(), Ins, LocalIns); 1762 1763 AnalyzeFormalArguments(CCInfo, LocalIns); 1764 1765 for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 1766 CCValAssign &VA = ArgLocs[i]; 1767 const ISD::InputArg &In = Ins[i]; 1768 EVT VT = In.VT; 1769 EVT MemVT = VA.getLocVT(); 1770 if (!VT.isVector() && MemVT.isVector()) { 1771 // Get load source type if scalarized. 1772 MemVT = MemVT.getVectorElementType(); 1773 } 1774 1775 if (AMDGPU::isShader(CallConv)) { 1776 unsigned Reg = MF.addLiveIn(VA.getLocReg(), &AMDGPU::R600_Reg128RegClass); 1777 SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT); 1778 InVals.push_back(Register); 1779 continue; 1780 } 1781 1782 PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1783 AMDGPUAS::CONSTANT_BUFFER_0); 1784 1785 // i64 isn't a legal type, so the register type used ends up as i32, which 1786 // isn't expected here. It attempts to create this sextload, but it ends up 1787 // being invalid. Somehow this seems to work with i64 arguments, but breaks 1788 // for <1 x i64>. 1789 1790 // The first 36 bytes of the input buffer contains information about 1791 // thread group and global sizes. 1792 ISD::LoadExtType Ext = ISD::NON_EXTLOAD; 1793 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) { 1794 // FIXME: This should really check the extload type, but the handling of 1795 // extload vector parameters seems to be broken. 1796 1797 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD; 1798 Ext = ISD::SEXTLOAD; 1799 } 1800 1801 // Compute the offset from the value. 1802 // XXX - I think PartOffset should give you this, but it seems to give the 1803 // size of the register which isn't useful. 1804 1805 unsigned ValBase = ArgLocs[In.getOrigArgIndex()].getLocMemOffset(); 1806 unsigned PartOffset = VA.getLocMemOffset(); 1807 unsigned Offset = 36 + VA.getLocMemOffset(); 1808 1809 MachinePointerInfo PtrInfo(UndefValue::get(PtrTy), PartOffset - ValBase); 1810 SDValue Arg = DAG.getLoad(ISD::UNINDEXED, Ext, VT, DL, Chain, 1811 DAG.getConstant(Offset, DL, MVT::i32), 1812 DAG.getUNDEF(MVT::i32), 1813 PtrInfo, 1814 MemVT, false, true, true, 4); 1815 1816 // 4 is the preferred alignment for the CONSTANT memory space. 1817 InVals.push_back(Arg); 1818 MFI->ABIArgOffset = Offset + MemVT.getStoreSize(); 1819 } 1820 return Chain; 1821 } 1822 1823 EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1824 EVT VT) const { 1825 if (!VT.isVector()) 1826 return MVT::i32; 1827 return VT.changeVectorElementTypeToInteger(); 1828 } 1829 1830 bool R600TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 1831 unsigned AddrSpace, 1832 unsigned Align, 1833 bool *IsFast) const { 1834 if (IsFast) 1835 *IsFast = false; 1836 1837 if (!VT.isSimple() || VT == MVT::Other) 1838 return false; 1839 1840 if (VT.bitsLT(MVT::i32)) 1841 return false; 1842 1843 // TODO: This is a rough estimate. 1844 if (IsFast) 1845 *IsFast = true; 1846 1847 return VT.bitsGT(MVT::i32) && Align % 4 == 0; 1848 } 1849 1850 static SDValue CompactSwizzlableVector( 1851 SelectionDAG &DAG, SDValue VectorEntry, 1852 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1853 assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 1854 assert(RemapSwizzle.empty()); 1855 SDValue NewBldVec[4] = { 1856 VectorEntry.getOperand(0), 1857 VectorEntry.getOperand(1), 1858 VectorEntry.getOperand(2), 1859 VectorEntry.getOperand(3) 1860 }; 1861 1862 for (unsigned i = 0; i < 4; i++) { 1863 if (NewBldVec[i].isUndef()) 1864 // We mask write here to teach later passes that the ith element of this 1865 // vector is undef. Thus we can use it to reduce 128 bits reg usage, 1866 // break false dependencies and additionnaly make assembly easier to read. 1867 RemapSwizzle[i] = 7; // SEL_MASK_WRITE 1868 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) { 1869 if (C->isZero()) { 1870 RemapSwizzle[i] = 4; // SEL_0 1871 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1872 } else if (C->isExactlyValue(1.0)) { 1873 RemapSwizzle[i] = 5; // SEL_1 1874 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1875 } 1876 } 1877 1878 if (NewBldVec[i].isUndef()) 1879 continue; 1880 for (unsigned j = 0; j < i; j++) { 1881 if (NewBldVec[i] == NewBldVec[j]) { 1882 NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); 1883 RemapSwizzle[i] = j; 1884 break; 1885 } 1886 } 1887 } 1888 1889 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1890 NewBldVec); 1891 } 1892 1893 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, 1894 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1895 assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 1896 assert(RemapSwizzle.empty()); 1897 SDValue NewBldVec[4] = { 1898 VectorEntry.getOperand(0), 1899 VectorEntry.getOperand(1), 1900 VectorEntry.getOperand(2), 1901 VectorEntry.getOperand(3) 1902 }; 1903 bool isUnmovable[4] = { false, false, false, false }; 1904 for (unsigned i = 0; i < 4; i++) { 1905 RemapSwizzle[i] = i; 1906 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1907 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1908 ->getZExtValue(); 1909 if (i == Idx) 1910 isUnmovable[Idx] = true; 1911 } 1912 } 1913 1914 for (unsigned i = 0; i < 4; i++) { 1915 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1916 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1917 ->getZExtValue(); 1918 if (isUnmovable[Idx]) 1919 continue; 1920 // Swap i and Idx 1921 std::swap(NewBldVec[Idx], NewBldVec[i]); 1922 std::swap(RemapSwizzle[i], RemapSwizzle[Idx]); 1923 break; 1924 } 1925 } 1926 1927 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1928 NewBldVec); 1929 } 1930 1931 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4], 1932 SelectionDAG &DAG, 1933 const SDLoc &DL) const { 1934 assert(BuildVector.getOpcode() == ISD::BUILD_VECTOR); 1935 // Old -> New swizzle values 1936 DenseMap<unsigned, unsigned> SwizzleRemap; 1937 1938 BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap); 1939 for (unsigned i = 0; i < 4; i++) { 1940 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1941 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1942 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1943 } 1944 1945 SwizzleRemap.clear(); 1946 BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap); 1947 for (unsigned i = 0; i < 4; i++) { 1948 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1949 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1950 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1951 } 1952 1953 return BuildVector; 1954 } 1955 1956 1957 //===----------------------------------------------------------------------===// 1958 // Custom DAG Optimizations 1959 //===----------------------------------------------------------------------===// 1960 1961 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N, 1962 DAGCombinerInfo &DCI) const { 1963 SelectionDAG &DAG = DCI.DAG; 1964 1965 switch (N->getOpcode()) { 1966 default: return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1967 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) 1968 case ISD::FP_ROUND: { 1969 SDValue Arg = N->getOperand(0); 1970 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) { 1971 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), N->getValueType(0), 1972 Arg.getOperand(0)); 1973 } 1974 break; 1975 } 1976 1977 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> 1978 // (i32 select_cc f32, f32, -1, 0 cc) 1979 // 1980 // Mesa's GLSL frontend generates the above pattern a lot and we can lower 1981 // this to one of the SET*_DX10 instructions. 1982 case ISD::FP_TO_SINT: { 1983 SDValue FNeg = N->getOperand(0); 1984 if (FNeg.getOpcode() != ISD::FNEG) { 1985 return SDValue(); 1986 } 1987 SDValue SelectCC = FNeg.getOperand(0); 1988 if (SelectCC.getOpcode() != ISD::SELECT_CC || 1989 SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS 1990 SelectCC.getOperand(2).getValueType() != MVT::f32 || // True 1991 !isHWTrueValue(SelectCC.getOperand(2)) || 1992 !isHWFalseValue(SelectCC.getOperand(3))) { 1993 return SDValue(); 1994 } 1995 1996 SDLoc dl(N); 1997 return DAG.getNode(ISD::SELECT_CC, dl, N->getValueType(0), 1998 SelectCC.getOperand(0), // LHS 1999 SelectCC.getOperand(1), // RHS 2000 DAG.getConstant(-1, dl, MVT::i32), // True 2001 DAG.getConstant(0, dl, MVT::i32), // False 2002 SelectCC.getOperand(4)); // CC 2003 2004 break; 2005 } 2006 2007 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx 2008 // => build_vector elt0, ... , NewEltIdx, ... , eltN 2009 case ISD::INSERT_VECTOR_ELT: { 2010 SDValue InVec = N->getOperand(0); 2011 SDValue InVal = N->getOperand(1); 2012 SDValue EltNo = N->getOperand(2); 2013 SDLoc dl(N); 2014 2015 // If the inserted element is an UNDEF, just use the input vector. 2016 if (InVal.isUndef()) 2017 return InVec; 2018 2019 EVT VT = InVec.getValueType(); 2020 2021 // If we can't generate a legal BUILD_VECTOR, exit 2022 if (!isOperationLegal(ISD::BUILD_VECTOR, VT)) 2023 return SDValue(); 2024 2025 // Check that we know which element is being inserted 2026 if (!isa<ConstantSDNode>(EltNo)) 2027 return SDValue(); 2028 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 2029 2030 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 2031 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 2032 // vector elements. 2033 SmallVector<SDValue, 8> Ops; 2034 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 2035 Ops.append(InVec.getNode()->op_begin(), 2036 InVec.getNode()->op_end()); 2037 } else if (InVec.isUndef()) { 2038 unsigned NElts = VT.getVectorNumElements(); 2039 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 2040 } else { 2041 return SDValue(); 2042 } 2043 2044 // Insert the element 2045 if (Elt < Ops.size()) { 2046 // All the operands of BUILD_VECTOR must have the same type; 2047 // we enforce that here. 2048 EVT OpVT = Ops[0].getValueType(); 2049 if (InVal.getValueType() != OpVT) 2050 InVal = OpVT.bitsGT(InVal.getValueType()) ? 2051 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : 2052 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); 2053 Ops[Elt] = InVal; 2054 } 2055 2056 // Return the new vector 2057 return DAG.getBuildVector(VT, dl, Ops); 2058 } 2059 2060 // Extract_vec (Build_vector) generated by custom lowering 2061 // also needs to be customly combined 2062 case ISD::EXTRACT_VECTOR_ELT: { 2063 SDValue Arg = N->getOperand(0); 2064 if (Arg.getOpcode() == ISD::BUILD_VECTOR) { 2065 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2066 unsigned Element = Const->getZExtValue(); 2067 return Arg->getOperand(Element); 2068 } 2069 } 2070 if (Arg.getOpcode() == ISD::BITCAST && 2071 Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { 2072 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2073 unsigned Element = Const->getZExtValue(); 2074 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getVTList(), 2075 Arg->getOperand(0).getOperand(Element)); 2076 } 2077 } 2078 break; 2079 } 2080 2081 case ISD::SELECT_CC: { 2082 // Try common optimizations 2083 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI)) 2084 return Ret; 2085 2086 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> 2087 // selectcc x, y, a, b, inv(cc) 2088 // 2089 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> 2090 // selectcc x, y, a, b, cc 2091 SDValue LHS = N->getOperand(0); 2092 if (LHS.getOpcode() != ISD::SELECT_CC) { 2093 return SDValue(); 2094 } 2095 2096 SDValue RHS = N->getOperand(1); 2097 SDValue True = N->getOperand(2); 2098 SDValue False = N->getOperand(3); 2099 ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2100 2101 if (LHS.getOperand(2).getNode() != True.getNode() || 2102 LHS.getOperand(3).getNode() != False.getNode() || 2103 RHS.getNode() != False.getNode()) { 2104 return SDValue(); 2105 } 2106 2107 switch (NCC) { 2108 default: return SDValue(); 2109 case ISD::SETNE: return LHS; 2110 case ISD::SETEQ: { 2111 ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); 2112 LHSCC = ISD::getSetCCInverse(LHSCC, 2113 LHS.getOperand(0).getValueType().isInteger()); 2114 if (DCI.isBeforeLegalizeOps() || 2115 isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) 2116 return DAG.getSelectCC(SDLoc(N), 2117 LHS.getOperand(0), 2118 LHS.getOperand(1), 2119 LHS.getOperand(2), 2120 LHS.getOperand(3), 2121 LHSCC); 2122 break; 2123 } 2124 } 2125 return SDValue(); 2126 } 2127 2128 case AMDGPUISD::EXPORT: { 2129 SDValue Arg = N->getOperand(1); 2130 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2131 break; 2132 2133 SDValue NewArgs[8] = { 2134 N->getOperand(0), // Chain 2135 SDValue(), 2136 N->getOperand(2), // ArrayBase 2137 N->getOperand(3), // Type 2138 N->getOperand(4), // SWZ_X 2139 N->getOperand(5), // SWZ_Y 2140 N->getOperand(6), // SWZ_Z 2141 N->getOperand(7) // SWZ_W 2142 }; 2143 SDLoc DL(N); 2144 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL); 2145 return DAG.getNode(AMDGPUISD::EXPORT, DL, N->getVTList(), NewArgs); 2146 } 2147 case AMDGPUISD::TEXTURE_FETCH: { 2148 SDValue Arg = N->getOperand(1); 2149 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2150 break; 2151 2152 SDValue NewArgs[19] = { 2153 N->getOperand(0), 2154 N->getOperand(1), 2155 N->getOperand(2), 2156 N->getOperand(3), 2157 N->getOperand(4), 2158 N->getOperand(5), 2159 N->getOperand(6), 2160 N->getOperand(7), 2161 N->getOperand(8), 2162 N->getOperand(9), 2163 N->getOperand(10), 2164 N->getOperand(11), 2165 N->getOperand(12), 2166 N->getOperand(13), 2167 N->getOperand(14), 2168 N->getOperand(15), 2169 N->getOperand(16), 2170 N->getOperand(17), 2171 N->getOperand(18), 2172 }; 2173 SDLoc DL(N); 2174 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL); 2175 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs); 2176 } 2177 } 2178 2179 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 2180 } 2181 2182 static bool 2183 FoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg, 2184 SDValue &Abs, SDValue &Sel, SDValue &Imm, SelectionDAG &DAG) { 2185 const R600InstrInfo *TII = 2186 static_cast<const R600InstrInfo *>(DAG.getSubtarget().getInstrInfo()); 2187 if (!Src.isMachineOpcode()) 2188 return false; 2189 switch (Src.getMachineOpcode()) { 2190 case AMDGPU::FNEG_R600: 2191 if (!Neg.getNode()) 2192 return false; 2193 Src = Src.getOperand(0); 2194 Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2195 return true; 2196 case AMDGPU::FABS_R600: 2197 if (!Abs.getNode()) 2198 return false; 2199 Src = Src.getOperand(0); 2200 Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2201 return true; 2202 case AMDGPU::CONST_COPY: { 2203 unsigned Opcode = ParentNode->getMachineOpcode(); 2204 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2205 2206 if (!Sel.getNode()) 2207 return false; 2208 2209 SDValue CstOffset = Src.getOperand(0); 2210 if (ParentNode->getValueType(0).isVector()) 2211 return false; 2212 2213 // Gather constants values 2214 int SrcIndices[] = { 2215 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2216 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2217 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2), 2218 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2219 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2220 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2221 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2222 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2223 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2224 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2225 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2226 }; 2227 std::vector<unsigned> Consts; 2228 for (int OtherSrcIdx : SrcIndices) { 2229 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); 2230 if (OtherSrcIdx < 0 || OtherSelIdx < 0) 2231 continue; 2232 if (HasDst) { 2233 OtherSrcIdx--; 2234 OtherSelIdx--; 2235 } 2236 if (RegisterSDNode *Reg = 2237 dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { 2238 if (Reg->getReg() == AMDGPU::ALU_CONST) { 2239 ConstantSDNode *Cst 2240 = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx)); 2241 Consts.push_back(Cst->getZExtValue()); 2242 } 2243 } 2244 } 2245 2246 ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset); 2247 Consts.push_back(Cst->getZExtValue()); 2248 if (!TII->fitsConstReadLimitations(Consts)) { 2249 return false; 2250 } 2251 2252 Sel = CstOffset; 2253 Src = DAG.getRegister(AMDGPU::ALU_CONST, MVT::f32); 2254 return true; 2255 } 2256 case AMDGPU::MOV_IMM_GLOBAL_ADDR: 2257 // Check if the Imm slot is used. Taken from below. 2258 if (cast<ConstantSDNode>(Imm)->getZExtValue()) 2259 return false; 2260 Imm = Src.getOperand(0); 2261 Src = DAG.getRegister(AMDGPU::ALU_LITERAL_X, MVT::i32); 2262 return true; 2263 case AMDGPU::MOV_IMM_I32: 2264 case AMDGPU::MOV_IMM_F32: { 2265 unsigned ImmReg = AMDGPU::ALU_LITERAL_X; 2266 uint64_t ImmValue = 0; 2267 2268 2269 if (Src.getMachineOpcode() == AMDGPU::MOV_IMM_F32) { 2270 ConstantFPSDNode *FPC = dyn_cast<ConstantFPSDNode>(Src.getOperand(0)); 2271 float FloatValue = FPC->getValueAPF().convertToFloat(); 2272 if (FloatValue == 0.0) { 2273 ImmReg = AMDGPU::ZERO; 2274 } else if (FloatValue == 0.5) { 2275 ImmReg = AMDGPU::HALF; 2276 } else if (FloatValue == 1.0) { 2277 ImmReg = AMDGPU::ONE; 2278 } else { 2279 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); 2280 } 2281 } else { 2282 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src.getOperand(0)); 2283 uint64_t Value = C->getZExtValue(); 2284 if (Value == 0) { 2285 ImmReg = AMDGPU::ZERO; 2286 } else if (Value == 1) { 2287 ImmReg = AMDGPU::ONE_INT; 2288 } else { 2289 ImmValue = Value; 2290 } 2291 } 2292 2293 // Check that we aren't already using an immediate. 2294 // XXX: It's possible for an instruction to have more than one 2295 // immediate operand, but this is not supported yet. 2296 if (ImmReg == AMDGPU::ALU_LITERAL_X) { 2297 if (!Imm.getNode()) 2298 return false; 2299 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Imm); 2300 assert(C); 2301 if (C->getZExtValue()) 2302 return false; 2303 Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32); 2304 } 2305 Src = DAG.getRegister(ImmReg, MVT::i32); 2306 return true; 2307 } 2308 default: 2309 return false; 2310 } 2311 } 2312 2313 2314 /// \brief Fold the instructions after selecting them 2315 SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node, 2316 SelectionDAG &DAG) const { 2317 const R600InstrInfo *TII = 2318 static_cast<const R600InstrInfo *>(DAG.getSubtarget().getInstrInfo()); 2319 if (!Node->isMachineOpcode()) 2320 return Node; 2321 unsigned Opcode = Node->getMachineOpcode(); 2322 SDValue FakeOp; 2323 2324 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end()); 2325 2326 if (Opcode == AMDGPU::DOT_4) { 2327 int OperandIdx[] = { 2328 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2329 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2330 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2331 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2332 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2333 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2334 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2335 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2336 }; 2337 int NegIdx[] = { 2338 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_X), 2339 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Y), 2340 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Z), 2341 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_W), 2342 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_X), 2343 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Y), 2344 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Z), 2345 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_W) 2346 }; 2347 int AbsIdx[] = { 2348 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_X), 2349 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Y), 2350 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Z), 2351 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_W), 2352 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_X), 2353 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Y), 2354 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Z), 2355 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_W) 2356 }; 2357 for (unsigned i = 0; i < 8; i++) { 2358 if (OperandIdx[i] < 0) 2359 return Node; 2360 SDValue &Src = Ops[OperandIdx[i] - 1]; 2361 SDValue &Neg = Ops[NegIdx[i] - 1]; 2362 SDValue &Abs = Ops[AbsIdx[i] - 1]; 2363 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2364 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2365 if (HasDst) 2366 SelIdx--; 2367 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2368 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG)) 2369 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2370 } 2371 } else if (Opcode == AMDGPU::REG_SEQUENCE) { 2372 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { 2373 SDValue &Src = Ops[i]; 2374 if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG)) 2375 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2376 } 2377 } else if (Opcode == AMDGPU::CLAMP_R600) { 2378 SDValue Src = Node->getOperand(0); 2379 if (!Src.isMachineOpcode() || 2380 !TII->hasInstrModifiers(Src.getMachineOpcode())) 2381 return Node; 2382 int ClampIdx = TII->getOperandIdx(Src.getMachineOpcode(), 2383 AMDGPU::OpName::clamp); 2384 if (ClampIdx < 0) 2385 return Node; 2386 SDLoc DL(Node); 2387 std::vector<SDValue> Ops(Src->op_begin(), Src->op_end()); 2388 Ops[ClampIdx - 1] = DAG.getTargetConstant(1, DL, MVT::i32); 2389 return DAG.getMachineNode(Src.getMachineOpcode(), DL, 2390 Node->getVTList(), Ops); 2391 } else { 2392 if (!TII->hasInstrModifiers(Opcode)) 2393 return Node; 2394 int OperandIdx[] = { 2395 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2396 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2397 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2) 2398 }; 2399 int NegIdx[] = { 2400 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg), 2401 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg), 2402 TII->getOperandIdx(Opcode, AMDGPU::OpName::src2_neg) 2403 }; 2404 int AbsIdx[] = { 2405 TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs), 2406 TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs), 2407 -1 2408 }; 2409 for (unsigned i = 0; i < 3; i++) { 2410 if (OperandIdx[i] < 0) 2411 return Node; 2412 SDValue &Src = Ops[OperandIdx[i] - 1]; 2413 SDValue &Neg = Ops[NegIdx[i] - 1]; 2414 SDValue FakeAbs; 2415 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; 2416 bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2417 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2418 int ImmIdx = TII->getOperandIdx(Opcode, AMDGPU::OpName::literal); 2419 if (HasDst) { 2420 SelIdx--; 2421 ImmIdx--; 2422 } 2423 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2424 SDValue &Imm = Ops[ImmIdx]; 2425 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG)) 2426 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2427 } 2428 } 2429 2430 return Node; 2431 } 2432