1//===-- AMDGPUInstructions.td - Common instruction defs ---*- tablegen -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file contains instruction defs that are common to all hw codegen 11// targets. 12// 13//===----------------------------------------------------------------------===// 14 15class AMDGPUInst <dag outs, dag ins, string asm = "", 16 list<dag> pattern = []> : Instruction { 17 field bit isRegisterLoad = 0; 18 field bit isRegisterStore = 0; 19 20 let Namespace = "AMDGPU"; 21 let OutOperandList = outs; 22 let InOperandList = ins; 23 let AsmString = asm; 24 let Pattern = pattern; 25 let Itinerary = NullALU; 26 27 // SoftFail is a field the disassembler can use to provide a way for 28 // instructions to not match without killing the whole decode process. It is 29 // mainly used for ARM, but Tablegen expects this field to exist or it fails 30 // to build the decode table. 31 field bits<64> SoftFail = 0; 32 33 let DecoderNamespace = Namespace; 34 35 let TSFlags{63} = isRegisterLoad; 36 let TSFlags{62} = isRegisterStore; 37} 38 39class AMDGPUShaderInst <dag outs, dag ins, string asm = "", 40 list<dag> pattern = []> : AMDGPUInst<outs, ins, asm, pattern> { 41 42 field bits<32> Inst = 0xffffffff; 43} 44 45//===---------------------------------------------------------------------===// 46// Return instruction 47//===---------------------------------------------------------------------===// 48 49class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern> 50: Instruction { 51 52 let Namespace = "AMDGPU"; 53 dag OutOperandList = outs; 54 dag InOperandList = ins; 55 let Pattern = pattern; 56 let AsmString = !strconcat(asmstr, "\n"); 57 let isPseudo = 1; 58 let Itinerary = NullALU; 59 bit hasIEEEFlag = 0; 60 bit hasZeroOpFlag = 0; 61 let mayLoad = 0; 62 let mayStore = 0; 63 let hasSideEffects = 0; 64 let isCodeGenOnly = 1; 65} 66 67def TruePredicate : Predicate<"true">; 68 69// Exists to help track down where SubtargetPredicate isn't set rather 70// than letting tablegen crash with an unhelpful error. 71def InvalidPred : Predicate<"predicate not set on instruction or pattern">; 72 73class PredicateControl { 74 Predicate SubtargetPredicate = InvalidPred; 75 list<Predicate> AssemblerPredicates = []; 76 Predicate AssemblerPredicate = TruePredicate; 77 list<Predicate> OtherPredicates = []; 78 list<Predicate> Predicates = !listconcat([SubtargetPredicate, 79 AssemblerPredicate], 80 AssemblerPredicates, 81 OtherPredicates); 82} 83class AMDGPUPat<dag pattern, dag result> : Pat<pattern, result>, 84 PredicateControl; 85 86def FP16Denormals : Predicate<"Subtarget->hasFP16Denormals()">; 87def FP32Denormals : Predicate<"Subtarget->hasFP32Denormals()">; 88def FP64Denormals : Predicate<"Subtarget->hasFP64Denormals()">; 89def NoFP16Denormals : Predicate<"!Subtarget->hasFP16Denormals()">; 90def NoFP32Denormals : Predicate<"!Subtarget->hasFP32Denormals()">; 91def NoFP64Denormals : Predicate<"!Subtarget->hasFP64Denormals()">; 92def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">; 93def FMA : Predicate<"Subtarget->hasFMA()">; 94 95def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>; 96 97def u16ImmTarget : AsmOperandClass { 98 let Name = "U16Imm"; 99 let RenderMethod = "addImmOperands"; 100} 101 102def s16ImmTarget : AsmOperandClass { 103 let Name = "S16Imm"; 104 let RenderMethod = "addImmOperands"; 105} 106 107let OperandType = "OPERAND_IMMEDIATE" in { 108 109def u32imm : Operand<i32> { 110 let PrintMethod = "printU32ImmOperand"; 111} 112 113def u16imm : Operand<i16> { 114 let PrintMethod = "printU16ImmOperand"; 115 let ParserMatchClass = u16ImmTarget; 116} 117 118def s16imm : Operand<i16> { 119 let PrintMethod = "printU16ImmOperand"; 120 let ParserMatchClass = s16ImmTarget; 121} 122 123def u8imm : Operand<i8> { 124 let PrintMethod = "printU8ImmOperand"; 125} 126 127} // End OperandType = "OPERAND_IMMEDIATE" 128 129//===--------------------------------------------------------------------===// 130// Custom Operands 131//===--------------------------------------------------------------------===// 132def brtarget : Operand<OtherVT>; 133 134//===----------------------------------------------------------------------===// 135// Misc. PatFrags 136//===----------------------------------------------------------------------===// 137 138class HasOneUseUnaryOp<SDPatternOperator op> : PatFrag< 139 (ops node:$src0), 140 (op $src0), 141 [{ return N->hasOneUse(); }] 142>; 143 144class HasOneUseBinOp<SDPatternOperator op> : PatFrag< 145 (ops node:$src0, node:$src1), 146 (op $src0, $src1), 147 [{ return N->hasOneUse(); }] 148>; 149 150class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag< 151 (ops node:$src0, node:$src1, node:$src2), 152 (op $src0, $src1, $src2), 153 [{ return N->hasOneUse(); }] 154>; 155 156let Properties = [SDNPCommutative, SDNPAssociative] in { 157def smax_oneuse : HasOneUseBinOp<smax>; 158def smin_oneuse : HasOneUseBinOp<smin>; 159def umax_oneuse : HasOneUseBinOp<umax>; 160def umin_oneuse : HasOneUseBinOp<umin>; 161 162def fminnum_oneuse : HasOneUseBinOp<fminnum>; 163def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>; 164 165def fminnum_ieee_oneuse : HasOneUseBinOp<fminnum_ieee>; 166def fmaxnum_ieee_oneuse : HasOneUseBinOp<fmaxnum_ieee>; 167 168 169def and_oneuse : HasOneUseBinOp<and>; 170def or_oneuse : HasOneUseBinOp<or>; 171def xor_oneuse : HasOneUseBinOp<xor>; 172} // Properties = [SDNPCommutative, SDNPAssociative] 173 174def not_oneuse : HasOneUseUnaryOp<not>; 175 176def add_oneuse : HasOneUseBinOp<add>; 177def sub_oneuse : HasOneUseBinOp<sub>; 178 179def srl_oneuse : HasOneUseBinOp<srl>; 180def shl_oneuse : HasOneUseBinOp<shl>; 181 182def select_oneuse : HasOneUseTernaryOp<select>; 183 184def AMDGPUmul_u24_oneuse : HasOneUseBinOp<AMDGPUmul_u24>; 185def AMDGPUmul_i24_oneuse : HasOneUseBinOp<AMDGPUmul_i24>; 186 187def srl_16 : PatFrag< 188 (ops node:$src0), (srl_oneuse node:$src0, (i32 16)) 189>; 190 191 192def hi_i16_elt : PatFrag< 193 (ops node:$src0), (i16 (trunc (i32 (srl_16 node:$src0)))) 194>; 195 196 197def hi_f16_elt : PatLeaf< 198 (vt), [{ 199 if (N->getOpcode() != ISD::BITCAST) 200 return false; 201 SDValue Tmp = N->getOperand(0); 202 203 if (Tmp.getOpcode() != ISD::SRL) 204 return false; 205 if (const auto *RHS = dyn_cast<ConstantSDNode>(Tmp.getOperand(1)) 206 return RHS->getZExtValue() == 16; 207 return false; 208}]>; 209 210//===----------------------------------------------------------------------===// 211// PatLeafs for floating-point comparisons 212//===----------------------------------------------------------------------===// 213 214def COND_OEQ : PatLeaf < 215 (cond), 216 [{return N->get() == ISD::SETOEQ || N->get() == ISD::SETEQ;}] 217>; 218 219def COND_ONE : PatLeaf < 220 (cond), 221 [{return N->get() == ISD::SETONE || N->get() == ISD::SETNE;}] 222>; 223 224def COND_OGT : PatLeaf < 225 (cond), 226 [{return N->get() == ISD::SETOGT || N->get() == ISD::SETGT;}] 227>; 228 229def COND_OGE : PatLeaf < 230 (cond), 231 [{return N->get() == ISD::SETOGE || N->get() == ISD::SETGE;}] 232>; 233 234def COND_OLT : PatLeaf < 235 (cond), 236 [{return N->get() == ISD::SETOLT || N->get() == ISD::SETLT;}] 237>; 238 239def COND_OLE : PatLeaf < 240 (cond), 241 [{return N->get() == ISD::SETOLE || N->get() == ISD::SETLE;}] 242>; 243 244def COND_O : PatLeaf <(cond), [{return N->get() == ISD::SETO;}]>; 245def COND_UO : PatLeaf <(cond), [{return N->get() == ISD::SETUO;}]>; 246 247//===----------------------------------------------------------------------===// 248// PatLeafs for unsigned / unordered comparisons 249//===----------------------------------------------------------------------===// 250 251def COND_UEQ : PatLeaf <(cond), [{return N->get() == ISD::SETUEQ;}]>; 252def COND_UNE : PatLeaf <(cond), [{return N->get() == ISD::SETUNE;}]>; 253def COND_UGT : PatLeaf <(cond), [{return N->get() == ISD::SETUGT;}]>; 254def COND_UGE : PatLeaf <(cond), [{return N->get() == ISD::SETUGE;}]>; 255def COND_ULT : PatLeaf <(cond), [{return N->get() == ISD::SETULT;}]>; 256def COND_ULE : PatLeaf <(cond), [{return N->get() == ISD::SETULE;}]>; 257 258// XXX - For some reason R600 version is preferring to use unordered 259// for setne? 260def COND_UNE_NE : PatLeaf < 261 (cond), 262 [{return N->get() == ISD::SETUNE || N->get() == ISD::SETNE;}] 263>; 264 265//===----------------------------------------------------------------------===// 266// PatLeafs for signed comparisons 267//===----------------------------------------------------------------------===// 268 269def COND_SGT : PatLeaf <(cond), [{return N->get() == ISD::SETGT;}]>; 270def COND_SGE : PatLeaf <(cond), [{return N->get() == ISD::SETGE;}]>; 271def COND_SLT : PatLeaf <(cond), [{return N->get() == ISD::SETLT;}]>; 272def COND_SLE : PatLeaf <(cond), [{return N->get() == ISD::SETLE;}]>; 273 274//===----------------------------------------------------------------------===// 275// PatLeafs for integer equality 276//===----------------------------------------------------------------------===// 277 278def COND_EQ : PatLeaf < 279 (cond), 280 [{return N->get() == ISD::SETEQ || N->get() == ISD::SETUEQ;}] 281>; 282 283def COND_NE : PatLeaf < 284 (cond), 285 [{return N->get() == ISD::SETNE || N->get() == ISD::SETUNE;}] 286>; 287 288def COND_NULL : PatLeaf < 289 (cond), 290 [{(void)N; return false;}] 291>; 292 293//===----------------------------------------------------------------------===// 294// PatLeafs for Texture Constants 295//===----------------------------------------------------------------------===// 296 297def TEX_ARRAY : PatLeaf< 298 (imm), 299 [{uint32_t TType = (uint32_t)N->getZExtValue(); 300 return TType == 9 || TType == 10 || TType == 16; 301 }] 302>; 303 304def TEX_RECT : PatLeaf< 305 (imm), 306 [{uint32_t TType = (uint32_t)N->getZExtValue(); 307 return TType == 5; 308 }] 309>; 310 311def TEX_SHADOW : PatLeaf< 312 (imm), 313 [{uint32_t TType = (uint32_t)N->getZExtValue(); 314 return (TType >= 6 && TType <= 8) || TType == 13; 315 }] 316>; 317 318def TEX_SHADOW_ARRAY : PatLeaf< 319 (imm), 320 [{uint32_t TType = (uint32_t)N->getZExtValue(); 321 return TType == 11 || TType == 12 || TType == 17; 322 }] 323>; 324 325//===----------------------------------------------------------------------===// 326// Load/Store Pattern Fragments 327//===----------------------------------------------------------------------===// 328 329class Aligned8Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{ 330 return cast<MemSDNode>(N)->getAlignment() % 8 == 0; 331}]>; 332 333class Aligned16Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{ 334 return cast<MemSDNode>(N)->getAlignment() >= 16; 335}]>; 336 337class LoadFrag <SDPatternOperator op> : PatFrag<(ops node:$ptr), (op node:$ptr)>; 338 339class StoreFrag<SDPatternOperator op> : PatFrag < 340 (ops node:$value, node:$ptr), (op node:$value, node:$ptr) 341>; 342 343class StoreHi16<SDPatternOperator op> : PatFrag < 344 (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr) 345>; 346 347class PrivateAddress : CodePatPred<[{ 348 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS; 349}]>; 350 351class ConstantAddress : CodePatPred<[{ 352 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS; 353}]>; 354 355class LocalAddress : CodePatPred<[{ 356 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 357}]>; 358 359class GlobalAddress : CodePatPred<[{ 360 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 361}]>; 362 363class GlobalLoadAddress : CodePatPred<[{ 364 auto AS = cast<MemSDNode>(N)->getAddressSpace(); 365 return AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::CONSTANT_ADDRESS; 366}]>; 367 368class FlatLoadAddress : CodePatPred<[{ 369 const auto AS = cast<MemSDNode>(N)->getAddressSpace(); 370 return AS == AMDGPUAS::FLAT_ADDRESS || 371 AS == AMDGPUAS::GLOBAL_ADDRESS || 372 AS == AMDGPUAS::CONSTANT_ADDRESS; 373}]>; 374 375class FlatStoreAddress : CodePatPred<[{ 376 const auto AS = cast<MemSDNode>(N)->getAddressSpace(); 377 return AS == AMDGPUAS::FLAT_ADDRESS || 378 AS == AMDGPUAS::GLOBAL_ADDRESS; 379}]>; 380 381class AZExtLoadBase <SDPatternOperator ld_node>: PatFrag<(ops node:$ptr), 382 (ld_node node:$ptr), [{ 383 LoadSDNode *L = cast<LoadSDNode>(N); 384 return L->getExtensionType() == ISD::ZEXTLOAD || 385 L->getExtensionType() == ISD::EXTLOAD; 386}]>; 387 388def az_extload : AZExtLoadBase <unindexedload>; 389 390def az_extloadi8 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{ 391 return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8; 392}]>; 393 394def az_extloadi16 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{ 395 return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16; 396}]>; 397 398def az_extloadi32 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{ 399 return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32; 400}]>; 401 402class PrivateLoad <SDPatternOperator op> : LoadFrag <op>, PrivateAddress; 403class PrivateStore <SDPatternOperator op> : StoreFrag <op>, PrivateAddress; 404 405class LocalLoad <SDPatternOperator op> : LoadFrag <op>, LocalAddress; 406class LocalStore <SDPatternOperator op> : StoreFrag <op>, LocalAddress; 407 408class GlobalLoad <SDPatternOperator op> : LoadFrag<op>, GlobalLoadAddress; 409class GlobalStore <SDPatternOperator op> : StoreFrag<op>, GlobalAddress; 410 411class FlatLoad <SDPatternOperator op> : LoadFrag <op>, FlatLoadAddress; 412class FlatStore <SDPatternOperator op> : StoreFrag <op>, FlatStoreAddress; 413 414class ConstantLoad <SDPatternOperator op> : LoadFrag <op>, ConstantAddress; 415 416 417def load_private : PrivateLoad <load>; 418def az_extloadi8_private : PrivateLoad <az_extloadi8>; 419def sextloadi8_private : PrivateLoad <sextloadi8>; 420def az_extloadi16_private : PrivateLoad <az_extloadi16>; 421def sextloadi16_private : PrivateLoad <sextloadi16>; 422 423def store_private : PrivateStore <store>; 424def truncstorei8_private : PrivateStore<truncstorei8>; 425def truncstorei16_private : PrivateStore <truncstorei16>; 426def store_hi16_private : StoreHi16 <truncstorei16>, PrivateAddress; 427def truncstorei8_hi16_private : StoreHi16<truncstorei8>, PrivateAddress; 428 429 430def load_global : GlobalLoad <load>; 431def sextloadi8_global : GlobalLoad <sextloadi8>; 432def az_extloadi8_global : GlobalLoad <az_extloadi8>; 433def sextloadi16_global : GlobalLoad <sextloadi16>; 434def az_extloadi16_global : GlobalLoad <az_extloadi16>; 435def atomic_load_global : GlobalLoad<atomic_load>; 436 437def store_global : GlobalStore <store>; 438def truncstorei8_global : GlobalStore <truncstorei8>; 439def truncstorei16_global : GlobalStore <truncstorei16>; 440def store_atomic_global : GlobalStore<atomic_store>; 441def truncstorei8_hi16_global : StoreHi16 <truncstorei8>, GlobalAddress; 442def truncstorei16_hi16_global : StoreHi16 <truncstorei16>, GlobalAddress; 443 444def load_local : LocalLoad <load>; 445def az_extloadi8_local : LocalLoad <az_extloadi8>; 446def sextloadi8_local : LocalLoad <sextloadi8>; 447def az_extloadi16_local : LocalLoad <az_extloadi16>; 448def sextloadi16_local : LocalLoad <sextloadi16>; 449def atomic_load_32_local : LocalLoad<atomic_load_32>; 450def atomic_load_64_local : LocalLoad<atomic_load_64>; 451 452def store_local : LocalStore <store>; 453def truncstorei8_local : LocalStore <truncstorei8>; 454def truncstorei16_local : LocalStore <truncstorei16>; 455def store_local_hi16 : StoreHi16 <truncstorei16>, LocalAddress; 456def truncstorei8_local_hi16 : StoreHi16<truncstorei8>, LocalAddress; 457def atomic_store_local : LocalStore <atomic_store>; 458 459def load_align8_local : Aligned8Bytes < 460 (ops node:$ptr), (load_local node:$ptr) 461>; 462 463def load_align16_local : Aligned16Bytes < 464 (ops node:$ptr), (load_local node:$ptr) 465>; 466 467def store_align8_local : Aligned8Bytes < 468 (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr) 469>; 470 471def store_align16_local : Aligned16Bytes < 472 (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr) 473>; 474 475def load_flat : FlatLoad <load>; 476def az_extloadi8_flat : FlatLoad <az_extloadi8>; 477def sextloadi8_flat : FlatLoad <sextloadi8>; 478def az_extloadi16_flat : FlatLoad <az_extloadi16>; 479def sextloadi16_flat : FlatLoad <sextloadi16>; 480def atomic_load_flat : FlatLoad<atomic_load>; 481 482def store_flat : FlatStore <store>; 483def truncstorei8_flat : FlatStore <truncstorei8>; 484def truncstorei16_flat : FlatStore <truncstorei16>; 485def atomic_store_flat : FlatStore <atomic_store>; 486def truncstorei8_hi16_flat : StoreHi16<truncstorei8>, FlatStoreAddress; 487def truncstorei16_hi16_flat : StoreHi16<truncstorei16>, FlatStoreAddress; 488 489 490def constant_load : ConstantLoad<load>; 491def sextloadi8_constant : ConstantLoad <sextloadi8>; 492def az_extloadi8_constant : ConstantLoad <az_extloadi8>; 493def sextloadi16_constant : ConstantLoad <sextloadi16>; 494def az_extloadi16_constant : ConstantLoad <az_extloadi16>; 495 496 497class local_binary_atomic_op<SDNode atomic_op> : 498 PatFrag<(ops node:$ptr, node:$value), 499 (atomic_op node:$ptr, node:$value), [{ 500 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 501}]>; 502 503def atomic_swap_local : local_binary_atomic_op<atomic_swap>; 504def atomic_load_add_local : local_binary_atomic_op<atomic_load_add>; 505def atomic_load_sub_local : local_binary_atomic_op<atomic_load_sub>; 506def atomic_load_and_local : local_binary_atomic_op<atomic_load_and>; 507def atomic_load_or_local : local_binary_atomic_op<atomic_load_or>; 508def atomic_load_xor_local : local_binary_atomic_op<atomic_load_xor>; 509def atomic_load_nand_local : local_binary_atomic_op<atomic_load_nand>; 510def atomic_load_min_local : local_binary_atomic_op<atomic_load_min>; 511def atomic_load_max_local : local_binary_atomic_op<atomic_load_max>; 512def atomic_load_umin_local : local_binary_atomic_op<atomic_load_umin>; 513def atomic_load_umax_local : local_binary_atomic_op<atomic_load_umax>; 514 515def mskor_global : PatFrag<(ops node:$val, node:$ptr), 516 (AMDGPUstore_mskor node:$val, node:$ptr), [{ 517 return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 518}]>; 519 520class AtomicCmpSwapLocal <SDNode cmp_swap_node> : PatFrag< 521 (ops node:$ptr, node:$cmp, node:$swap), 522 (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{ 523 AtomicSDNode *AN = cast<AtomicSDNode>(N); 524 return AN->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 525}]>; 526 527def atomic_cmp_swap_local : AtomicCmpSwapLocal <atomic_cmp_swap>; 528 529multiclass global_binary_atomic_op<SDNode atomic_op> { 530 def "" : PatFrag< 531 (ops node:$ptr, node:$value), 532 (atomic_op node:$ptr, node:$value), 533 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>; 534 535 def _noret : PatFrag< 536 (ops node:$ptr, node:$value), 537 (atomic_op node:$ptr, node:$value), 538 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; 539 540 def _ret : PatFrag< 541 (ops node:$ptr, node:$value), 542 (atomic_op node:$ptr, node:$value), 543 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; 544} 545 546defm atomic_swap_global : global_binary_atomic_op<atomic_swap>; 547defm atomic_add_global : global_binary_atomic_op<atomic_load_add>; 548defm atomic_and_global : global_binary_atomic_op<atomic_load_and>; 549defm atomic_max_global : global_binary_atomic_op<atomic_load_max>; 550defm atomic_min_global : global_binary_atomic_op<atomic_load_min>; 551defm atomic_or_global : global_binary_atomic_op<atomic_load_or>; 552defm atomic_sub_global : global_binary_atomic_op<atomic_load_sub>; 553defm atomic_umax_global : global_binary_atomic_op<atomic_load_umax>; 554defm atomic_umin_global : global_binary_atomic_op<atomic_load_umin>; 555defm atomic_xor_global : global_binary_atomic_op<atomic_load_xor>; 556 557// Legacy. 558def AMDGPUatomic_cmp_swap_global : PatFrag< 559 (ops node:$ptr, node:$value), 560 (AMDGPUatomic_cmp_swap node:$ptr, node:$value)>, GlobalAddress; 561 562def atomic_cmp_swap_global : PatFrag< 563 (ops node:$ptr, node:$cmp, node:$value), 564 (atomic_cmp_swap node:$ptr, node:$cmp, node:$value)>, GlobalAddress; 565 566 567def atomic_cmp_swap_global_noret : PatFrag< 568 (ops node:$ptr, node:$cmp, node:$value), 569 (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), 570 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; 571 572def atomic_cmp_swap_global_ret : PatFrag< 573 (ops node:$ptr, node:$cmp, node:$value), 574 (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), 575 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; 576 577//===----------------------------------------------------------------------===// 578// Misc Pattern Fragments 579//===----------------------------------------------------------------------===// 580 581class Constants { 582int TWO_PI = 0x40c90fdb; 583int PI = 0x40490fdb; 584int TWO_PI_INV = 0x3e22f983; 585int FP_UINT_MAX_PLUS_1 = 0x4f800000; // 1 << 32 in floating point encoding 586int FP16_ONE = 0x3C00; 587int FP16_NEG_ONE = 0xBC00; 588int V2FP16_ONE = 0x3C003C00; 589int FP32_ONE = 0x3f800000; 590int FP32_NEG_ONE = 0xbf800000; 591int FP64_ONE = 0x3ff0000000000000; 592int FP64_NEG_ONE = 0xbff0000000000000; 593} 594def CONST : Constants; 595 596def FP_ZERO : PatLeaf < 597 (fpimm), 598 [{return N->getValueAPF().isZero();}] 599>; 600 601def FP_ONE : PatLeaf < 602 (fpimm), 603 [{return N->isExactlyValue(1.0);}] 604>; 605 606def FP_HALF : PatLeaf < 607 (fpimm), 608 [{return N->isExactlyValue(0.5);}] 609>; 610 611/* Generic helper patterns for intrinsics */ 612/* -------------------------------------- */ 613 614class POW_Common <AMDGPUInst log_ieee, AMDGPUInst exp_ieee, AMDGPUInst mul> 615 : AMDGPUPat < 616 (fpow f32:$src0, f32:$src1), 617 (exp_ieee (mul f32:$src1, (log_ieee f32:$src0))) 618>; 619 620/* Other helper patterns */ 621/* --------------------- */ 622 623/* Extract element pattern */ 624class Extract_Element <ValueType sub_type, ValueType vec_type, int sub_idx, 625 SubRegIndex sub_reg> 626 : AMDGPUPat< 627 (sub_type (extractelt vec_type:$src, sub_idx)), 628 (EXTRACT_SUBREG $src, sub_reg) 629> { 630 let SubtargetPredicate = TruePredicate; 631} 632 633/* Insert element pattern */ 634class Insert_Element <ValueType elem_type, ValueType vec_type, 635 int sub_idx, SubRegIndex sub_reg> 636 : AMDGPUPat < 637 (insertelt vec_type:$vec, elem_type:$elem, sub_idx), 638 (INSERT_SUBREG $vec, $elem, sub_reg) 639> { 640 let SubtargetPredicate = TruePredicate; 641} 642 643// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer 644// can handle COPY instructions. 645// bitconvert pattern 646class BitConvert <ValueType dt, ValueType st, RegisterClass rc> : AMDGPUPat < 647 (dt (bitconvert (st rc:$src0))), 648 (dt rc:$src0) 649>; 650 651// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer 652// can handle COPY instructions. 653class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat < 654 (vt (AMDGPUdwordaddr (vt rc:$addr))), 655 (vt rc:$addr) 656>; 657 658// BFI_INT patterns 659 660multiclass BFIPatterns <Instruction BFI_INT, 661 Instruction LoadImm32, 662 RegisterClass RC64> { 663 // Definition from ISA doc: 664 // (y & x) | (z & ~x) 665 def : AMDGPUPat < 666 (or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))), 667 (BFI_INT $x, $y, $z) 668 >; 669 670 // 64-bit version 671 def : AMDGPUPat < 672 (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))), 673 (REG_SEQUENCE RC64, 674 (BFI_INT (i32 (EXTRACT_SUBREG $x, sub0)), 675 (i32 (EXTRACT_SUBREG $y, sub0)), 676 (i32 (EXTRACT_SUBREG $z, sub0))), sub0, 677 (BFI_INT (i32 (EXTRACT_SUBREG $x, sub1)), 678 (i32 (EXTRACT_SUBREG $y, sub1)), 679 (i32 (EXTRACT_SUBREG $z, sub1))), sub1) 680 >; 681 682 // SHA-256 Ch function 683 // z ^ (x & (y ^ z)) 684 def : AMDGPUPat < 685 (xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))), 686 (BFI_INT $x, $y, $z) 687 >; 688 689 // 64-bit version 690 def : AMDGPUPat < 691 (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), 692 (REG_SEQUENCE RC64, 693 (BFI_INT (i32 (EXTRACT_SUBREG $x, sub0)), 694 (i32 (EXTRACT_SUBREG $y, sub0)), 695 (i32 (EXTRACT_SUBREG $z, sub0))), sub0, 696 (BFI_INT (i32 (EXTRACT_SUBREG $x, sub1)), 697 (i32 (EXTRACT_SUBREG $y, sub1)), 698 (i32 (EXTRACT_SUBREG $z, sub1))), sub1) 699 >; 700 701 def : AMDGPUPat < 702 (fcopysign f32:$src0, f32:$src1), 703 (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, $src1) 704 >; 705 706 def : AMDGPUPat < 707 (f32 (fcopysign f32:$src0, f64:$src1)), 708 (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, 709 (i32 (EXTRACT_SUBREG $src1, sub1))) 710 >; 711 712 def : AMDGPUPat < 713 (f64 (fcopysign f64:$src0, f64:$src1)), 714 (REG_SEQUENCE RC64, 715 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 716 (BFI_INT (LoadImm32 (i32 0x7fffffff)), 717 (i32 (EXTRACT_SUBREG $src0, sub1)), 718 (i32 (EXTRACT_SUBREG $src1, sub1))), sub1) 719 >; 720 721 def : AMDGPUPat < 722 (f64 (fcopysign f64:$src0, f32:$src1)), 723 (REG_SEQUENCE RC64, 724 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 725 (BFI_INT (LoadImm32 (i32 0x7fffffff)), 726 (i32 (EXTRACT_SUBREG $src0, sub1)), 727 $src1), sub1) 728 >; 729} 730 731// SHA-256 Ma patterns 732 733// ((x & z) | (y & (x | z))) -> BFI_INT (XOR x, y), z, y 734multiclass SHA256MaPattern <Instruction BFI_INT, Instruction XOR, RegisterClass RC64> { 735 def : AMDGPUPat < 736 (or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))), 737 (BFI_INT (XOR i32:$x, i32:$y), i32:$z, i32:$y) 738 >; 739 740 def : AMDGPUPat < 741 (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))), 742 (REG_SEQUENCE RC64, 743 (BFI_INT (XOR (i32 (EXTRACT_SUBREG $x, sub0)), 744 (i32 (EXTRACT_SUBREG $y, sub0))), 745 (i32 (EXTRACT_SUBREG $z, sub0)), 746 (i32 (EXTRACT_SUBREG $y, sub0))), sub0, 747 (BFI_INT (XOR (i32 (EXTRACT_SUBREG $x, sub1)), 748 (i32 (EXTRACT_SUBREG $y, sub1))), 749 (i32 (EXTRACT_SUBREG $z, sub1)), 750 (i32 (EXTRACT_SUBREG $y, sub1))), sub1) 751 >; 752} 753 754// Bitfield extract patterns 755 756def IMMZeroBasedBitfieldMask : PatLeaf <(imm), [{ 757 return isMask_32(N->getZExtValue()); 758}]>; 759 760def IMMPopCount : SDNodeXForm<imm, [{ 761 return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N), 762 MVT::i32); 763}]>; 764 765multiclass BFEPattern <Instruction UBFE, Instruction SBFE, Instruction MOV> { 766 def : AMDGPUPat < 767 (i32 (and (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask)), 768 (UBFE $src, $rshift, (MOV (i32 (IMMPopCount $mask)))) 769 >; 770 771 // x & ((1 << y) - 1) 772 def : AMDGPUPat < 773 (and i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)), 774 (UBFE $src, (MOV (i32 0)), $width) 775 >; 776 777 // x & ~(-1 << y) 778 def : AMDGPUPat < 779 (and i32:$src, (xor_oneuse (shl_oneuse -1, i32:$width), -1)), 780 (UBFE $src, (MOV (i32 0)), $width) 781 >; 782 783 // x & (-1 >> (bitwidth - y)) 784 def : AMDGPUPat < 785 (and i32:$src, (srl_oneuse -1, (sub 32, i32:$width))), 786 (UBFE $src, (MOV (i32 0)), $width) 787 >; 788 789 // x << (bitwidth - y) >> (bitwidth - y) 790 def : AMDGPUPat < 791 (srl (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)), 792 (UBFE $src, (MOV (i32 0)), $width) 793 >; 794 795 def : AMDGPUPat < 796 (sra (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)), 797 (SBFE $src, (MOV (i32 0)), $width) 798 >; 799} 800 801// rotr pattern 802class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat < 803 (rotr i32:$src0, i32:$src1), 804 (BIT_ALIGN $src0, $src0, $src1) 805>; 806 807multiclass IntMed3Pat<Instruction med3Inst, 808 SDPatternOperator min, 809 SDPatternOperator max, 810 SDPatternOperator min_oneuse, 811 SDPatternOperator max_oneuse, 812 ValueType vt = i32> { 813 814 // This matches 16 permutations of 815 // min(max(a, b), max(min(a, b), c)) 816 def : AMDGPUPat < 817 (min (max_oneuse vt:$src0, vt:$src1), 818 (max_oneuse (min_oneuse vt:$src0, vt:$src1), vt:$src2)), 819 (med3Inst vt:$src0, vt:$src1, vt:$src2) 820>; 821 822 // This matches 16 permutations of 823 // max(min(x, y), min(max(x, y), z)) 824 def : AMDGPUPat < 825 (max (min_oneuse vt:$src0, vt:$src1), 826 (min_oneuse (max_oneuse vt:$src0, vt:$src1), vt:$src2)), 827 (med3Inst $src0, $src1, $src2) 828>; 829} 830 831// Special conversion patterns 832 833def cvt_rpi_i32_f32 : PatFrag < 834 (ops node:$src), 835 (fp_to_sint (ffloor (fadd $src, FP_HALF))), 836 [{ (void) N; return TM.Options.NoNaNsFPMath; }] 837>; 838 839def cvt_flr_i32_f32 : PatFrag < 840 (ops node:$src), 841 (fp_to_sint (ffloor $src)), 842 [{ (void)N; return TM.Options.NoNaNsFPMath; }] 843>; 844 845let AddedComplexity = 2 in { 846class IMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat < 847 (add (AMDGPUmul_i24 i32:$src0, i32:$src1), i32:$src2), 848 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)), 849 (Inst $src0, $src1, $src2)) 850>; 851 852class UMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat < 853 (add (AMDGPUmul_u24 i32:$src0, i32:$src1), i32:$src2), 854 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)), 855 (Inst $src0, $src1, $src2)) 856>; 857} // AddedComplexity. 858 859class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat < 860 (fdiv FP_ONE, vt:$src), 861 (RcpInst $src) 862>; 863 864class RsqPat<Instruction RsqInst, ValueType vt> : AMDGPUPat < 865 (AMDGPUrcp (fsqrt vt:$src)), 866 (RsqInst $src) 867>; 868 869// Instructions which select to the same v_min_f* 870def fminnum_like : PatFrags<(ops node:$src0, node:$src1), 871 [(fminnum_ieee node:$src0, node:$src1), 872 (fminnum node:$src0, node:$src1)] 873>; 874 875// Instructions which select to the same v_max_f* 876def fmaxnum_like : PatFrags<(ops node:$src0, node:$src1), 877 [(fmaxnum_ieee node:$src0, node:$src1), 878 (fmaxnum node:$src0, node:$src1)] 879>; 880 881def fminnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1), 882 [(fminnum_ieee_oneuse node:$src0, node:$src1), 883 (fminnum_oneuse node:$src0, node:$src1)] 884>; 885 886def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1), 887 [(fmaxnum_ieee_oneuse node:$src0, node:$src1), 888 (fmaxnum_oneuse node:$src0, node:$src1)] 889>; 890