1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FLATOffset : ComplexPattern<i64, 2, "SelectFlatOffset<false>", [], [SDNPWantRoot], -10>; 10def FLATOffsetSigned : ComplexPattern<i64, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 11 12def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 13 14//===----------------------------------------------------------------------===// 15// FLAT classes 16//===----------------------------------------------------------------------===// 17 18class FLAT_Pseudo<string opName, dag outs, dag ins, 19 string asmOps, list<dag> pattern=[]> : 20 InstSI<outs, ins, "", pattern>, 21 SIMCInstr<opName, SIEncodingFamily.NONE> { 22 23 let isPseudo = 1; 24 let isCodeGenOnly = 1; 25 26 let FLAT = 1; 27 28 let UseNamedOperandTable = 1; 29 let hasSideEffects = 0; 30 let SchedRW = [WriteVMEM]; 31 32 string Mnemonic = opName; 33 string AsmOperands = asmOps; 34 35 bits<1> is_flat_global = 0; 36 bits<1> is_flat_scratch = 0; 37 38 bits<1> has_vdst = 1; 39 40 // We need to distinguish having saddr and enabling saddr because 41 // saddr is only valid for scratch and global instructions. Pre-gfx9 42 // these bits were reserved, so we also don't necessarily want to 43 // set these bits to the disabled value for the original flat 44 // segment instructions. 45 bits<1> has_saddr = 0; 46 bits<1> enabled_saddr = 0; 47 bits<7> saddr_value = 0; 48 bits<1> has_vaddr = 1; 49 50 bits<1> has_data = 1; 51 bits<1> has_glc = 1; 52 bits<1> glcValue = 0; 53 bits<1> has_dlc = 1; 54 bits<1> dlcValue = 0; 55 56 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 57 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 58 59 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 60 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 61 62 // Internally, FLAT instruction are executed as both an LDS and a 63 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 64 // and are not considered done until both have been decremented. 65 let VM_CNT = 1; 66 let LGKM_CNT = !if(!or(is_flat_global, is_flat_scratch), 0, 1); 67 68 let IsNonFlatSeg = !if(!or(is_flat_global, is_flat_scratch), 1, 0); 69} 70 71class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 72 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 73 Enc64 { 74 75 let isPseudo = 0; 76 let isCodeGenOnly = 0; 77 78 // copy relevant pseudo op flags 79 let SubtargetPredicate = ps.SubtargetPredicate; 80 let AsmMatchConverter = ps.AsmMatchConverter; 81 let OtherPredicates = ps.OtherPredicates; 82 let TSFlags = ps.TSFlags; 83 let UseNamedOperandTable = ps.UseNamedOperandTable; 84 85 // encoding fields 86 bits<8> vaddr; 87 bits<8> vdata; 88 bits<7> saddr; 89 bits<8> vdst; 90 91 bits<1> slc; 92 bits<1> glc; 93 bits<1> dlc; 94 95 // Only valid on gfx9 96 bits<1> lds = 0; // XXX - What does this actually do? 97 98 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 99 bits<2> seg = !if(ps.is_flat_global, 0b10, 100 !if(ps.is_flat_scratch, 0b01, 0)); 101 102 // Signed offset. Highest bit ignored for flat and treated as 12-bit 103 // unsigned for flat accesses. 104 bits<13> offset; 105 bits<1> nv = 0; // XXX - What does this actually do? 106 107 // We don't use tfe right now, and it was removed in gfx9. 108 bits<1> tfe = 0; 109 110 // Only valid on GFX9+ 111 let Inst{12-0} = offset; 112 let Inst{13} = lds; 113 let Inst{15-14} = seg; 114 115 let Inst{16} = !if(ps.has_glc, glc, ps.glcValue); 116 let Inst{17} = slc; 117 let Inst{24-18} = op; 118 let Inst{31-26} = 0x37; // Encoding. 119 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 120 let Inst{47-40} = !if(ps.has_data, vdata, ?); 121 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 122 123 // 54-48 is reserved. 124 let Inst{55} = nv; // nv on GFX9+, TFE before. 125 let Inst{63-56} = !if(ps.has_vdst, vdst, ?); 126} 127 128class GlobalSaddrTable <bit is_saddr, string Name = ""> { 129 bit IsSaddr = is_saddr; 130 string SaddrOp = Name; 131} 132 133// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 134// same encoding value as exec_hi, so it isn't possible to use that if 135// saddr is 32-bit (which isn't handled here yet). 136class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 137 bit HasTiedOutput = 0, 138 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 139 opName, 140 (outs regClass:$vdst), 141 !con( 142 !con( 143 !if(EnableSaddr, 144 (ins SReg_64:$saddr, VGPR_32:$vaddr), 145 (ins VReg_64:$vaddr)), 146 (ins flat_offset:$offset)), 147 // FIXME: Operands with default values do not work with following non-optional operands. 148 !if(HasTiedOutput, (ins GLC:$glc, SLC:$slc, DLC:$dlc, regClass:$vdst_in), 149 (ins GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc))), 150 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc"> { 151 let has_data = 0; 152 let mayLoad = 1; 153 let has_saddr = HasSaddr; 154 let enabled_saddr = EnableSaddr; 155 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 156 let maybeAtomic = 1; 157 158 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 159 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 160} 161 162class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 163 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 164 opName, 165 (outs), 166 !con( 167 !if(EnableSaddr, 168 (ins VGPR_32:$vaddr, vdataClass:$vdata, SReg_64:$saddr), 169 (ins VReg_64:$vaddr, vdataClass:$vdata)), 170 (ins flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc)), 171 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc"> { 172 let mayLoad = 0; 173 let mayStore = 1; 174 let has_vdst = 0; 175 let has_saddr = HasSaddr; 176 let enabled_saddr = EnableSaddr; 177 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 178 let maybeAtomic = 1; 179} 180 181multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 182 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 183 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 184 GlobalSaddrTable<0, opName>; 185 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 186 GlobalSaddrTable<1, opName>; 187 } 188} 189 190class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 191 bit HasTiedOutput = 0, bit HasSignedOffset = 0> : FLAT_Pseudo< 192 opName, 193 (outs regClass:$vdst), 194 !con((ins SReg_64:$saddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc), 195 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 196 " $vdst, $saddr$offset$glc$slc$dlc"> { 197 let is_flat_global = 1; 198 let has_data = 0; 199 let mayLoad = 1; 200 let has_vaddr = 0; 201 let has_saddr = 1; 202 let enabled_saddr = 1; 203 let maybeAtomic = 1; 204 205 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 206 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 207} 208 209multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 210 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 211 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 212 GlobalSaddrTable<0, opName>; 213 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 214 GlobalSaddrTable<1, opName>; 215 } 216} 217 218class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 219 bit HasSignedOffset = 0> : FLAT_Pseudo< 220 opName, 221 (outs), 222 !con( 223 (ins vdataClass:$vdata, SReg_64:$saddr), 224 (ins flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc)), 225 " $vdata, $saddr$offset$glc$slc$dlc"> { 226 let is_flat_global = 1; 227 let mayLoad = 0; 228 let mayStore = 1; 229 let has_vdst = 0; 230 let has_vaddr = 0; 231 let has_saddr = 1; 232 let enabled_saddr = 1; 233 let maybeAtomic = 1; 234} 235 236class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 237 bit HasTiedOutput = 0, 238 bit EnableSaddr = 0, 239 bit EnableVaddr = !not(EnableSaddr)> 240 : FLAT_Pseudo< 241 opName, 242 (outs regClass:$vdst), 243 !con( 244 !if(EnableSaddr, 245 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 246 !if(EnableVaddr, 247 (ins VGPR_32:$vaddr, flat_offset:$offset), 248 (ins flat_offset:$offset))), 249 !if(HasTiedOutput, (ins GLC:$glc, SLC:$slc, DLC:$dlc, regClass:$vdst_in), 250 (ins GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc))), 251 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc"> { 252 let has_data = 0; 253 let mayLoad = 1; 254 let has_saddr = 1; 255 let enabled_saddr = EnableSaddr; 256 let has_vaddr = EnableVaddr; 257 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 258 let maybeAtomic = 1; 259 260 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 261 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 262} 263 264class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 265 bit EnableVaddr = !not(EnableSaddr)> : FLAT_Pseudo< 266 opName, 267 (outs), 268 !if(EnableSaddr, 269 (ins vdataClass:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc), 270 !if(EnableVaddr, 271 (ins vdataClass:$vdata, VGPR_32:$vaddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc), 272 (ins vdataClass:$vdata, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc))), 273 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc"> { 274 let mayLoad = 0; 275 let mayStore = 1; 276 let has_vdst = 0; 277 let has_saddr = 1; 278 let enabled_saddr = EnableSaddr; 279 let has_vaddr = EnableVaddr; 280 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 281 let maybeAtomic = 1; 282} 283 284multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 285 let is_flat_scratch = 1 in { 286 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>; 287 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>; 288 289 let SubtargetPredicate = HasFlatScratchSTMode in 290 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0>; 291 } 292} 293 294multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 295 let is_flat_scratch = 1 in { 296 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>; 297 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>; 298 299 let SubtargetPredicate = HasFlatScratchSTMode in 300 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0>; 301 } 302} 303 304class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 305 string asm, list<dag> pattern = []> : 306 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 307 let mayLoad = 1; 308 let mayStore = 1; 309 let has_glc = 0; 310 let glcValue = 0; 311 let has_dlc = 0; 312 let dlcValue = 0; 313 let has_vdst = 0; 314 let maybeAtomic = 1; 315} 316 317class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 318 string asm, list<dag> pattern = []> 319 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 320 let hasPostISelHook = 1; 321 let has_vdst = 1; 322 let glcValue = 1; 323 let dlcValue = 0; 324 let PseudoInstr = NAME # "_RTN"; 325} 326 327multiclass FLAT_Atomic_Pseudo< 328 string opName, 329 RegisterClass vdst_rc, 330 ValueType vt, 331 SDPatternOperator atomic = null_frag, 332 ValueType data_vt = vt, 333 RegisterClass data_rc = vdst_rc, 334 bit isFP = isFloatType<data_vt>.ret> { 335 def "" : FLAT_AtomicNoRet_Pseudo <opName, 336 (outs), 337 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 338 " $vaddr, $vdata$offset$slc">, 339 GlobalSaddrTable<0, opName>, 340 AtomicNoRet <opName, 0> { 341 let PseudoInstr = NAME; 342 let FPAtomic = isFP; 343 let AddedComplexity = -1; // Prefer global atomics if available 344 } 345 346 def _RTN : FLAT_AtomicRet_Pseudo <opName, 347 (outs vdst_rc:$vdst), 348 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 349 " $vdst, $vaddr, $vdata$offset glc$slc", 350 [(set vt:$vdst, 351 (atomic (FLATOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 352 GlobalSaddrTable<0, opName#"_rtn">, 353 AtomicNoRet <opName, 1>{ 354 let FPAtomic = isFP; 355 let AddedComplexity = -1; // Prefer global atomics if available 356 } 357} 358 359multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 360 string opName, 361 RegisterClass vdst_rc, 362 ValueType vt, 363 SDPatternOperator atomic = null_frag, 364 ValueType data_vt = vt, 365 RegisterClass data_rc = vdst_rc, 366 bit isFP = isFloatType<data_vt>.ret> { 367 368 def "" : FLAT_AtomicNoRet_Pseudo <opName, 369 (outs), 370 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 371 " $vaddr, $vdata, off$offset$slc">, 372 GlobalSaddrTable<0, opName>, 373 AtomicNoRet <opName, 0> { 374 let has_saddr = 1; 375 let PseudoInstr = NAME; 376 let FPAtomic = isFP; 377 } 378 379 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 380 (outs), 381 (ins VGPR_32:$vaddr, data_rc:$vdata, SReg_64:$saddr, flat_offset:$offset, SLC_0:$slc), 382 " $vaddr, $vdata, $saddr$offset$slc">, 383 GlobalSaddrTable<1, opName>, 384 AtomicNoRet <opName#"_saddr", 0> { 385 let has_saddr = 1; 386 let enabled_saddr = 1; 387 let PseudoInstr = NAME#"_SADDR"; 388 let FPAtomic = isFP; 389 } 390} 391 392multiclass FLAT_Global_Atomic_Pseudo_RTN< 393 string opName, 394 RegisterClass vdst_rc, 395 ValueType vt, 396 SDPatternOperator atomic = null_frag, 397 ValueType data_vt = vt, 398 RegisterClass data_rc = vdst_rc, 399 bit isFP = isFloatType<data_vt>.ret> { 400 401 def _RTN : FLAT_AtomicRet_Pseudo <opName, 402 (outs vdst_rc:$vdst), 403 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 404 " $vdst, $vaddr, $vdata, off$offset glc$slc", 405 [(set vt:$vdst, 406 (atomic (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 407 GlobalSaddrTable<0, opName#"_rtn">, 408 AtomicNoRet <opName, 1> { 409 let has_saddr = 1; 410 let FPAtomic = isFP; 411 } 412 413 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 414 (outs vdst_rc:$vdst), 415 (ins VGPR_32:$vaddr, data_rc:$vdata, SReg_64:$saddr, flat_offset:$offset, SLC_0:$slc), 416 " $vdst, $vaddr, $vdata, $saddr$offset glc$slc">, 417 GlobalSaddrTable<1, opName#"_rtn">, 418 AtomicNoRet <opName#"_saddr", 1> { 419 let has_saddr = 1; 420 let enabled_saddr = 1; 421 let PseudoInstr = NAME#"_SADDR_RTN"; 422 let FPAtomic = isFP; 423 } 424} 425 426multiclass FLAT_Global_Atomic_Pseudo< 427 string opName, 428 RegisterClass vdst_rc, 429 ValueType vt, 430 SDPatternOperator atomic_rtn = null_frag, 431 SDPatternOperator atomic_no_rtn = null_frag, 432 ValueType data_vt = vt, 433 RegisterClass data_rc = vdst_rc> { 434 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 435 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>; 436 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; 437 } 438} 439 440//===----------------------------------------------------------------------===// 441// Flat Instructions 442//===----------------------------------------------------------------------===// 443 444def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 445def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 446def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 447def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 448def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 449def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 450def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 451def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 452 453def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 454def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 455def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 456def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 457def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 458def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 459 460let SubtargetPredicate = HasD16LoadStore in { 461def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 462def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 463def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 464def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 465def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 466def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 467 468def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 469def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 470} 471 472defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 473 VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, 474 v2i32, VReg_64>; 475 476defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 477 VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, 478 v2i64, VReg_128>; 479 480defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 481 VGPR_32, i32, atomic_swap_flat_32>; 482 483defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 484 VReg_64, i64, atomic_swap_flat_64>; 485 486defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 487 VGPR_32, i32, atomic_load_add_flat_32>; 488 489defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 490 VGPR_32, i32, atomic_load_sub_flat_32>; 491 492defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 493 VGPR_32, i32, atomic_load_min_flat_32>; 494 495defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 496 VGPR_32, i32, atomic_load_umin_flat_32>; 497 498defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 499 VGPR_32, i32, atomic_load_max_flat_32>; 500 501defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 502 VGPR_32, i32, atomic_load_umax_flat_32>; 503 504defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 505 VGPR_32, i32, atomic_load_and_flat_32>; 506 507defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 508 VGPR_32, i32, atomic_load_or_flat_32>; 509 510defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 511 VGPR_32, i32, atomic_load_xor_flat_32>; 512 513defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 514 VGPR_32, i32, atomic_inc_flat_32>; 515 516defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 517 VGPR_32, i32, atomic_dec_flat_32>; 518 519defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 520 VReg_64, i64, atomic_load_add_flat_64>; 521 522defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 523 VReg_64, i64, atomic_load_sub_flat_64>; 524 525defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 526 VReg_64, i64, atomic_load_min_flat_64>; 527 528defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 529 VReg_64, i64, atomic_load_umin_flat_64>; 530 531defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 532 VReg_64, i64, atomic_load_max_flat_64>; 533 534defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 535 VReg_64, i64, atomic_load_umax_flat_64>; 536 537defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 538 VReg_64, i64, atomic_load_and_flat_64>; 539 540defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 541 VReg_64, i64, atomic_load_or_flat_64>; 542 543defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 544 VReg_64, i64, atomic_load_xor_flat_64>; 545 546defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 547 VReg_64, i64, atomic_inc_flat_64>; 548 549defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 550 VReg_64, i64, atomic_dec_flat_64>; 551 552// GFX7-, GFX10-only flat instructions. 553let SubtargetPredicate = isGFX7GFX10 in { 554 555defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 556 VGPR_32, f32, null_frag, v2f32, VReg_64>; 557 558defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 559 VReg_64, f64, null_frag, v2f64, VReg_128>; 560 561defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 562 VGPR_32, f32>; 563 564defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 565 VGPR_32, f32>; 566 567defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 568 VReg_64, f64>; 569 570defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 571 VReg_64, f64>; 572 573} // End SubtargetPredicate = isGFX7GFX10 574 575defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 576defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 577defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 578defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 579defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 580defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 581defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 582defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 583 584defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 585defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 586defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 587defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 588defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 589defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 590let OtherPredicates = [HasGFX10_BEncoding] in 591def GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 592 593defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 594defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 595defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 596defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 597defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 598defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 599let OtherPredicates = [HasGFX10_BEncoding] in 600def GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 601 602defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 603defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 604 605let is_flat_global = 1 in { 606defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 607 VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag, 608 v2i32, VReg_64>; 609 610defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 611 VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, 612 null_frag, 613 v2i64, VReg_128>; 614 615defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 616 VGPR_32, i32, atomic_swap_global_32>; 617 618defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 619 VReg_64, i64, atomic_swap_global_64>; 620 621defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 622 VGPR_32, i32, atomic_load_add_global_32>; 623 624defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 625 VGPR_32, i32, atomic_load_sub_global_32>; 626 627defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 628 VGPR_32, i32, atomic_load_min_global_32>; 629 630defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 631 VGPR_32, i32, atomic_load_umin_global_32>; 632 633defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 634 VGPR_32, i32, atomic_load_max_global_32>; 635 636defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 637 VGPR_32, i32, atomic_load_umax_global_32>; 638 639defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 640 VGPR_32, i32, atomic_load_and_global_32>; 641 642defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 643 VGPR_32, i32, atomic_load_or_global_32>; 644 645defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 646 VGPR_32, i32, atomic_load_xor_global_32>; 647 648defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 649 VGPR_32, i32, atomic_inc_global_32>; 650 651defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 652 VGPR_32, i32, atomic_dec_global_32>; 653 654defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 655 VReg_64, i64, atomic_load_add_global_64>; 656 657defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 658 VReg_64, i64, atomic_load_sub_global_64>; 659 660defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 661 VReg_64, i64, atomic_load_min_global_64>; 662 663defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 664 VReg_64, i64, atomic_load_umin_global_64>; 665 666defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 667 VReg_64, i64, atomic_load_max_global_64>; 668 669defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 670 VReg_64, i64, atomic_load_umax_global_64>; 671 672defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 673 VReg_64, i64, atomic_load_and_global_64>; 674 675defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 676 VReg_64, i64, atomic_load_or_global_64>; 677 678defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 679 VReg_64, i64, atomic_load_xor_global_64>; 680 681defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 682 VReg_64, i64, atomic_inc_global_64>; 683 684defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 685 VReg_64, i64, atomic_dec_global_64>; 686 687let SubtargetPredicate = HasGFX10_BEncoding in 688defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 689 VGPR_32, i32, int_amdgcn_global_atomic_csub>; 690} // End is_flat_global = 1 691 692 693 694let SubtargetPredicate = HasFlatScratchInsts in { 695defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 696defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 697defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 698defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 699defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 700defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 701defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 702defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 703 704defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 705defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 706defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 707defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 708defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 709defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 710 711defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 712defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 713defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 714defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 715defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 716defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 717 718defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 719defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 720 721} // End SubtargetPredicate = HasFlatScratchInsts 722 723let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 724 defm GLOBAL_ATOMIC_FCMPSWAP : 725 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32>; 726 defm GLOBAL_ATOMIC_FMIN : 727 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 728 defm GLOBAL_ATOMIC_FMAX : 729 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 730 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 731 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>; 732 defm GLOBAL_ATOMIC_FMIN_X2 : 733 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 734 defm GLOBAL_ATOMIC_FMAX_X2 : 735 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 736} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 737 738let is_flat_global = 1 in { 739let OtherPredicates = [HasAtomicFaddInsts] in { 740 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 741 "global_atomic_add_f32", VGPR_32, f32 742 >; 743 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 744 "global_atomic_pk_add_f16", VGPR_32, v2f16 745 >; 746} // End OtherPredicates = [HasAtomicFaddInsts] 747} // End is_flat_global = 1 748 749//===----------------------------------------------------------------------===// 750// Flat Patterns 751//===----------------------------------------------------------------------===// 752 753// Patterns for global loads with no offset. 754class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 755 (vt (node (FLATOffset i64:$vaddr, i16:$offset))), 756 (inst $vaddr, $offset) 757>; 758 759class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 760 (node (FLATOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 761 (inst $vaddr, $offset, 0, 0, 0, $in) 762>; 763 764class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 765 (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 766 (inst $vaddr, $offset, 0, 0, 0, $in) 767>; 768 769class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 770 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 771 (inst $saddr, $voffset, $offset, 0, 0, 0, $in) 772>; 773 774class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 775 (vt (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset))), 776 (inst $vaddr, $offset) 777>; 778 779class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 780 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 781 (inst $saddr, $voffset, $offset, 0, 0, 0) 782>; 783 784class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 785 ValueType vt> : GCNPat < 786 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 787 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 788>; 789 790class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 791 ValueType vt> : GCNPat < 792 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 793 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 794>; 795 796class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 797 ValueType vt, ValueType data_vt = vt> : GCNPat < 798 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 799 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 800>; 801 802class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 803 ValueType vt> : GCNPat < 804 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 805 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 806>; 807 808class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 809 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset)), 810 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 811>; 812 813class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 814 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset)), 815 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 816>; 817 818class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 819 // atomic store follows atomic binop convention so the address comes 820 // first. 821 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 822 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 823>; 824 825class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 826 ValueType vt, ValueType data_vt = vt> : GCNPat < 827 // atomic store follows atomic binop convention so the address comes 828 // first. 829 (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data), 830 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 831>; 832 833class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 834 ValueType data_vt = vt> : GCNPat < 835 (vt (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data)), 836 (inst $vaddr, $data, $offset) 837>; 838 839class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 840 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 841 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 842>; 843 844class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 845 (node (FLATOffsetSigned i64:$vaddr, i16:$offset), vt:$data), 846 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 847>; 848 849class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 850 ValueType data_vt = vt> : GCNPat < 851 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$data)), 852 (inst $vaddr, $data, $offset) 853>; 854 855let OtherPredicates = [HasFlatAddressSpace] in { 856 857def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 858def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 859def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 860def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 861def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 862def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 863def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 864def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 865def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 866def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 867def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 868 869def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 870def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 871 872def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 873def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 874 875foreach vt = Reg32Types.types in { 876def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 877def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 878} 879 880foreach vt = VReg_64.RegTypes in { 881def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 882def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 883} 884 885def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 886 887foreach vt = VReg_128.RegTypes in { 888def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 889def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 890} 891 892def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; 893def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64>; 894 895def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_load_add_global_32, i32>; 896def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_load_sub_global_32, i32>; 897def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global_32, i32>; 898def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global_32, i32>; 899def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_load_and_global_32, i32>; 900def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_load_max_global_32, i32>; 901def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_load_umax_global_32, i32>; 902def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i32>; 903def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; 904def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; 905def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; 906def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 907def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; 908 909def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; 910def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_load_sub_global_64, i64>; 911def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global_64, i64>; 912def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global_64, i64>; 913def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_load_and_global_64, i64>; 914def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_load_max_global_64, i64>; 915def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_load_umax_global_64, i64>; 916def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, i64>; 917def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; 918def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; 919def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; 920def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 921def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; 922 923def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 924def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 925 926let OtherPredicates = [D16PreservesUnusedBits] in { 927def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 928def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 929 930def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 931def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 932def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 933def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 934def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 935def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 936 937def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 938def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 939def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 940def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 941def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 942def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 943} 944 945} // End OtherPredicates = [HasFlatAddressSpace] 946 947 948multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 949 def : FlatLoadSignedPat <inst, node, vt> { 950 let AddedComplexity = 10; 951 } 952 953 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 954 let AddedComplexity = 11; 955 } 956} 957 958multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 959 def : FlatSignedLoadPat_D16 <inst, node, vt> { 960 let AddedComplexity = 10; 961 } 962 963 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 964 let AddedComplexity = 11; 965 } 966} 967 968multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 969 ValueType vt> { 970 def : FlatStoreSignedPat <inst, node, vt> { 971 let AddedComplexity = 10; 972 } 973 974 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 975 let AddedComplexity = 11; 976 } 977} 978 979// Deal with swapped operands for atomic_store vs. regular store 980multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 981 def : FlatStoreSignedAtomicPat <inst, node, vt> { 982 let AddedComplexity = 10; 983 } 984 985 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 986 let AddedComplexity = 11; 987 } 988} 989 990multiclass GlobalFLATAtomicPats<string nortn_inst_name, SDPatternOperator node, 991 ValueType vt, ValueType data_vt = vt> { 992 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 993 let AddedComplexity = 10; 994 } 995 996 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 997 let AddedComplexity = 11; 998 } 999} 1000 1001multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1002 ValueType vt> { 1003 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1004 let AddedComplexity = 10; 1005 } 1006 1007 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1008 let AddedComplexity = 11; 1009 } 1010} 1011 1012let OtherPredicates = [HasFlatGlobalInsts] in { 1013 1014defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1015defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1016defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1017defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1018defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1019defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1020defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1021defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1022defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1023defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1024 1025foreach vt = Reg32Types.types in { 1026defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1027defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1028} 1029 1030foreach vt = VReg_64.RegTypes in { 1031defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1032defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1033} 1034 1035defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1036 1037foreach vt = VReg_128.RegTypes in { 1038defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1039defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1040} 1041 1042// There is no distinction for atomic load lowering during selection; 1043// the memory legalizer will set the cache bits and insert the 1044// appropriate waits. 1045defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1046defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1047 1048defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1049defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1050defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1051defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1052defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1053 1054let OtherPredicates = [D16PreservesUnusedBits] in { 1055defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1056defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1057 1058defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1059defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1060defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1061defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1062defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1063defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1064 1065defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1066defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1067defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1068defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1069defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1070defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1071} 1072 1073defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; 1074defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64>; 1075 1076defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", atomic_load_add_global_32, i32>; 1077defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", atomic_load_sub_global_32, i32>; 1078defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", atomic_inc_global_32, i32>; 1079defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", atomic_dec_global_32, i32>; 1080defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", atomic_load_and_global_32, i32>; 1081defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", atomic_load_max_global_32, i32>; 1082defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", atomic_load_umax_global_32, i32>; 1083defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", atomic_load_min_global_32, i32>; 1084defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", atomic_load_umin_global_32, i32>; 1085defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", atomic_load_or_global_32, i32>; 1086defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", atomic_swap_global_32, i32>; 1087defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1088defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", atomic_load_xor_global_32, i32>; 1089defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1090 1091defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", atomic_load_add_global_64, i64>; 1092defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", atomic_load_sub_global_64, i64>; 1093defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", atomic_inc_global_64, i64>; 1094defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", atomic_dec_global_64, i64>; 1095defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", atomic_load_and_global_64, i64>; 1096defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", atomic_load_max_global_64, i64>; 1097defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", atomic_load_umax_global_64, i64>; 1098defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", atomic_load_min_global_64, i64>; 1099defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", atomic_load_umin_global_64, i64>; 1100defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", atomic_load_or_global_64, i64>; 1101defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", atomic_swap_global_64, i64>; 1102defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1103defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", atomic_load_xor_global_64, i64>; 1104 1105let OtherPredicates = [HasAtomicFaddInsts] in { 1106defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1107defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1108} 1109 1110} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1111 1112 1113//===----------------------------------------------------------------------===// 1114// Target 1115//===----------------------------------------------------------------------===// 1116 1117//===----------------------------------------------------------------------===// 1118// CI 1119//===----------------------------------------------------------------------===// 1120 1121class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1122 FLAT_Real <op, ps>, 1123 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1124 let AssemblerPredicate = isGFX7Only; 1125 let DecoderNamespace="GFX7"; 1126} 1127 1128def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1129def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1130def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1131def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1132def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1133def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1134def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1135def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1136 1137def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1138def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1139def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1140def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1141def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1142def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1143 1144multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1145 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1146 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1147} 1148 1149defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1150defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1151defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1152defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1153defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1154defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1155defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1156defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1157defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1158defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1159defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1160defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1161defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1162defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1163defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1164defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1165defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1166defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1167defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1168defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1169defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1170defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1171defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1172defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1173defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1174defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1175 1176// CI Only flat instructions 1177defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1178defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1179defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1180defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1181defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1182defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1183 1184 1185//===----------------------------------------------------------------------===// 1186// VI 1187//===----------------------------------------------------------------------===// 1188 1189class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps> : 1190 FLAT_Real <op, ps>, 1191 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1192 let AssemblerPredicate = isGFX8GFX9; 1193 let DecoderNamespace = "GFX8"; 1194} 1195 1196multiclass FLAT_Real_AllAddr_vi<bits<7> op> { 1197 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)>; 1198 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1199} 1200 1201def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1202def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1203def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1204def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1205def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1206def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1207def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1208def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1209 1210def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1211def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1212def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1213def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1214def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1215def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1216def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1217def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1218 1219def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1220def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1221def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1222def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1223def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1224def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1225 1226multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> { 1227 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1228 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1229} 1230 1231multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> : 1232 FLAT_Real_AllAddr_vi<op> { 1233 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1234 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1235} 1236 1237 1238defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1239defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1240defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1241defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1242defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1243defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1244defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1245defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1246defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1247defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1248defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1249defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1250defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1251defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1252defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1253defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1254defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1255defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1256defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1257defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1258defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1259defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1260defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1261defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1262defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1263defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1264 1265defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1266defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1267defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1268defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1269defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1270defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1271defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1272defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1273 1274defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1275defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1276defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1277defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1278defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1279defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1280 1281defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1282defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1283defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1284defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1285defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1286defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1287defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1288defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1289 1290 1291defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1292defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1293defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1294defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1295defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1296defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1297defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1298defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1299defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1300defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1301defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1302defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1303defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1304defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1305defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1306defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1307defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1308defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1309defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1310defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1311defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1312defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1313defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1314defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1315defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1316defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1317 1318defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1319defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1320defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1321defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1322defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1323defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1324defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1325defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1326defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1327defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1328defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1329defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1330defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1331defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1332defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1333defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1334defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1335defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1336defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1337defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1338defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1339defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1340 1341 1342//===----------------------------------------------------------------------===// 1343// GFX10. 1344//===----------------------------------------------------------------------===// 1345 1346class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1347 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1348 let AssemblerPredicate = isGFX10Plus; 1349 let DecoderNamespace = "GFX10"; 1350 1351 let Inst{11-0} = offset{11-0}; 1352 let Inst{12} = !if(ps.has_dlc, dlc, ps.dlcValue); 1353 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1354 let Inst{55} = 0; 1355} 1356 1357 1358multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1359 def _gfx10 : 1360 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1361} 1362 1363multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1364 def _RTN_gfx10 : 1365 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1366} 1367 1368multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1369 def _SADDR_gfx10 : 1370 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1371} 1372 1373multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1374 def _SADDR_RTN_gfx10 : 1375 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1376} 1377 1378multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1379 def _ST_gfx10 : 1380 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1381 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1382 let OtherPredicates = [HasFlatScratchSTMode]; 1383 } 1384} 1385 1386multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1387 FLAT_Real_Base_gfx10<op>, 1388 FLAT_Real_SADDR_gfx10<op>; 1389 1390multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1391 FLAT_Real_Base_gfx10<op>, 1392 FLAT_Real_RTN_gfx10<op>; 1393 1394multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1395 FLAT_Real_AllAddr_gfx10<op>, 1396 FLAT_Real_RTN_gfx10<op>, 1397 FLAT_Real_SADDR_RTN_gfx10<op>; 1398 1399multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1400 FLAT_Real_RTN_gfx10<op>, 1401 FLAT_Real_SADDR_RTN_gfx10<op>; 1402 1403multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1404 FLAT_Real_Base_gfx10<op>, 1405 FLAT_Real_SADDR_gfx10<op>, 1406 FLAT_Real_ST_gfx10<op>; 1407 1408// ENC_FLAT. 1409defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1410defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1411defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1412defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1413defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1414defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1415defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1416defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1417defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1418defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1419defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1420defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1421defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1422defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1423defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1424defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1425defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1426defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1427defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1428defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1429defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1430defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1431defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1432defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1433defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1434defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1435defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1436defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1437defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1438defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1439defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1440defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1441defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1442defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1443defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1444defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1445defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1446defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1447defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1448defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1449defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1450defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1451defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1452defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1453defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1454defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1455defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1456defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1457defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1458defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1459defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1460defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1461defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1462defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1463 1464 1465// ENC_FLAT_GLBL. 1466defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1467defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1468defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1469defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1470defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1471defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1472defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1473defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1474defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1475defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1476defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1477defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1478defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1479defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1480defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1481defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1482defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1483defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1484defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1485defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1486defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1487defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1488defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 1489defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 1490defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 1491defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 1492defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 1493defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 1494defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 1495defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 1496defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 1497defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 1498defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 1499defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 1500defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 1501defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 1502defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 1503defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 1504defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 1505defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 1506defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 1507defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 1508defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 1509defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 1510defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 1511defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 1512defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 1513defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 1514defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 1515defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 1516defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 1517defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 1518defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 1519defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 1520defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 1521defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_Base_gfx10<0x016>; 1522defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_Base_gfx10<0x017>; 1523 1524// ENC_FLAT_SCRATCH. 1525defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 1526defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 1527defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 1528defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 1529defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 1530defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 1531defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 1532defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 1533defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 1534defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 1535defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 1536defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 1537defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 1538defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 1539defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 1540defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 1541defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 1542defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 1543defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 1544defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 1545defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 1546defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 1547 1548let SubtargetPredicate = HasAtomicFaddInsts in { 1549 1550defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Real_AllAddr_vi <0x04d>; 1551defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Real_AllAddr_vi <0x04e>; 1552 1553} // End SubtargetPredicate = HasAtomicFaddInsts 1554