1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FLATOffset : ComplexPattern<i64, 2, "SelectFlatOffset<false>", [], [SDNPWantRoot], -10>; 10def FLATOffsetSigned : ComplexPattern<i64, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 11 12def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 13 14//===----------------------------------------------------------------------===// 15// FLAT classes 16//===----------------------------------------------------------------------===// 17 18class FLAT_Pseudo<string opName, dag outs, dag ins, 19 string asmOps, list<dag> pattern=[]> : 20 InstSI<outs, ins, "", pattern>, 21 SIMCInstr<opName, SIEncodingFamily.NONE> { 22 23 let isPseudo = 1; 24 let isCodeGenOnly = 1; 25 26 let FLAT = 1; 27 28 let UseNamedOperandTable = 1; 29 let hasSideEffects = 0; 30 let SchedRW = [WriteVMEM]; 31 32 string Mnemonic = opName; 33 string AsmOperands = asmOps; 34 35 bits<1> is_flat_global = 0; 36 bits<1> is_flat_scratch = 0; 37 38 bits<1> has_vdst = 1; 39 40 // We need to distinguish having saddr and enabling saddr because 41 // saddr is only valid for scratch and global instructions. Pre-gfx9 42 // these bits were reserved, so we also don't necessarily want to 43 // set these bits to the disabled value for the original flat 44 // segment instructions. 45 bits<1> has_saddr = 0; 46 bits<1> enabled_saddr = 0; 47 bits<7> saddr_value = 0; 48 bits<1> has_vaddr = 1; 49 50 bits<1> has_data = 1; 51 bits<1> has_glc = 1; 52 bits<1> glcValue = 0; 53 bits<1> has_dlc = 1; 54 bits<1> dlcValue = 0; 55 56 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 57 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 58 59 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 60 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 61 62 // Internally, FLAT instruction are executed as both an LDS and a 63 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 64 // and are not considered done until both have been decremented. 65 let VM_CNT = 1; 66 let LGKM_CNT = !if(!or(is_flat_global, is_flat_scratch), 0, 1); 67 68 let IsNonFlatSeg = !if(!or(is_flat_global, is_flat_scratch), 1, 0); 69} 70 71class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 72 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 73 Enc64 { 74 75 let isPseudo = 0; 76 let isCodeGenOnly = 0; 77 78 // copy relevant pseudo op flags 79 let SubtargetPredicate = ps.SubtargetPredicate; 80 let AsmMatchConverter = ps.AsmMatchConverter; 81 let TSFlags = ps.TSFlags; 82 let UseNamedOperandTable = ps.UseNamedOperandTable; 83 84 // encoding fields 85 bits<8> vaddr; 86 bits<8> vdata; 87 bits<7> saddr; 88 bits<8> vdst; 89 90 bits<1> slc; 91 bits<1> glc; 92 bits<1> dlc; 93 94 // Only valid on gfx9 95 bits<1> lds = 0; // XXX - What does this actually do? 96 97 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 98 bits<2> seg = !if(ps.is_flat_global, 0b10, 99 !if(ps.is_flat_scratch, 0b01, 0)); 100 101 // Signed offset. Highest bit ignored for flat and treated as 12-bit 102 // unsigned for flat accesses. 103 bits<13> offset; 104 bits<1> nv = 0; // XXX - What does this actually do? 105 106 // We don't use tfe right now, and it was removed in gfx9. 107 bits<1> tfe = 0; 108 109 // Only valid on GFX9+ 110 let Inst{12-0} = offset; 111 let Inst{13} = lds; 112 let Inst{15-14} = seg; 113 114 let Inst{16} = !if(ps.has_glc, glc, ps.glcValue); 115 let Inst{17} = slc; 116 let Inst{24-18} = op; 117 let Inst{31-26} = 0x37; // Encoding. 118 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 119 let Inst{47-40} = !if(ps.has_data, vdata, ?); 120 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 121 122 // 54-48 is reserved. 123 let Inst{55} = nv; // nv on GFX9+, TFE before. 124 let Inst{63-56} = !if(ps.has_vdst, vdst, ?); 125} 126 127class GlobalSaddrTable <bit is_saddr, string Name = ""> { 128 bit IsSaddr = is_saddr; 129 string SaddrOp = Name; 130} 131 132// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 133// same encoding value as exec_hi, so it isn't possible to use that if 134// saddr is 32-bit (which isn't handled here yet). 135class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 136 bit HasTiedOutput = 0, 137 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 138 opName, 139 (outs regClass:$vdst), 140 !con( 141 !con( 142 !if(EnableSaddr, 143 (ins SReg_64:$saddr, VGPR_32:$vaddr), 144 (ins VReg_64:$vaddr)), 145 (ins flat_offset:$offset)), 146 // FIXME: Operands with default values do not work with following non-optional operands. 147 !if(HasTiedOutput, (ins GLC:$glc, SLC:$slc, DLC:$dlc, regClass:$vdst_in), 148 (ins GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc))), 149 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc"> { 150 let has_data = 0; 151 let mayLoad = 1; 152 let has_saddr = HasSaddr; 153 let enabled_saddr = EnableSaddr; 154 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 155 let maybeAtomic = 1; 156 157 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 158 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 159} 160 161class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 162 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 163 opName, 164 (outs), 165 !con( 166 !if(EnableSaddr, 167 (ins VGPR_32:$vaddr, vdataClass:$vdata, SReg_64:$saddr), 168 (ins VReg_64:$vaddr, vdataClass:$vdata)), 169 (ins flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc)), 170 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc"> { 171 let mayLoad = 0; 172 let mayStore = 1; 173 let has_vdst = 0; 174 let has_saddr = HasSaddr; 175 let enabled_saddr = EnableSaddr; 176 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 177 let maybeAtomic = 1; 178} 179 180multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 181 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 182 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 183 GlobalSaddrTable<0, opName>; 184 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 185 GlobalSaddrTable<1, opName>; 186 } 187} 188 189class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 190 bit HasTiedOutput = 0, bit HasSignedOffset = 0> : FLAT_Pseudo< 191 opName, 192 (outs regClass:$vdst), 193 !con((ins SReg_64:$saddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc), 194 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 195 " $vdst, $saddr$offset$glc$slc$dlc"> { 196 let is_flat_global = 1; 197 let has_data = 0; 198 let mayLoad = 1; 199 let has_vaddr = 0; 200 let has_saddr = 1; 201 let enabled_saddr = 1; 202 let maybeAtomic = 1; 203 204 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 205 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 206} 207 208multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 209 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 210 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 211 GlobalSaddrTable<0, opName>; 212 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 213 GlobalSaddrTable<1, opName>; 214 } 215} 216 217class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 218 bit HasSignedOffset = 0> : FLAT_Pseudo< 219 opName, 220 (outs), 221 !con( 222 (ins vdataClass:$vdata, SReg_64:$saddr), 223 (ins flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc)), 224 " $vdata, $saddr$offset$glc$slc$dlc"> { 225 let is_flat_global = 1; 226 let mayLoad = 0; 227 let mayStore = 1; 228 let has_vdst = 0; 229 let has_vaddr = 0; 230 let has_saddr = 1; 231 let enabled_saddr = 1; 232 let maybeAtomic = 1; 233} 234 235class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 236 bit EnableSaddr = 0>: FLAT_Pseudo< 237 opName, 238 (outs regClass:$vdst), 239 !if(EnableSaddr, 240 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc), 241 (ins VGPR_32:$vaddr, flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc)), 242 " $vdst, "#!if(EnableSaddr, "off", "$vaddr")#!if(EnableSaddr, ", $saddr", ", off")#"$offset$glc$slc$dlc"> { 243 let has_data = 0; 244 let mayLoad = 1; 245 let has_saddr = 1; 246 let enabled_saddr = EnableSaddr; 247 let has_vaddr = !if(EnableSaddr, 0, 1); 248 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 249 let maybeAtomic = 1; 250} 251 252class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0> : FLAT_Pseudo< 253 opName, 254 (outs), 255 !if(EnableSaddr, 256 (ins vdataClass:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc), 257 (ins vdataClass:$vdata, VGPR_32:$vaddr, flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc)), 258 " "#!if(EnableSaddr, "off", "$vaddr")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc"> { 259 let mayLoad = 0; 260 let mayStore = 1; 261 let has_vdst = 0; 262 let has_saddr = 1; 263 let enabled_saddr = EnableSaddr; 264 let has_vaddr = !if(EnableSaddr, 0, 1); 265 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 266 let maybeAtomic = 1; 267} 268 269multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass> { 270 let is_flat_scratch = 1 in { 271 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass>; 272 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, 1>; 273 } 274} 275 276multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 277 let is_flat_scratch = 1 in { 278 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>; 279 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>; 280 } 281} 282 283class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 284 string asm, list<dag> pattern = []> : 285 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 286 let mayLoad = 1; 287 let mayStore = 1; 288 let has_glc = 0; 289 let glcValue = 0; 290 let has_dlc = 0; 291 let dlcValue = 0; 292 let has_vdst = 0; 293 let maybeAtomic = 1; 294} 295 296class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 297 string asm, list<dag> pattern = []> 298 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 299 let hasPostISelHook = 1; 300 let has_vdst = 1; 301 let glcValue = 1; 302 let dlcValue = 0; 303 let PseudoInstr = NAME # "_RTN"; 304} 305 306multiclass FLAT_Atomic_Pseudo< 307 string opName, 308 RegisterClass vdst_rc, 309 ValueType vt, 310 SDPatternOperator atomic = null_frag, 311 ValueType data_vt = vt, 312 RegisterClass data_rc = vdst_rc, 313 bit isFP = isFloatType<data_vt>.ret> { 314 def "" : FLAT_AtomicNoRet_Pseudo <opName, 315 (outs), 316 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 317 " $vaddr, $vdata$offset$slc">, 318 GlobalSaddrTable<0, opName>, 319 AtomicNoRet <opName, 0> { 320 let PseudoInstr = NAME; 321 let FPAtomic = isFP; 322 let AddedComplexity = -1; // Prefer global atomics if available 323 } 324 325 def _RTN : FLAT_AtomicRet_Pseudo <opName, 326 (outs vdst_rc:$vdst), 327 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 328 " $vdst, $vaddr, $vdata$offset glc$slc", 329 [(set vt:$vdst, 330 (atomic (FLATOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 331 GlobalSaddrTable<0, opName#"_rtn">, 332 AtomicNoRet <opName, 1>{ 333 let FPAtomic = isFP; 334 let AddedComplexity = -1; // Prefer global atomics if available 335 } 336} 337 338multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 339 string opName, 340 RegisterClass vdst_rc, 341 ValueType vt, 342 SDPatternOperator atomic = null_frag, 343 ValueType data_vt = vt, 344 RegisterClass data_rc = vdst_rc, 345 bit isFP = isFloatType<data_vt>.ret> { 346 347 def "" : FLAT_AtomicNoRet_Pseudo <opName, 348 (outs), 349 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 350 " $vaddr, $vdata, off$offset$slc">, 351 GlobalSaddrTable<0, opName>, 352 AtomicNoRet <opName, 0> { 353 let has_saddr = 1; 354 let PseudoInstr = NAME; 355 let FPAtomic = isFP; 356 } 357 358 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 359 (outs), 360 (ins VGPR_32:$vaddr, data_rc:$vdata, SReg_64:$saddr, flat_offset:$offset, SLC_0:$slc), 361 " $vaddr, $vdata, $saddr$offset$slc">, 362 GlobalSaddrTable<1, opName>, 363 AtomicNoRet <opName#"_saddr", 0> { 364 let has_saddr = 1; 365 let enabled_saddr = 1; 366 let PseudoInstr = NAME#"_SADDR"; 367 let FPAtomic = isFP; 368 } 369} 370 371multiclass FLAT_Global_Atomic_Pseudo_RTN< 372 string opName, 373 RegisterClass vdst_rc, 374 ValueType vt, 375 SDPatternOperator atomic = null_frag, 376 ValueType data_vt = vt, 377 RegisterClass data_rc = vdst_rc, 378 bit isFP = isFloatType<data_vt>.ret> { 379 380 def _RTN : FLAT_AtomicRet_Pseudo <opName, 381 (outs vdst_rc:$vdst), 382 (ins VReg_64:$vaddr, data_rc:$vdata, flat_offset:$offset, SLC_0:$slc), 383 " $vdst, $vaddr, $vdata, off$offset glc$slc", 384 [(set vt:$vdst, 385 (atomic (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 386 GlobalSaddrTable<0, opName#"_rtn">, 387 AtomicNoRet <opName, 1> { 388 let has_saddr = 1; 389 let FPAtomic = isFP; 390 } 391 392 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 393 (outs vdst_rc:$vdst), 394 (ins VGPR_32:$vaddr, data_rc:$vdata, SReg_64:$saddr, flat_offset:$offset, SLC_0:$slc), 395 " $vdst, $vaddr, $vdata, $saddr$offset glc$slc">, 396 GlobalSaddrTable<1, opName#"_rtn">, 397 AtomicNoRet <opName#"_saddr", 1> { 398 let has_saddr = 1; 399 let enabled_saddr = 1; 400 let PseudoInstr = NAME#"_SADDR_RTN"; 401 let FPAtomic = isFP; 402 } 403} 404 405multiclass FLAT_Global_Atomic_Pseudo< 406 string opName, 407 RegisterClass vdst_rc, 408 ValueType vt, 409 SDPatternOperator atomic_rtn = null_frag, 410 SDPatternOperator atomic_no_rtn = null_frag, 411 ValueType data_vt = vt, 412 RegisterClass data_rc = vdst_rc> { 413 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 414 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>; 415 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; 416 } 417} 418 419//===----------------------------------------------------------------------===// 420// Flat Instructions 421//===----------------------------------------------------------------------===// 422 423def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 424def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 425def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 426def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 427def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 428def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 429def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 430def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 431 432def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 433def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 434def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 435def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 436def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 437def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 438 439let SubtargetPredicate = HasD16LoadStore in { 440def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 441def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 442def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 443def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 444def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 445def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 446 447def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 448def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 449} 450 451defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 452 VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, 453 v2i32, VReg_64>; 454 455defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 456 VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, 457 v2i64, VReg_128>; 458 459defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 460 VGPR_32, i32, atomic_swap_flat_32>; 461 462defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 463 VReg_64, i64, atomic_swap_flat_64>; 464 465defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 466 VGPR_32, i32, atomic_load_add_flat_32>; 467 468defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 469 VGPR_32, i32, atomic_load_sub_flat_32>; 470 471defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 472 VGPR_32, i32, atomic_load_min_flat_32>; 473 474defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 475 VGPR_32, i32, atomic_load_umin_flat_32>; 476 477defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 478 VGPR_32, i32, atomic_load_max_flat_32>; 479 480defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 481 VGPR_32, i32, atomic_load_umax_flat_32>; 482 483defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 484 VGPR_32, i32, atomic_load_and_flat_32>; 485 486defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 487 VGPR_32, i32, atomic_load_or_flat_32>; 488 489defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 490 VGPR_32, i32, atomic_load_xor_flat_32>; 491 492defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 493 VGPR_32, i32, atomic_inc_flat_32>; 494 495defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 496 VGPR_32, i32, atomic_dec_flat_32>; 497 498defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 499 VReg_64, i64, atomic_load_add_flat_64>; 500 501defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 502 VReg_64, i64, atomic_load_sub_flat_64>; 503 504defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 505 VReg_64, i64, atomic_load_min_flat_64>; 506 507defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 508 VReg_64, i64, atomic_load_umin_flat_64>; 509 510defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 511 VReg_64, i64, atomic_load_max_flat_64>; 512 513defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 514 VReg_64, i64, atomic_load_umax_flat_64>; 515 516defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 517 VReg_64, i64, atomic_load_and_flat_64>; 518 519defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 520 VReg_64, i64, atomic_load_or_flat_64>; 521 522defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 523 VReg_64, i64, atomic_load_xor_flat_64>; 524 525defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 526 VReg_64, i64, atomic_inc_flat_64>; 527 528defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 529 VReg_64, i64, atomic_dec_flat_64>; 530 531// GFX7-, GFX10-only flat instructions. 532let SubtargetPredicate = isGFX7GFX10 in { 533 534defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 535 VGPR_32, f32, null_frag, v2f32, VReg_64>; 536 537defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 538 VReg_64, f64, null_frag, v2f64, VReg_128>; 539 540defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 541 VGPR_32, f32>; 542 543defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 544 VGPR_32, f32>; 545 546defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 547 VReg_64, f64>; 548 549defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 550 VReg_64, f64>; 551 552} // End SubtargetPredicate = isGFX7GFX10 553 554defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 555defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 556defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 557defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 558defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 559defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 560defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 561defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 562 563defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 564defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 565defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 566defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 567defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 568defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 569let OtherPredicates = [HasGFX10_BEncoding] in 570def GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 571 572defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 573defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 574defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 575defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 576defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 577defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 578let OtherPredicates = [HasGFX10_BEncoding] in 579def GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 580 581defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 582defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 583 584let is_flat_global = 1 in { 585defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 586 VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag, 587 v2i32, VReg_64>; 588 589defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 590 VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, 591 null_frag, 592 v2i64, VReg_128>; 593 594defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 595 VGPR_32, i32, atomic_swap_global_32>; 596 597defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 598 VReg_64, i64, atomic_swap_global_64>; 599 600defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 601 VGPR_32, i32, atomic_load_add_global_32>; 602 603defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 604 VGPR_32, i32, atomic_load_sub_global_32>; 605 606defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 607 VGPR_32, i32, atomic_load_min_global_32>; 608 609defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 610 VGPR_32, i32, atomic_load_umin_global_32>; 611 612defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 613 VGPR_32, i32, atomic_load_max_global_32>; 614 615defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 616 VGPR_32, i32, atomic_load_umax_global_32>; 617 618defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 619 VGPR_32, i32, atomic_load_and_global_32>; 620 621defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 622 VGPR_32, i32, atomic_load_or_global_32>; 623 624defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 625 VGPR_32, i32, atomic_load_xor_global_32>; 626 627defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 628 VGPR_32, i32, atomic_inc_global_32>; 629 630defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 631 VGPR_32, i32, atomic_dec_global_32>; 632 633defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 634 VReg_64, i64, atomic_load_add_global_64>; 635 636defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 637 VReg_64, i64, atomic_load_sub_global_64>; 638 639defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 640 VReg_64, i64, atomic_load_min_global_64>; 641 642defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 643 VReg_64, i64, atomic_load_umin_global_64>; 644 645defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 646 VReg_64, i64, atomic_load_max_global_64>; 647 648defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 649 VReg_64, i64, atomic_load_umax_global_64>; 650 651defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 652 VReg_64, i64, atomic_load_and_global_64>; 653 654defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 655 VReg_64, i64, atomic_load_or_global_64>; 656 657defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 658 VReg_64, i64, atomic_load_xor_global_64>; 659 660defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 661 VReg_64, i64, atomic_inc_global_64>; 662 663defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 664 VReg_64, i64, atomic_dec_global_64>; 665 666let SubtargetPredicate = HasGFX10_BEncoding in 667defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 668 VGPR_32, i32, int_amdgcn_global_atomic_csub>; 669} // End is_flat_global = 1 670 671 672 673let SubtargetPredicate = HasFlatScratchInsts in { 674defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 675defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 676defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 677defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 678defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 679defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 680defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 681defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 682 683defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32>; 684defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>; 685defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32>; 686defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>; 687defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32>; 688defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>; 689 690defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 691defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 692defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 693defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 694defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 695defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 696 697defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 698defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 699 700} // End SubtargetPredicate = HasFlatScratchInsts 701 702let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 703 defm GLOBAL_ATOMIC_FCMPSWAP : 704 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32>; 705 defm GLOBAL_ATOMIC_FMIN : 706 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 707 defm GLOBAL_ATOMIC_FMAX : 708 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 709 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 710 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>; 711 defm GLOBAL_ATOMIC_FMIN_X2 : 712 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 713 defm GLOBAL_ATOMIC_FMAX_X2 : 714 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 715} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 716 717let SubtargetPredicate = HasAtomicFaddInsts, is_flat_global = 1 in { 718 719defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 720 "global_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret 721>; 722defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 723 "global_atomic_pk_add_f16", VGPR_32, v2f16, atomic_fadd_global_noret 724>; 725 726} // End SubtargetPredicate = HasAtomicFaddInsts 727 728//===----------------------------------------------------------------------===// 729// Flat Patterns 730//===----------------------------------------------------------------------===// 731 732// Patterns for global loads with no offset. 733class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 734 (vt (node (FLATOffset i64:$vaddr, i16:$offset))), 735 (inst $vaddr, $offset) 736>; 737 738class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 739 (node (FLATOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 740 (inst $vaddr, $offset, 0, 0, 0, $in) 741>; 742 743class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 744 (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 745 (inst $vaddr, $offset, 0, 0, 0, $in) 746>; 747 748class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 749 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 750 (inst $saddr, $voffset, $offset, 0, 0, 0, $in) 751>; 752 753class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 754 (vt (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset))), 755 (inst $vaddr, $offset) 756>; 757 758class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 759 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 760 (inst $saddr, $voffset, $offset, 0, 0, 0) 761>; 762 763class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 764 ValueType vt> : GCNPat < 765 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 766 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 767>; 768 769class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 770 ValueType vt> : GCNPat < 771 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 772 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 773>; 774 775class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 776 ValueType vt, ValueType data_vt = vt> : GCNPat < 777 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 778 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 779>; 780 781class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 782 ValueType vt> : GCNPat < 783 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 784 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 785>; 786 787class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 788 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset)), 789 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 790>; 791 792class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 793 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset)), 794 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 795>; 796 797class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 798 // atomic store follows atomic binop convention so the address comes 799 // first. 800 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 801 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 802>; 803 804class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 805 ValueType vt, ValueType data_vt = vt> : GCNPat < 806 // atomic store follows atomic binop convention so the address comes 807 // first. 808 (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data), 809 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 810>; 811 812class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 813 ValueType data_vt = vt> : GCNPat < 814 (vt (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data)), 815 (inst $vaddr, $data, $offset) 816>; 817 818class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 819 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 820 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 821>; 822 823class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 824 (node (FLATOffsetSigned i64:$vaddr, i16:$offset), vt:$data), 825 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 826>; 827 828class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 829 ValueType data_vt = vt> : GCNPat < 830 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$data)), 831 (inst $vaddr, $data, $offset) 832>; 833 834let OtherPredicates = [HasFlatAddressSpace] in { 835 836def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 837def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 838def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 839def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 840def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 841def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 842def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 843def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 844def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 845def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 846def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 847 848def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 849def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 850 851def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 852def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 853 854foreach vt = Reg32Types.types in { 855def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 856def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 857} 858 859foreach vt = VReg_64.RegTypes in { 860def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 861def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 862} 863 864def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 865 866foreach vt = VReg_128.RegTypes in { 867def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 868def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 869} 870 871def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; 872def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64>; 873 874def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_load_add_global_32, i32>; 875def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_load_sub_global_32, i32>; 876def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global_32, i32>; 877def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global_32, i32>; 878def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_load_and_global_32, i32>; 879def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_load_max_global_32, i32>; 880def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_load_umax_global_32, i32>; 881def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i32>; 882def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; 883def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; 884def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; 885def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 886def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; 887 888def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; 889def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_load_sub_global_64, i64>; 890def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global_64, i64>; 891def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global_64, i64>; 892def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_load_and_global_64, i64>; 893def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_load_max_global_64, i64>; 894def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_load_umax_global_64, i64>; 895def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, i64>; 896def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; 897def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; 898def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; 899def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 900def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; 901 902def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 903def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 904 905let OtherPredicates = [D16PreservesUnusedBits] in { 906def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 907def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 908 909def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 910def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 911def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 912def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 913def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 914def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 915 916def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 917def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 918def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 919def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 920def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 921def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 922} 923 924} // End OtherPredicates = [HasFlatAddressSpace] 925 926 927multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 928 def : FlatLoadSignedPat <inst, node, vt> { 929 let AddedComplexity = 10; 930 } 931 932 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 933 let AddedComplexity = 11; 934 } 935} 936 937multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 938 def : FlatSignedLoadPat_D16 <inst, node, vt> { 939 let AddedComplexity = 10; 940 } 941 942 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 943 let AddedComplexity = 11; 944 } 945} 946 947multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 948 ValueType vt> { 949 def : FlatStoreSignedPat <inst, node, vt> { 950 let AddedComplexity = 10; 951 } 952 953 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 954 let AddedComplexity = 11; 955 } 956} 957 958// Deal with swapped operands for atomic_store vs. regular store 959multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 960 def : FlatStoreSignedAtomicPat <inst, node, vt> { 961 let AddedComplexity = 10; 962 } 963 964 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 965 let AddedComplexity = 11; 966 } 967} 968 969multiclass GlobalFLATAtomicPats<string nortn_inst_name, SDPatternOperator node, 970 ValueType vt, ValueType data_vt = vt> { 971 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 972 let AddedComplexity = 10; 973 } 974 975 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 976 let AddedComplexity = 11; 977 } 978} 979 980multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 981 ValueType vt> { 982 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 983 let AddedComplexity = 10; 984 } 985 986 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 987 let AddedComplexity = 11; 988 } 989} 990 991let OtherPredicates = [HasFlatGlobalInsts] in { 992 993defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 994defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 995defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 996defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 997defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 998defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 999defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1000defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1001defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1002defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1003 1004foreach vt = Reg32Types.types in { 1005defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1006defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1007} 1008 1009foreach vt = VReg_64.RegTypes in { 1010defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1011defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1012} 1013 1014defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1015 1016foreach vt = VReg_128.RegTypes in { 1017defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1018defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1019} 1020 1021// There is no distinction for atomic load lowering during selection; 1022// the memory legalizer will set the cache bits and insert the 1023// appropriate waits. 1024defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1025defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1026 1027defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1028defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1029defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1030defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1031defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1032 1033let OtherPredicates = [D16PreservesUnusedBits] in { 1034defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1035defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1036 1037defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1038defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1039defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1040defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1041defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1042defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1043 1044defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1045defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1046defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1047defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1048defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1049defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1050} 1051 1052defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; 1053defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64>; 1054 1055defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", atomic_load_add_global_32, i32>; 1056defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", atomic_load_sub_global_32, i32>; 1057defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", atomic_inc_global_32, i32>; 1058defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", atomic_dec_global_32, i32>; 1059defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", atomic_load_and_global_32, i32>; 1060defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", atomic_load_max_global_32, i32>; 1061defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", atomic_load_umax_global_32, i32>; 1062defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", atomic_load_min_global_32, i32>; 1063defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", atomic_load_umin_global_32, i32>; 1064defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", atomic_load_or_global_32, i32>; 1065defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", atomic_swap_global_32, i32>; 1066defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1067defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", atomic_load_xor_global_32, i32>; 1068defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1069 1070defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", atomic_load_add_global_64, i64>; 1071defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", atomic_load_sub_global_64, i64>; 1072defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", atomic_inc_global_64, i64>; 1073defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", atomic_dec_global_64, i64>; 1074defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", atomic_load_and_global_64, i64>; 1075defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", atomic_load_max_global_64, i64>; 1076defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", atomic_load_umax_global_64, i64>; 1077defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", atomic_load_min_global_64, i64>; 1078defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", atomic_load_umin_global_64, i64>; 1079defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", atomic_load_or_global_64, i64>; 1080defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", atomic_swap_global_64, i64>; 1081defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1082defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", atomic_load_xor_global_64, i64>; 1083 1084defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_fadd_global_noret, f32>; 1085defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_fadd_global_noret, v2f16>; 1086 1087} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1088 1089 1090//===----------------------------------------------------------------------===// 1091// Target 1092//===----------------------------------------------------------------------===// 1093 1094//===----------------------------------------------------------------------===// 1095// CI 1096//===----------------------------------------------------------------------===// 1097 1098class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1099 FLAT_Real <op, ps>, 1100 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1101 let AssemblerPredicate = isGFX7Only; 1102 let DecoderNamespace="GFX7"; 1103} 1104 1105def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1106def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1107def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1108def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1109def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1110def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1111def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1112def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1113 1114def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1115def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1116def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1117def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1118def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1119def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1120 1121multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1122 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1123 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1124} 1125 1126defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1127defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1128defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1129defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1130defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1131defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1132defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1133defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1134defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1135defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1136defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1137defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1138defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1139defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1140defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1141defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1142defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1143defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1144defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1145defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1146defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1147defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1148defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1149defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1150defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1151defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1152 1153// CI Only flat instructions 1154defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1155defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1156defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1157defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1158defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1159defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1160 1161 1162//===----------------------------------------------------------------------===// 1163// VI 1164//===----------------------------------------------------------------------===// 1165 1166class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps> : 1167 FLAT_Real <op, ps>, 1168 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1169 let AssemblerPredicate = isGFX8GFX9; 1170 let DecoderNamespace = "GFX8"; 1171} 1172 1173multiclass FLAT_Real_AllAddr_vi<bits<7> op> { 1174 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)>; 1175 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1176} 1177 1178def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1179def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1180def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1181def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1182def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1183def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1184def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1185def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1186 1187def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1188def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1189def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1190def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1191def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1192def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1193def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1194def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1195 1196def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1197def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1198def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1199def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1200def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1201def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1202 1203multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> { 1204 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1205 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1206} 1207 1208multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> : 1209 FLAT_Real_AllAddr_vi<op> { 1210 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1211 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1212} 1213 1214 1215defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1216defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1217defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1218defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1219defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1220defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1221defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1222defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1223defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1224defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1225defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1226defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1227defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1228defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1229defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1230defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1231defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1232defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1233defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1234defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1235defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1236defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1237defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1238defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1239defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1240defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1241 1242defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1243defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1244defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1245defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1246defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1247defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1248defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1249defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1250 1251defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1252defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1253defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1254defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1255defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1256defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1257 1258defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1259defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1260defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1261defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1262defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1263defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1264defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1265defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1266 1267 1268defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1269defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1270defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1271defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1272defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1273defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1274defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1275defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1276defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1277defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1278defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1279defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1280defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1281defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1282defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1283defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1284defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1285defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1286defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1287defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1288defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1289defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1290defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1291defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1292defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1293defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1294 1295defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1296defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1297defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1298defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1299defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1300defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1301defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1302defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1303defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1304defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1305defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1306defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1307defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1308defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1309defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1310defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1311defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1312defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1313defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1314defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1315defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1316defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1317 1318 1319//===----------------------------------------------------------------------===// 1320// GFX10. 1321//===----------------------------------------------------------------------===// 1322 1323class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1324 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1325 let AssemblerPredicate = isGFX10Plus; 1326 let DecoderNamespace = "GFX10"; 1327 1328 let Inst{11-0} = offset{11-0}; 1329 let Inst{12} = !if(ps.has_dlc, dlc, ps.dlcValue); 1330 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1331 let Inst{55} = 0; 1332} 1333 1334 1335multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1336 def _gfx10 : 1337 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1338} 1339 1340multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1341 def _RTN_gfx10 : 1342 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1343} 1344 1345multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1346 def _SADDR_gfx10 : 1347 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1348} 1349 1350multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1351 def _SADDR_RTN_gfx10 : 1352 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1353} 1354 1355 1356multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1357 FLAT_Real_Base_gfx10<op>, 1358 FLAT_Real_SADDR_gfx10<op>; 1359 1360multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1361 FLAT_Real_Base_gfx10<op>, 1362 FLAT_Real_RTN_gfx10<op>; 1363 1364multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1365 FLAT_Real_AllAddr_gfx10<op>, 1366 FLAT_Real_RTN_gfx10<op>, 1367 FLAT_Real_SADDR_RTN_gfx10<op>; 1368 1369multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1370 FLAT_Real_RTN_gfx10<op>, 1371 FLAT_Real_SADDR_RTN_gfx10<op>; 1372 1373// ENC_FLAT. 1374defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1375defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1376defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1377defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1378defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1379defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1380defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1381defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1382defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1383defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1384defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1385defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1386defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1387defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1388defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1389defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1390defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1391defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1392defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1393defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1394defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1395defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1396defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1397defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1398defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1399defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1400defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1401defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1402defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1403defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1404defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1405defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1406defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1407defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1408defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1409defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1410defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1411defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1412defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1413defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1414defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1415defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1416defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1417defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1418defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1419defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1420defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1421defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1422defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1423defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1424defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1425defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1426defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1427defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1428 1429 1430// ENC_FLAT_GLBL. 1431defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1432defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1433defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1434defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1435defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1436defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1437defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1438defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1439defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1440defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1441defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1442defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1443defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1444defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1445defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1446defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1447defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1448defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1449defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1450defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1451defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1452defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1453defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 1454defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 1455defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 1456defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 1457defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 1458defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 1459defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 1460defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 1461defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 1462defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 1463defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 1464defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 1465defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 1466defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 1467defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 1468defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 1469defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 1470defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 1471defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 1472defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 1473defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 1474defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 1475defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 1476defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 1477defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 1478defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 1479defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 1480defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 1481defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 1482defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 1483defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 1484defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 1485defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 1486defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_Base_gfx10<0x016>; 1487defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_Base_gfx10<0x017>; 1488 1489// ENC_FLAT_SCRATCH. 1490defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1491defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1492defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1493defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1494defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1495defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1496defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1497defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1498defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1499defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1500defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1501defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1502defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1503defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1504defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1505defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1506defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1507defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1508defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1509defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1510defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1511defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1512 1513let SubtargetPredicate = HasAtomicFaddInsts in { 1514 1515defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Real_AllAddr_vi <0x04d>; 1516defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Real_AllAddr_vi <0x04e>; 1517 1518} // End SubtargetPredicate = HasAtomicFaddInsts 1519