1//===-- FLATInstructions.td - FLAT Instruction Defintions -----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def FLATAtomic : ComplexPattern<i64, 3, "SelectFlatAtomic", [], [], -10>; 11def FLATOffset : ComplexPattern<i64, 3, "SelectFlatOffset<false>", [], [], -10>; 12 13def FLATOffsetSigned : ComplexPattern<i64, 3, "SelectFlatOffset<true>", [], [], -10>; 14def FLATSignedAtomic : ComplexPattern<i64, 3, "SelectFlatAtomicSigned", [], [], -10>; 15 16//===----------------------------------------------------------------------===// 17// FLAT classes 18//===----------------------------------------------------------------------===// 19 20class FLAT_Pseudo<string opName, dag outs, dag ins, 21 string asmOps, list<dag> pattern=[]> : 22 InstSI<outs, ins, "", pattern>, 23 SIMCInstr<opName, SIEncodingFamily.NONE> { 24 25 let isPseudo = 1; 26 let isCodeGenOnly = 1; 27 28 let FLAT = 1; 29 30 let UseNamedOperandTable = 1; 31 let hasSideEffects = 0; 32 let SchedRW = [WriteVMEM]; 33 34 string Mnemonic = opName; 35 string AsmOperands = asmOps; 36 37 bits<1> is_flat_global = 0; 38 bits<1> is_flat_scratch = 0; 39 40 bits<1> has_vdst = 1; 41 42 // We need to distinguish having saddr and enabling saddr because 43 // saddr is only valid for scratch and global instructions. Pre-gfx9 44 // these bits were reserved, so we also don't necessarily want to 45 // set these bits to the disabled value for the original flat 46 // segment instructions. 47 bits<1> has_saddr = 0; 48 bits<1> enabled_saddr = 0; 49 bits<7> saddr_value = 0; 50 bits<1> has_vaddr = 1; 51 52 bits<1> has_data = 1; 53 bits<1> has_glc = 1; 54 bits<1> glcValue = 0; 55 56 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 57 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 58 59 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 60 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 61 62 // Internally, FLAT instruction are executed as both an LDS and a 63 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 64 // and are not considered done until both have been decremented. 65 let VM_CNT = 1; 66 let LGKM_CNT = !if(!or(is_flat_global, is_flat_scratch), 0, 1); 67} 68 69class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 70 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 71 Enc64 { 72 73 let isPseudo = 0; 74 let isCodeGenOnly = 0; 75 76 // copy relevant pseudo op flags 77 let SubtargetPredicate = ps.SubtargetPredicate; 78 let AsmMatchConverter = ps.AsmMatchConverter; 79 let TSFlags = ps.TSFlags; 80 let UseNamedOperandTable = ps.UseNamedOperandTable; 81 82 // encoding fields 83 bits<8> vaddr; 84 bits<8> vdata; 85 bits<7> saddr; 86 bits<8> vdst; 87 88 bits<1> slc; 89 bits<1> glc; 90 91 // Only valid on gfx9 92 bits<1> lds = 0; // XXX - What does this actually do? 93 94 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 95 bits<2> seg = !if(ps.is_flat_global, 0b10, 96 !if(ps.is_flat_scratch, 0b01, 0)); 97 98 // Signed offset. Highest bit ignored for flat and treated as 12-bit 99 // unsigned for flat acceses. 100 bits<13> offset; 101 bits<1> nv = 0; // XXX - What does this actually do? 102 103 // We don't use tfe right now, and it was removed in gfx9. 104 bits<1> tfe = 0; 105 106 // Only valid on GFX9+ 107 let Inst{12-0} = offset; 108 let Inst{13} = lds; 109 let Inst{15-14} = seg; 110 111 let Inst{16} = !if(ps.has_glc, glc, ps.glcValue); 112 let Inst{17} = slc; 113 let Inst{24-18} = op; 114 let Inst{31-26} = 0x37; // Encoding. 115 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 116 let Inst{47-40} = !if(ps.has_data, vdata, ?); 117 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 118 119 // 54-48 is reserved. 120 let Inst{55} = nv; // nv on GFX9+, TFE before. 121 let Inst{63-56} = !if(ps.has_vdst, vdst, ?); 122} 123 124class GlobalSaddrTable <bit is_saddr, string Name = ""> { 125 bit IsSaddr = is_saddr; 126 string SaddrOp = Name; 127} 128 129// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 130// same encoding value as exec_hi, so it isn't possible to use that if 131// saddr is 32-bit (which isn't handled here yet). 132class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 133 bit HasTiedOutput = 0, 134 bit HasSignedOffset = 0, bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 135 opName, 136 (outs regClass:$vdst), 137 !con( 138 !con( 139 !con( 140 !con((ins VReg_64:$vaddr), 141 !if(EnableSaddr, (ins SReg_64:$saddr), (ins))), 142 (ins !if(HasSignedOffset,offset_s13,offset_u12):$offset)), 143 (ins GLC:$glc, SLC:$slc)), 144 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 145 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc"> { 146 let has_data = 0; 147 let mayLoad = 1; 148 let has_saddr = HasSaddr; 149 let enabled_saddr = EnableSaddr; 150 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 151 let maybeAtomic = 1; 152 153 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 154 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 155} 156 157class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 158 bit HasSignedOffset = 0, bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 159 opName, 160 (outs), 161 !con( 162 !con( 163 !con((ins VReg_64:$vaddr, vdataClass:$vdata), 164 !if(EnableSaddr, (ins SReg_64:$saddr), (ins))), 165 (ins !if(HasSignedOffset,offset_s13,offset_u12):$offset)), 166 (ins GLC:$glc, SLC:$slc)), 167 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc"> { 168 let mayLoad = 0; 169 let mayStore = 1; 170 let has_vdst = 0; 171 let has_saddr = HasSaddr; 172 let enabled_saddr = EnableSaddr; 173 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 174 let maybeAtomic = 1; 175} 176 177multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 178 let is_flat_global = 1 in { 179 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 180 GlobalSaddrTable<0, opName>; 181 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1, 1>, 182 GlobalSaddrTable<1, opName>; 183 } 184} 185 186multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 187 let is_flat_global = 1 in { 188 def "" : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 189 GlobalSaddrTable<0, opName>; 190 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1, 1>, 191 GlobalSaddrTable<1, opName>; 192 } 193} 194 195class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 196 bit EnableSaddr = 0>: FLAT_Pseudo< 197 opName, 198 (outs regClass:$vdst), 199 !if(EnableSaddr, 200 (ins SReg_32_XEXEC_HI:$saddr, offset_s13:$offset, GLC:$glc, SLC:$slc), 201 (ins VGPR_32:$vaddr, offset_s13:$offset, GLC:$glc, SLC:$slc)), 202 " $vdst, "#!if(EnableSaddr, "off", "$vaddr")#!if(EnableSaddr, ", $saddr", ", off")#"$offset$glc$slc"> { 203 let has_data = 0; 204 let mayLoad = 1; 205 let has_saddr = 1; 206 let enabled_saddr = EnableSaddr; 207 let has_vaddr = !if(EnableSaddr, 0, 1); 208 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 209 let maybeAtomic = 1; 210} 211 212class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0> : FLAT_Pseudo< 213 opName, 214 (outs), 215 !if(EnableSaddr, 216 (ins vdataClass:$vdata, SReg_32_XEXEC_HI:$saddr, offset_s13:$offset, GLC:$glc, SLC:$slc), 217 (ins vdataClass:$vdata, VGPR_32:$vaddr, offset_s13:$offset, GLC:$glc, SLC:$slc)), 218 " "#!if(EnableSaddr, "off", "$vaddr")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc"> { 219 let mayLoad = 0; 220 let mayStore = 1; 221 let has_vdst = 0; 222 let has_saddr = 1; 223 let enabled_saddr = EnableSaddr; 224 let has_vaddr = !if(EnableSaddr, 0, 1); 225 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 226 let maybeAtomic = 1; 227} 228 229multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass> { 230 let is_flat_scratch = 1 in { 231 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass>; 232 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, 1>; 233 } 234} 235 236multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 237 let is_flat_scratch = 1 in { 238 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>; 239 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>; 240 } 241} 242 243class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 244 string asm, list<dag> pattern = []> : 245 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 246 let mayLoad = 1; 247 let mayStore = 1; 248 let has_glc = 0; 249 let glcValue = 0; 250 let has_vdst = 0; 251 let maybeAtomic = 1; 252} 253 254class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 255 string asm, list<dag> pattern = []> 256 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 257 let hasPostISelHook = 1; 258 let has_vdst = 1; 259 let glcValue = 1; 260 let PseudoInstr = NAME # "_RTN"; 261} 262 263multiclass FLAT_Atomic_Pseudo< 264 string opName, 265 RegisterClass vdst_rc, 266 ValueType vt, 267 SDPatternOperator atomic = null_frag, 268 ValueType data_vt = vt, 269 RegisterClass data_rc = vdst_rc> { 270 def "" : FLAT_AtomicNoRet_Pseudo <opName, 271 (outs), 272 (ins VReg_64:$vaddr, data_rc:$vdata, offset_u12:$offset, SLC:$slc), 273 " $vaddr, $vdata$offset$slc">, 274 GlobalSaddrTable<0, opName>, 275 AtomicNoRet <opName, 0> { 276 let PseudoInstr = NAME; 277 } 278 279 def _RTN : FLAT_AtomicRet_Pseudo <opName, 280 (outs vdst_rc:$vdst), 281 (ins VReg_64:$vaddr, data_rc:$vdata, offset_u12:$offset, SLC:$slc), 282 " $vdst, $vaddr, $vdata$offset glc$slc", 283 [(set vt:$vdst, 284 (atomic (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$vdata))]>, 285 GlobalSaddrTable<0, opName#"_rtn">, 286 AtomicNoRet <opName, 1>; 287} 288 289multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 290 string opName, 291 RegisterClass vdst_rc, 292 ValueType vt, 293 SDPatternOperator atomic = null_frag, 294 ValueType data_vt = vt, 295 RegisterClass data_rc = vdst_rc> { 296 297 def "" : FLAT_AtomicNoRet_Pseudo <opName, 298 (outs), 299 (ins VReg_64:$vaddr, data_rc:$vdata, offset_s13:$offset, SLC:$slc), 300 " $vaddr, $vdata, off$offset$slc">, 301 GlobalSaddrTable<0, opName>, 302 AtomicNoRet <opName, 0> { 303 let has_saddr = 1; 304 let PseudoInstr = NAME; 305 } 306 307 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 308 (outs), 309 (ins VReg_64:$vaddr, data_rc:$vdata, SReg_64:$saddr, offset_s13:$offset, SLC:$slc), 310 " $vaddr, $vdata, $saddr$offset$slc">, 311 GlobalSaddrTable<1, opName>, 312 AtomicNoRet <opName#"_saddr", 0> { 313 let has_saddr = 1; 314 let enabled_saddr = 1; 315 let PseudoInstr = NAME#"_SADDR"; 316 } 317} 318 319multiclass FLAT_Global_Atomic_Pseudo_RTN< 320 string opName, 321 RegisterClass vdst_rc, 322 ValueType vt, 323 SDPatternOperator atomic = null_frag, 324 ValueType data_vt = vt, 325 RegisterClass data_rc = vdst_rc> { 326 327 def _RTN : FLAT_AtomicRet_Pseudo <opName, 328 (outs vdst_rc:$vdst), 329 (ins VReg_64:$vaddr, data_rc:$vdata, offset_s13:$offset, SLC:$slc), 330 " $vdst, $vaddr, $vdata, off$offset glc$slc", 331 [(set vt:$vdst, 332 (atomic (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$vdata))]>, 333 GlobalSaddrTable<0, opName#"_rtn">, 334 AtomicNoRet <opName, 1> { 335 let has_saddr = 1; 336 } 337 338 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 339 (outs vdst_rc:$vdst), 340 (ins VReg_64:$vaddr, data_rc:$vdata, SReg_64:$saddr, offset_s13:$offset, SLC:$slc), 341 " $vdst, $vaddr, $vdata, $saddr$offset glc$slc">, 342 GlobalSaddrTable<1, opName#"_rtn">, 343 AtomicNoRet <opName#"_saddr", 1> { 344 let has_saddr = 1; 345 let enabled_saddr = 1; 346 let PseudoInstr = NAME#"_SADDR_RTN"; 347 } 348} 349 350multiclass FLAT_Global_Atomic_Pseudo< 351 string opName, 352 RegisterClass vdst_rc, 353 ValueType vt, 354 SDPatternOperator atomic = null_frag, 355 ValueType data_vt = vt, 356 RegisterClass data_rc = vdst_rc> : 357 FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>, 358 FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>; 359 360class flat_binary_atomic_op<SDNode atomic_op> : PatFrag< 361 (ops node:$ptr, node:$value), 362 (atomic_op node:$ptr, node:$value), 363 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS;}] 364>; 365 366def atomic_cmp_swap_flat : flat_binary_atomic_op<AMDGPUatomic_cmp_swap>; 367def atomic_swap_flat : flat_binary_atomic_op<atomic_swap>; 368def atomic_add_flat : flat_binary_atomic_op<atomic_load_add>; 369def atomic_and_flat : flat_binary_atomic_op<atomic_load_and>; 370def atomic_max_flat : flat_binary_atomic_op<atomic_load_max>; 371def atomic_min_flat : flat_binary_atomic_op<atomic_load_min>; 372def atomic_or_flat : flat_binary_atomic_op<atomic_load_or>; 373def atomic_sub_flat : flat_binary_atomic_op<atomic_load_sub>; 374def atomic_umax_flat : flat_binary_atomic_op<atomic_load_umax>; 375def atomic_umin_flat : flat_binary_atomic_op<atomic_load_umin>; 376def atomic_xor_flat : flat_binary_atomic_op<atomic_load_xor>; 377def atomic_inc_flat : flat_binary_atomic_op<SIatomic_inc>; 378def atomic_dec_flat : flat_binary_atomic_op<SIatomic_dec>; 379 380 381 382//===----------------------------------------------------------------------===// 383// Flat Instructions 384//===----------------------------------------------------------------------===// 385 386def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 387def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 388def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 389def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 390def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 391def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 392def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 393def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 394 395def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 396def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 397def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 398def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 399def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 400def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 401 402let SubtargetPredicate = HasD16LoadStore in { 403def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 404def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 405def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 406def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 407def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 408def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 409 410def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 411def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 412} 413 414defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 415 VGPR_32, i32, atomic_cmp_swap_flat, 416 v2i32, VReg_64>; 417 418defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 419 VReg_64, i64, atomic_cmp_swap_flat, 420 v2i64, VReg_128>; 421 422defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 423 VGPR_32, i32, atomic_swap_flat>; 424 425defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 426 VReg_64, i64, atomic_swap_flat>; 427 428defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 429 VGPR_32, i32, atomic_add_flat>; 430 431defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 432 VGPR_32, i32, atomic_sub_flat>; 433 434defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 435 VGPR_32, i32, atomic_min_flat>; 436 437defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 438 VGPR_32, i32, atomic_umin_flat>; 439 440defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 441 VGPR_32, i32, atomic_max_flat>; 442 443defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 444 VGPR_32, i32, atomic_umax_flat>; 445 446defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 447 VGPR_32, i32, atomic_and_flat>; 448 449defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 450 VGPR_32, i32, atomic_or_flat>; 451 452defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 453 VGPR_32, i32, atomic_xor_flat>; 454 455defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 456 VGPR_32, i32, atomic_inc_flat>; 457 458defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 459 VGPR_32, i32, atomic_dec_flat>; 460 461defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 462 VReg_64, i64, atomic_add_flat>; 463 464defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 465 VReg_64, i64, atomic_sub_flat>; 466 467defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 468 VReg_64, i64, atomic_min_flat>; 469 470defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 471 VReg_64, i64, atomic_umin_flat>; 472 473defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 474 VReg_64, i64, atomic_max_flat>; 475 476defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 477 VReg_64, i64, atomic_umax_flat>; 478 479defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 480 VReg_64, i64, atomic_and_flat>; 481 482defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 483 VReg_64, i64, atomic_or_flat>; 484 485defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 486 VReg_64, i64, atomic_xor_flat>; 487 488defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 489 VReg_64, i64, atomic_inc_flat>; 490 491defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 492 VReg_64, i64, atomic_dec_flat>; 493 494let SubtargetPredicate = isCI in { // CI Only flat instructions : FIXME Only? 495 496defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 497 VGPR_32, f32, null_frag, v2f32, VReg_64>; 498 499defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 500 VReg_64, f64, null_frag, v2f64, VReg_128>; 501 502defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 503 VGPR_32, f32>; 504 505defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 506 VGPR_32, f32>; 507 508defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 509 VReg_64, f64>; 510 511defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 512 VReg_64, f64>; 513 514} // End SubtargetPredicate = isCI 515 516let SubtargetPredicate = HasFlatGlobalInsts in { 517defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 518defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 519defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 520defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 521defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 522defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 523defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 524defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 525 526defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 527defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 528defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 529defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 530defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 531defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 532 533defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 534defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 535defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 536defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 537defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 538defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 539 540defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 541defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 542 543let is_flat_global = 1 in { 544defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 545 VGPR_32, i32, AMDGPUatomic_cmp_swap_global, 546 v2i32, VReg_64>; 547 548defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 549 VReg_64, i64, AMDGPUatomic_cmp_swap_global, 550 v2i64, VReg_128>; 551 552defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 553 VGPR_32, i32, atomic_swap_global>; 554 555defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 556 VReg_64, i64, atomic_swap_global>; 557 558defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 559 VGPR_32, i32, atomic_add_global>; 560 561defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 562 VGPR_32, i32, atomic_sub_global>; 563 564defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 565 VGPR_32, i32, atomic_min_global>; 566 567defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 568 VGPR_32, i32, atomic_umin_global>; 569 570defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 571 VGPR_32, i32, atomic_max_global>; 572 573defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 574 VGPR_32, i32, atomic_umax_global>; 575 576defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 577 VGPR_32, i32, atomic_and_global>; 578 579defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 580 VGPR_32, i32, atomic_or_global>; 581 582defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 583 VGPR_32, i32, atomic_xor_global>; 584 585defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 586 VGPR_32, i32, atomic_inc_global>; 587 588defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 589 VGPR_32, i32, atomic_dec_global>; 590 591defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 592 VReg_64, i64, atomic_add_global>; 593 594defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 595 VReg_64, i64, atomic_sub_global>; 596 597defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 598 VReg_64, i64, atomic_min_global>; 599 600defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 601 VReg_64, i64, atomic_umin_global>; 602 603defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 604 VReg_64, i64, atomic_max_global>; 605 606defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 607 VReg_64, i64, atomic_umax_global>; 608 609defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 610 VReg_64, i64, atomic_and_global>; 611 612defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 613 VReg_64, i64, atomic_or_global>; 614 615defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 616 VReg_64, i64, atomic_xor_global>; 617 618defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 619 VReg_64, i64, atomic_inc_global>; 620 621defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 622 VReg_64, i64, atomic_dec_global>; 623} // End is_flat_global = 1 624 625} // End SubtargetPredicate = HasFlatGlobalInsts 626 627 628let SubtargetPredicate = HasFlatScratchInsts in { 629defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 630defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 631defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 632defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 633defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 634defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 635defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 636defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 637 638defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32>; 639defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>; 640defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32>; 641defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>; 642defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32>; 643defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>; 644 645defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 646defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 647defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 648defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 649defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 650defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 651 652defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 653defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 654 655} // End SubtargetPredicate = HasFlatScratchInsts 656 657//===----------------------------------------------------------------------===// 658// Flat Patterns 659//===----------------------------------------------------------------------===// 660 661// Patterns for global loads with no offset. 662class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 663 (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))), 664 (inst $vaddr, $offset, 0, $slc) 665>; 666 667multiclass FlatLoadPat_Hi16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> { 668 def : GCNPat < 669 (build_vector vt:$elt0, (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)))), 670 (v2i16 (inst $vaddr, $offset, 0, $slc, $elt0)) 671 >; 672 673 def : GCNPat < 674 (build_vector f16:$elt0, (f16 (bitconvert (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)))))), 675 (v2f16 (inst $vaddr, $offset, 0, $slc, $elt0)) 676 >; 677} 678 679multiclass FlatSignedLoadPat_Hi16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> { 680 def : GCNPat < 681 (build_vector vt:$elt0, (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)))), 682 (v2i16 (inst $vaddr, $offset, 0, $slc, $elt0)) 683 >; 684 685 def : GCNPat < 686 (build_vector f16:$elt0, (f16 (bitconvert (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)))))), 687 (v2f16 (inst $vaddr, $offset, 0, $slc, $elt0)) 688 >; 689} 690 691multiclass FlatLoadPat_Lo16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> { 692 def : GCNPat < 693 (build_vector (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))), (vt (Hi16Elt vt:$hi))), 694 (v2i16 (inst $vaddr, $offset, 0, $slc, $hi)) 695 >; 696 697 def : GCNPat < 698 (build_vector (f16 (bitconvert (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))))), (f16 (Hi16Elt f16:$hi))), 699 (v2f16 (inst $vaddr, $offset, 0, $slc, $hi)) 700 >; 701} 702 703multiclass FlatSignedLoadPat_Lo16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> { 704 def : GCNPat < 705 (build_vector (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))), (vt (Hi16Elt vt:$hi))), 706 (v2i16 (inst $vaddr, $offset, 0, $slc, $hi)) 707 >; 708 709 def : GCNPat < 710 (build_vector (f16 (bitconvert (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))))), (f16 (Hi16Elt f16:$hi))), 711 (v2f16 (inst $vaddr, $offset, 0, $slc, $hi)) 712 >; 713} 714 715class FlatLoadAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 716 (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc))), 717 (inst $vaddr, $offset, 0, $slc) 718>; 719 720class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 721 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))), 722 (inst $vaddr, $offset, 0, $slc) 723>; 724 725class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 726 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)), 727 (inst $vaddr, $data, $offset, 0, $slc) 728>; 729 730class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 731 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)), 732 (inst $vaddr, $data, $offset, 0, $slc) 733>; 734 735class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 736 // atomic store follows atomic binop convention so the address comes 737 // first. 738 (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data), 739 (inst $vaddr, $data, $offset, 0, $slc) 740>; 741 742class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 743 // atomic store follows atomic binop convention so the address comes 744 // first. 745 (node (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data), 746 (inst $vaddr, $data, $offset, 0, $slc) 747>; 748 749class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 750 ValueType data_vt = vt> : GCNPat < 751 (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)), 752 (inst $vaddr, $data, $offset, $slc) 753>; 754 755class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 756 ValueType data_vt = vt> : GCNPat < 757 (vt (node (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)), 758 (inst $vaddr, $data, $offset, $slc) 759>; 760 761let OtherPredicates = [HasFlatAddressSpace] in { 762 763def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i32>; 764def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 765def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i16>; 766def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 767def : FlatLoadPat <FLAT_LOAD_USHORT, az_extloadi16_flat, i32>; 768def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 769def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 770def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, i32>; 771def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, v2i32>; 772def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, v4i32>; 773 774def : FlatLoadAtomicPat <FLAT_LOAD_DWORD, atomic_load_flat, i32>; 775def : FlatLoadAtomicPat <FLAT_LOAD_DWORDX2, atomic_load_flat, i64>; 776 777def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 778def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 779def : FlatStorePat <FLAT_STORE_DWORD, store_flat, i32>; 780def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, v2i32>; 781def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, v4i32>; 782 783def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat, i32>; 784def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat, i64>; 785 786def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_add_global, i32>; 787def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_sub_global, i32>; 788def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global, i32>; 789def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global, i32>; 790def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_and_global, i32>; 791def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_max_global, i32>; 792def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_umax_global, i32>; 793def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_min_global, i32>; 794def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_umin_global, i32>; 795def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_or_global, i32>; 796def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global, i32>; 797def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>; 798def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_xor_global, i32>; 799 800def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_add_global, i64>; 801def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_sub_global, i64>; 802def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global, i64>; 803def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global, i64>; 804def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_and_global, i64>; 805def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_max_global, i64>; 806def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_umax_global, i64>; 807def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_min_global, i64>; 808def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_umin_global, i64>; 809def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_or_global, i64>; 810def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>; 811def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>; 812def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>; 813 814def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 815def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 816 817let OtherPredicates = [D16PreservesUnusedBits] in { 818def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 819def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 820 821let AddedComplexity = 3 in { 822defm : FlatLoadPat_Hi16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_flat>; 823defm : FlatLoadPat_Hi16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_flat>; 824defm : FlatLoadPat_Hi16 <FLAT_LOAD_SHORT_D16_HI, load_flat>; 825} 826 827let AddedComplexity = 9 in { 828defm : FlatLoadPat_Lo16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_flat>; 829defm : FlatLoadPat_Lo16 <FLAT_LOAD_SBYTE_D16, sextloadi8_flat>; 830defm : FlatLoadPat_Lo16 <FLAT_LOAD_SHORT_D16, load_flat>; 831} 832} 833 834} // End OtherPredicates = [HasFlatAddressSpace] 835 836let OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 in { 837 838def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i32>; 839def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 840def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i16>; 841def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 842def : FlatLoadSignedPat <GLOBAL_LOAD_USHORT, az_extloadi16_global, i32>; 843def : FlatLoadSignedPat <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 844def : FlatLoadSignedPat <GLOBAL_LOAD_USHORT, load_global, i16>; 845 846def : FlatLoadSignedPat <GLOBAL_LOAD_DWORD, load_global, i32>; 847def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX2, load_global, v2i32>; 848def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, load_global, v4i32>; 849 850def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORD, atomic_load_global, i32>; 851def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORDX2, atomic_load_global, i64>; 852 853def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 854def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 855def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 856def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, store_global, i16>; 857def : FlatStoreSignedPat <GLOBAL_STORE_DWORD, store_global, i32>; 858def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, store_global, v2i32>; 859def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, v4i32>; 860 861let OtherPredicates = [D16PreservesUnusedBits] in { 862def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 863def : FlatStoreSignedPat <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 864 865defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_global>; 866defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_global>; 867defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SHORT_D16_HI, load_global>; 868 869defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_global>; 870defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_global>; 871defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_SHORT_D16, load_global>; 872 873} 874 875def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, store_atomic_global, i32>; 876def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, store_atomic_global, i64>; 877 878def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_RTN, atomic_add_global, i32>; 879def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_RTN, atomic_sub_global, i32>; 880def : FlatSignedAtomicPat <GLOBAL_ATOMIC_INC_RTN, atomic_inc_global, i32>; 881def : FlatSignedAtomicPat <GLOBAL_ATOMIC_DEC_RTN, atomic_dec_global, i32>; 882def : FlatSignedAtomicPat <GLOBAL_ATOMIC_AND_RTN, atomic_and_global, i32>; 883def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMAX_RTN, atomic_max_global, i32>; 884def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMAX_RTN, atomic_umax_global, i32>; 885def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_RTN, atomic_min_global, i32>; 886def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_RTN, atomic_umin_global, i32>; 887def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_RTN, atomic_or_global, i32>; 888def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_RTN, atomic_swap_global, i32>; 889def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>; 890def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_RTN, atomic_xor_global, i32>; 891 892def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_X2_RTN, atomic_add_global, i64>; 893def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_X2_RTN, atomic_sub_global, i64>; 894def : FlatSignedAtomicPat <GLOBAL_ATOMIC_INC_X2_RTN, atomic_inc_global, i64>; 895def : FlatSignedAtomicPat <GLOBAL_ATOMIC_DEC_X2_RTN, atomic_dec_global, i64>; 896def : FlatSignedAtomicPat <GLOBAL_ATOMIC_AND_X2_RTN, atomic_and_global, i64>; 897def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMAX_X2_RTN, atomic_max_global, i64>; 898def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMAX_X2_RTN, atomic_umax_global, i64>; 899def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_X2_RTN, atomic_min_global, i64>; 900def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_X2_RTN, atomic_umin_global, i64>; 901def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_X2_RTN, atomic_or_global, i64>; 902def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>; 903def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>; 904def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>; 905 906} // End OtherPredicates = [HasFlatGlobalInsts] 907 908 909//===----------------------------------------------------------------------===// 910// Target 911//===----------------------------------------------------------------------===// 912 913//===----------------------------------------------------------------------===// 914// CI 915//===----------------------------------------------------------------------===// 916 917class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 918 FLAT_Real <op, ps>, 919 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 920 let AssemblerPredicate = isCIOnly; 921 let DecoderNamespace="CI"; 922} 923 924def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 925def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 926def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 927def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 928def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 929def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 930def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 931def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 932 933def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 934def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 935def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 936def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 937def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 938def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 939 940multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 941 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 942 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 943} 944 945defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 946defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 947defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 948defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 949defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 950defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 951defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 952defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 953defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 954defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 955defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 956defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 957defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 958defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 959defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 960defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 961defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 962defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 963defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 964defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 965defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 966defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 967defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 968defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 969defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 970defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 971 972// CI Only flat instructions 973defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 974defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 975defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 976defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 977defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 978defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 979 980 981//===----------------------------------------------------------------------===// 982// VI 983//===----------------------------------------------------------------------===// 984 985class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps> : 986 FLAT_Real <op, ps>, 987 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 988 let AssemblerPredicate = isVI; 989 let DecoderNamespace="VI"; 990} 991 992multiclass FLAT_Real_AllAddr_vi<bits<7> op> { 993 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)>; 994 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 995} 996 997def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 998def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 999def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1000def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1001def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1002def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1003def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1004def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1005 1006def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1007def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1008def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1009def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1010def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1011def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1012def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1013def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1014 1015def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1016def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1017def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1018def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1019def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1020def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1021 1022multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> { 1023 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1024 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1025} 1026 1027multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> : 1028 FLAT_Real_AllAddr_vi<op> { 1029 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1030 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1031} 1032 1033 1034defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1035defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1036defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1037defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1038defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1039defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1040defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1041defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1042defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1043defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1044defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1045defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1046defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1047defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1048defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1049defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1050defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1051defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1052defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1053defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1054defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1055defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1056defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1057defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1058defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1059defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1060 1061defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1062defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1063defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1064defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1065defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1066defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1067defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1068defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1069 1070defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1071defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1072defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1073defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1074defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1075defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1076 1077defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1078defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1079defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1080defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1081defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1082defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1083defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1084defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1085 1086 1087defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1088defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1089defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1090defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1091defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1092defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1093defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1094defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1095defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1096defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1097defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1098defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1099defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1100defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1101defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1102defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1103defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1104defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1105defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1106defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1107defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1108defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1109defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1110defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1111defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1112defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1113 1114defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1115defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1116defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1117defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1118defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1119defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1120defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1121defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1122defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1123defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1124defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1125defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1126defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1127defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1128defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1129defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1130defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1131defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1132defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1133defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1134defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1135defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1136