1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FLATOffset : ComplexPattern<i64, 2, "SelectFlatOffset<false>", [], [SDNPWantRoot], -10>; 10def FLATOffsetSigned : ComplexPattern<i64, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 11def ScratchOffset : ComplexPattern<i32, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 12 13def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 14def ScratchSAddr : ComplexPattern<i32, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; 15 16//===----------------------------------------------------------------------===// 17// FLAT classes 18//===----------------------------------------------------------------------===// 19 20class FLAT_Pseudo<string opName, dag outs, dag ins, 21 string asmOps, list<dag> pattern=[]> : 22 InstSI<outs, ins, "", pattern>, 23 SIMCInstr<opName, SIEncodingFamily.NONE> { 24 25 let isPseudo = 1; 26 let isCodeGenOnly = 1; 27 28 let FLAT = 1; 29 30 let UseNamedOperandTable = 1; 31 let hasSideEffects = 0; 32 let SchedRW = [WriteVMEM]; 33 34 string Mnemonic = opName; 35 string AsmOperands = asmOps; 36 37 bits<1> is_flat_global = 0; 38 bits<1> is_flat_scratch = 0; 39 40 bits<1> has_vdst = 1; 41 42 // We need to distinguish having saddr and enabling saddr because 43 // saddr is only valid for scratch and global instructions. Pre-gfx9 44 // these bits were reserved, so we also don't necessarily want to 45 // set these bits to the disabled value for the original flat 46 // segment instructions. 47 bits<1> has_saddr = 0; 48 bits<1> enabled_saddr = 0; 49 bits<7> saddr_value = 0; 50 bits<1> has_vaddr = 1; 51 52 bits<1> has_data = 1; 53 bits<1> has_glc = 1; 54 bits<1> glcValue = 0; 55 bits<1> has_dlc = 1; 56 bits<1> dlcValue = 0; 57 bits<1> has_sccb = 1; 58 bits<1> sccbValue = 0; 59 60 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 61 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 62 63 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 64 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 65 66 // Internally, FLAT instruction are executed as both an LDS and a 67 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 68 // and are not considered done until both have been decremented. 69 let VM_CNT = 1; 70 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); 71 72 let IsFlatGlobal = is_flat_global; 73 74 let IsFlatScratch = is_flat_scratch; 75} 76 77class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 78 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 79 Enc64 { 80 81 let isPseudo = 0; 82 let isCodeGenOnly = 0; 83 84 // copy relevant pseudo op flags 85 let SubtargetPredicate = ps.SubtargetPredicate; 86 let AsmMatchConverter = ps.AsmMatchConverter; 87 let OtherPredicates = ps.OtherPredicates; 88 let TSFlags = ps.TSFlags; 89 let UseNamedOperandTable = ps.UseNamedOperandTable; 90 let SchedRW = ps.SchedRW; 91 92 // encoding fields 93 bits<8> vaddr; 94 bits<10> vdata; 95 bits<7> saddr; 96 bits<10> vdst; 97 98 bits<5> cpol; 99 100 // Only valid on gfx9 101 bits<1> lds = 0; // XXX - What does this actually do? 102 103 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 104 bits<2> seg = !if(ps.is_flat_global, 0b10, 105 !if(ps.is_flat_scratch, 0b01, 0)); 106 107 // Signed offset. Highest bit ignored for flat and treated as 12-bit 108 // unsigned for flat accesses. 109 bits<13> offset; 110 // GFX90A+ only: instruction uses AccVGPR for data 111 bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); 112 113 // We don't use tfe right now, and it was removed in gfx9. 114 bits<1> tfe = 0; 115 116 // Only valid on GFX9+ 117 let Inst{12-0} = offset; 118 let Inst{13} = lds; 119 let Inst{15-14} = seg; 120 121 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); 122 let Inst{17} = cpol{CPolBit.SLC}; 123 let Inst{24-18} = op; 124 let Inst{31-26} = 0x37; // Encoding. 125 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 126 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); 127 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 128 129 // 54-48 is reserved. 130 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. 131 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); 132} 133 134class GlobalSaddrTable <bit is_saddr, string Name = ""> { 135 bit IsSaddr = is_saddr; 136 string SaddrOp = Name; 137} 138 139// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 140// same encoding value as exec_hi, so it isn't possible to use that if 141// saddr is 32-bit (which isn't handled here yet). 142class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 143 bit HasTiedOutput = 0, 144 bit HasSaddr = 0, bit EnableSaddr = 0, 145 RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< 146 opName, 147 (outs vdata_op:$vdst), 148 !con( 149 !con( 150 !if(EnableSaddr, 151 (ins SReg_64:$saddr, VGPR_32:$vaddr), 152 (ins VReg_64:$vaddr)), 153 (ins flat_offset:$offset)), 154 // FIXME: Operands with default values do not work with following non-optional operands. 155 !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in), 156 (ins CPol_0:$cpol))), 157 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 158 let has_data = 0; 159 let mayLoad = 1; 160 let has_saddr = HasSaddr; 161 let enabled_saddr = EnableSaddr; 162 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 163 let maybeAtomic = 1; 164 165 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 166 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 167} 168 169class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 170 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 171 opName, 172 (outs), 173 !con( 174 !if(EnableSaddr, 175 (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64:$saddr), 176 (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), 177 (ins flat_offset:$offset, CPol_0:$cpol)), 178 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 179 let mayLoad = 0; 180 let mayStore = 1; 181 let has_vdst = 0; 182 let has_saddr = HasSaddr; 183 let enabled_saddr = EnableSaddr; 184 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 185 let maybeAtomic = 1; 186} 187 188multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 189 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 190 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 191 GlobalSaddrTable<0, opName>; 192 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 193 GlobalSaddrTable<1, opName>; 194 } 195} 196 197class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 198 bit HasTiedOutput = 0, bit HasSignedOffset = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 199 opName, 200 (outs regClass:$vdst), 201 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), 202 (ins flat_offset:$offset, CPol_0:$cpol), 203 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 204 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 205 let is_flat_global = 1; 206 let has_data = 0; 207 let mayLoad = 1; 208 let has_vaddr = 0; 209 let has_saddr = 1; 210 let enabled_saddr = EnableSaddr; 211 let maybeAtomic = 1; 212 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 213 214 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 215 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 216} 217 218multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, 219 bit HasTiedOutput = 0, bit HasSignedOffset = 0> { 220 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, HasSignedOffset>, 221 GlobalSaddrTable<0, opName>; 222 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, HasSignedOffset, 1>, 223 GlobalSaddrTable<1, opName>; 224} 225 226multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 227 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 228 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 229 GlobalSaddrTable<0, opName>; 230 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 231 GlobalSaddrTable<1, opName>; 232 } 233} 234 235class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 236 bit HasSignedOffset = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 237 opName, 238 (outs), 239 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), 240 (ins flat_offset:$offset, CPol:$cpol)), 241 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 242 let is_flat_global = 1; 243 let mayLoad = 0; 244 let mayStore = 1; 245 let has_vdst = 0; 246 let has_vaddr = 0; 247 let has_saddr = 1; 248 let enabled_saddr = EnableSaddr; 249 let maybeAtomic = 1; 250 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 251} 252 253multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass, 254 bit HasSignedOffset = 0> { 255 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, HasSignedOffset>, 256 GlobalSaddrTable<0, opName>; 257 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, HasSignedOffset, 1>, 258 GlobalSaddrTable<1, opName>; 259} 260 261class FlatScratchInst <string sv_op, string mode> { 262 string SVOp = sv_op; 263 string Mode = mode; 264} 265 266class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 267 bit HasTiedOutput = 0, 268 bit EnableSaddr = 0, 269 bit EnableVaddr = !not(EnableSaddr)> 270 : FLAT_Pseudo< 271 opName, 272 (outs getLdStRegisterOperand<regClass>.ret:$vdst), 273 !con( 274 !if(EnableSaddr, 275 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 276 !if(EnableVaddr, 277 (ins VGPR_32:$vaddr, flat_offset:$offset), 278 (ins flat_offset:$offset))), 279 !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in), 280 (ins CPol_0:$cpol))), 281 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 282 let has_data = 0; 283 let mayLoad = 1; 284 let has_saddr = 1; 285 let enabled_saddr = EnableSaddr; 286 let has_vaddr = EnableVaddr; 287 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 288 let maybeAtomic = 1; 289 290 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 291 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 292} 293 294class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 295 bit EnableVaddr = !not(EnableSaddr), 296 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< 297 opName, 298 (outs), 299 !if(EnableSaddr, 300 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 301 !if(EnableVaddr, 302 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol), 303 (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol))), 304 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 305 let mayLoad = 0; 306 let mayStore = 1; 307 let has_vdst = 0; 308 let has_saddr = 1; 309 let enabled_saddr = EnableSaddr; 310 let has_vaddr = EnableVaddr; 311 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 312 let maybeAtomic = 1; 313} 314 315multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 316 let is_flat_scratch = 1 in { 317 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, 318 FlatScratchInst<opName, "SV">; 319 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, 320 FlatScratchInst<opName, "SS">; 321 322 let SubtargetPredicate = HasFlatScratchSTMode in 323 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0>, 324 FlatScratchInst<opName, "ST">; 325 } 326} 327 328multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 329 let is_flat_scratch = 1 in { 330 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, 331 FlatScratchInst<opName, "SV">; 332 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, 333 FlatScratchInst<opName, "SS">; 334 335 let SubtargetPredicate = HasFlatScratchSTMode in 336 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0>, 337 FlatScratchInst<opName, "ST">; 338 } 339} 340 341class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 342 string asm, list<dag> pattern = []> : 343 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 344 let mayLoad = 1; 345 let mayStore = 1; 346 let has_glc = 0; 347 let glcValue = 0; 348 let has_dlc = 0; 349 let dlcValue = 0; 350 let has_vdst = 0; 351 let has_sccb = 1; 352 let sccbValue = 0; 353 let maybeAtomic = 1; 354 let IsAtomicNoRet = 1; 355} 356 357class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 358 string asm, list<dag> pattern = []> 359 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 360 let hasPostISelHook = 1; 361 let has_vdst = 1; 362 let glcValue = 1; 363 let dlcValue = 0; 364 let sccbValue = 0; 365 let IsAtomicNoRet = 0; 366 let IsAtomicRet = 1; 367 let PseudoInstr = NAME # "_RTN"; 368} 369 370multiclass FLAT_Atomic_Pseudo< 371 string opName, 372 RegisterClass vdst_rc, 373 ValueType vt, 374 SDPatternOperator atomic = null_frag, 375 ValueType data_vt = vt, 376 RegisterClass data_rc = vdst_rc, 377 bit isFP = isFloatType<data_vt>.ret, 378 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 379 def "" : FLAT_AtomicNoRet_Pseudo <opName, 380 (outs), 381 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 382 " $vaddr, $vdata$offset$cpol">, 383 GlobalSaddrTable<0, opName>, 384 AtomicNoRet <opName, 0> { 385 let PseudoInstr = NAME; 386 let FPAtomic = isFP; 387 let AddedComplexity = -1; // Prefer global atomics if available 388 } 389 390 def _RTN : FLAT_AtomicRet_Pseudo <opName, 391 (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), 392 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 393 " $vdst, $vaddr, $vdata$offset$cpol", 394 [(set vt:$vdst, 395 (atomic (FLATOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 396 GlobalSaddrTable<0, opName#"_rtn">, 397 AtomicNoRet <opName, 1>{ 398 let FPAtomic = isFP; 399 let AddedComplexity = -1; // Prefer global atomics if available 400 } 401} 402 403multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 404 string opName, 405 RegisterClass vdst_rc, 406 ValueType vt, 407 SDPatternOperator atomic = null_frag, 408 ValueType data_vt = vt, 409 RegisterClass data_rc = vdst_rc, 410 bit isFP = isFloatType<data_vt>.ret, 411 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 412 413 def "" : FLAT_AtomicNoRet_Pseudo <opName, 414 (outs), 415 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 416 " $vaddr, $vdata, off$offset$cpol">, 417 GlobalSaddrTable<0, opName>, 418 AtomicNoRet <opName, 0> { 419 let has_saddr = 1; 420 let PseudoInstr = NAME; 421 let FPAtomic = isFP; 422 } 423 424 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 425 (outs), 426 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_0:$cpol), 427 " $vaddr, $vdata, $saddr$offset$cpol">, 428 GlobalSaddrTable<1, opName>, 429 AtomicNoRet <opName#"_saddr", 0> { 430 let has_saddr = 1; 431 let enabled_saddr = 1; 432 let PseudoInstr = NAME#"_SADDR"; 433 let FPAtomic = isFP; 434 } 435} 436 437multiclass FLAT_Global_Atomic_Pseudo_RTN< 438 string opName, 439 RegisterClass vdst_rc, 440 ValueType vt, 441 SDPatternOperator atomic = null_frag, 442 ValueType data_vt = vt, 443 RegisterClass data_rc = vdst_rc, 444 bit isFP = isFloatType<data_vt>.ret, 445 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, 446 RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { 447 448 def _RTN : FLAT_AtomicRet_Pseudo <opName, 449 (outs vdst_op:$vdst), 450 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 451 " $vdst, $vaddr, $vdata, off$offset$cpol", 452 [(set vt:$vdst, 453 (atomic (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 454 GlobalSaddrTable<0, opName#"_rtn">, 455 AtomicNoRet <opName, 1> { 456 let has_saddr = 1; 457 let FPAtomic = isFP; 458 } 459 460 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 461 (outs vdst_op:$vdst), 462 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_GLC1:$cpol), 463 " $vdst, $vaddr, $vdata, $saddr$offset$cpol">, 464 GlobalSaddrTable<1, opName#"_rtn">, 465 AtomicNoRet <opName#"_saddr", 1> { 466 let has_saddr = 1; 467 let enabled_saddr = 1; 468 let PseudoInstr = NAME#"_SADDR_RTN"; 469 let FPAtomic = isFP; 470 } 471} 472 473multiclass FLAT_Global_Atomic_Pseudo< 474 string opName, 475 RegisterClass vdst_rc, 476 ValueType vt, 477 SDPatternOperator atomic_rtn = null_frag, 478 SDPatternOperator atomic_no_rtn = null_frag, 479 ValueType data_vt = vt, 480 RegisterClass data_rc = vdst_rc> { 481 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 482 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>; 483 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; 484 } 485} 486 487//===----------------------------------------------------------------------===// 488// Flat Instructions 489//===----------------------------------------------------------------------===// 490 491def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 492def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 493def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 494def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 495def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 496def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 497def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 498def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 499 500def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 501def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 502def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 503def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 504def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 505def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 506 507let SubtargetPredicate = HasD16LoadStore in { 508def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 509def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 510def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 511def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 512def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 513def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 514 515def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 516def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 517} 518 519defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 520 VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, 521 v2i32, VReg_64>; 522 523defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 524 VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, 525 v2i64, VReg_128>; 526 527defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 528 VGPR_32, i32, atomic_swap_flat_32>; 529 530defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 531 VReg_64, i64, atomic_swap_flat_64>; 532 533defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 534 VGPR_32, i32, atomic_load_add_flat_32>; 535 536defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 537 VGPR_32, i32, atomic_load_sub_flat_32>; 538 539defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 540 VGPR_32, i32, atomic_load_min_flat_32>; 541 542defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 543 VGPR_32, i32, atomic_load_umin_flat_32>; 544 545defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 546 VGPR_32, i32, atomic_load_max_flat_32>; 547 548defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 549 VGPR_32, i32, atomic_load_umax_flat_32>; 550 551defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 552 VGPR_32, i32, atomic_load_and_flat_32>; 553 554defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 555 VGPR_32, i32, atomic_load_or_flat_32>; 556 557defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 558 VGPR_32, i32, atomic_load_xor_flat_32>; 559 560defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 561 VGPR_32, i32, atomic_inc_flat_32>; 562 563defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 564 VGPR_32, i32, atomic_dec_flat_32>; 565 566defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 567 VReg_64, i64, atomic_load_add_flat_64>; 568 569defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 570 VReg_64, i64, atomic_load_sub_flat_64>; 571 572defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 573 VReg_64, i64, atomic_load_min_flat_64>; 574 575defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 576 VReg_64, i64, atomic_load_umin_flat_64>; 577 578defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 579 VReg_64, i64, atomic_load_max_flat_64>; 580 581defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 582 VReg_64, i64, atomic_load_umax_flat_64>; 583 584defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 585 VReg_64, i64, atomic_load_and_flat_64>; 586 587defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 588 VReg_64, i64, atomic_load_or_flat_64>; 589 590defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 591 VReg_64, i64, atomic_load_xor_flat_64>; 592 593defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 594 VReg_64, i64, atomic_inc_flat_64>; 595 596defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 597 VReg_64, i64, atomic_dec_flat_64>; 598 599// GFX7-, GFX10-only flat instructions. 600let SubtargetPredicate = isGFX7GFX10 in { 601 602defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 603 VGPR_32, f32, null_frag, v2f32, VReg_64>; 604 605defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 606 VReg_64, f64, null_frag, v2f64, VReg_128>; 607 608defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 609 VGPR_32, f32>; 610 611defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 612 VGPR_32, f32>; 613 614defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 615 VReg_64, f64>; 616 617defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 618 VReg_64, f64>; 619 620} // End SubtargetPredicate = isGFX7GFX10 621 622let SubtargetPredicate = isGFX90APlus in { 623 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64, int_amdgcn_flat_atomic_fadd>; 624 defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo<"flat_atomic_min_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmin>; 625 defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo<"flat_atomic_max_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmax>; 626 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 627 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 628 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 629} // End SubtargetPredicate = isGFX90APlus 630 631defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 632defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 633defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 634defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 635defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 636defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 637defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 638defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 639 640defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 641defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 642defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 643defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 644defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 645defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 646let OtherPredicates = [HasGFX10_BEncoding] in 647defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 648 649defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 650defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 651defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 652defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 653defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 654defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 655let OtherPredicates = [HasGFX10_BEncoding] in 656defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 657 658defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 659defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 660 661let is_flat_global = 1 in { 662defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 663 VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag, 664 v2i32, VReg_64>; 665 666defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 667 VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, 668 null_frag, 669 v2i64, VReg_128>; 670 671defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 672 VGPR_32, i32, atomic_swap_global_32>; 673 674defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 675 VReg_64, i64, atomic_swap_global_64>; 676 677defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 678 VGPR_32, i32, atomic_load_add_global_32>; 679 680defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 681 VGPR_32, i32, atomic_load_sub_global_32>; 682 683defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 684 VGPR_32, i32, atomic_load_min_global_32>; 685 686defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 687 VGPR_32, i32, atomic_load_umin_global_32>; 688 689defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 690 VGPR_32, i32, atomic_load_max_global_32>; 691 692defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 693 VGPR_32, i32, atomic_load_umax_global_32>; 694 695defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 696 VGPR_32, i32, atomic_load_and_global_32>; 697 698defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 699 VGPR_32, i32, atomic_load_or_global_32>; 700 701defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 702 VGPR_32, i32, atomic_load_xor_global_32>; 703 704defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 705 VGPR_32, i32, atomic_inc_global_32>; 706 707defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 708 VGPR_32, i32, atomic_dec_global_32>; 709 710defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 711 VReg_64, i64, atomic_load_add_global_64>; 712 713defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 714 VReg_64, i64, atomic_load_sub_global_64>; 715 716defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 717 VReg_64, i64, atomic_load_min_global_64>; 718 719defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 720 VReg_64, i64, atomic_load_umin_global_64>; 721 722defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 723 VReg_64, i64, atomic_load_max_global_64>; 724 725defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 726 VReg_64, i64, atomic_load_umax_global_64>; 727 728defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 729 VReg_64, i64, atomic_load_and_global_64>; 730 731defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 732 VReg_64, i64, atomic_load_or_global_64>; 733 734defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 735 VReg_64, i64, atomic_load_xor_global_64>; 736 737defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 738 VReg_64, i64, atomic_inc_global_64>; 739 740defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 741 VReg_64, i64, atomic_dec_global_64>; 742 743let SubtargetPredicate = HasGFX10_BEncoding in 744defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 745 VGPR_32, i32, int_amdgcn_global_atomic_csub>; 746} // End is_flat_global = 1 747 748 749 750let SubtargetPredicate = HasFlatScratchInsts in { 751defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 752defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 753defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 754defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 755defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 756defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 757defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 758defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 759 760defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 761defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 762defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 763defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 764defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 765defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 766 767defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 768defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 769defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 770defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 771defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 772defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 773 774defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 775defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 776 777} // End SubtargetPredicate = HasFlatScratchInsts 778 779let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 780 defm GLOBAL_ATOMIC_FCMPSWAP : 781 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32>; 782 defm GLOBAL_ATOMIC_FMIN : 783 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 784 defm GLOBAL_ATOMIC_FMAX : 785 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 786 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 787 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>; 788 defm GLOBAL_ATOMIC_FMIN_X2 : 789 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 790 defm GLOBAL_ATOMIC_FMAX_X2 : 791 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 792} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 793 794let is_flat_global = 1 in { 795let OtherPredicates = [HasAtomicFaddInsts] in { 796 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 797 "global_atomic_add_f32", VGPR_32, f32 798 >; 799 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 800 "global_atomic_pk_add_f16", VGPR_32, v2f16 801 >; 802} // End OtherPredicates = [HasAtomicFaddInsts] 803 804let OtherPredicates = [isGFX90APlus] in { 805 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < 806 "global_atomic_add_f32", VGPR_32, f32, int_amdgcn_global_atomic_fadd 807 >; 808 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < 809 "global_atomic_pk_add_f16", VGPR_32, v2f16, int_amdgcn_global_atomic_fadd 810 >; 811} // End OtherPredicates = [isGFX90APlus] 812} // End is_flat_global = 1 813 814//===----------------------------------------------------------------------===// 815// Flat Patterns 816//===----------------------------------------------------------------------===// 817 818// Patterns for global loads with no offset. 819class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 820 (vt (node (FLATOffset i64:$vaddr, i16:$offset))), 821 (inst $vaddr, $offset) 822>; 823 824class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 825 (node (FLATOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 826 (inst $vaddr, $offset, 0, $in) 827>; 828 829class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 830 (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 831 (inst $vaddr, $offset, 0, $in) 832>; 833 834class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 835 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 836 (inst $saddr, $voffset, $offset, 0, $in) 837>; 838 839class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 840 (vt (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset))), 841 (inst $vaddr, $offset) 842>; 843 844class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 845 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 846 (inst $saddr, $voffset, $offset, 0) 847>; 848 849class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 850 ValueType vt> : GCNPat < 851 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 852 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 853>; 854 855class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 856 ValueType vt> : GCNPat < 857 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 858 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 859>; 860 861class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 862 ValueType vt, ValueType data_vt = vt> : GCNPat < 863 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 864 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 865>; 866 867class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 868 ValueType vt> : GCNPat < 869 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 870 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 871>; 872 873class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 874 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset)), 875 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 876>; 877 878class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 879 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset)), 880 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 881>; 882 883class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 884 // atomic store follows atomic binop convention so the address comes 885 // first. 886 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 887 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 888>; 889 890class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 891 ValueType vt, ValueType data_vt = vt> : GCNPat < 892 // atomic store follows atomic binop convention so the address comes 893 // first. 894 (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data), 895 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 896>; 897 898class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 899 ValueType data_vt = vt> : GCNPat < 900 (vt (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data)), 901 (inst $vaddr, $data, $offset) 902>; 903 904class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 905 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 906 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 907>; 908 909class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 910 (node (FLATOffsetSigned i64:$vaddr, i16:$offset), vt:$data), 911 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 912>; 913 914class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 915 ValueType data_vt = vt> : GCNPat < 916 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$data)), 917 (inst $vaddr, $data, $offset) 918>; 919 920class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 921 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset))), 922 (inst $vaddr, $offset) 923>; 924 925class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 926 (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset), vt:$in), 927 (inst $vaddr, $offset, 0, $in) 928>; 929 930class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 931 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset)), 932 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) 933>; 934 935class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 936 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset))), 937 (inst $saddr, $offset) 938>; 939 940class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 941 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 942 (inst $saddr, $offset, 0, $in) 943>; 944 945class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 946 ValueType vt> : GCNPat < 947 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)), 948 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 949>; 950 951let OtherPredicates = [HasFlatAddressSpace] in { 952 953def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 954def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 955def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 956def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 957def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 958def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 959def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 960def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 961def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 962def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 963def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 964 965def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 966def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 967 968def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 969def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 970 971foreach vt = Reg32Types.types in { 972def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 973def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 974} 975 976foreach vt = VReg_64.RegTypes in { 977def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 978def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 979} 980 981def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 982 983foreach vt = VReg_128.RegTypes in { 984def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 985def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 986} 987 988def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; 989def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64>; 990 991def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_load_add_global_32, i32>; 992def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_load_sub_global_32, i32>; 993def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global_32, i32>; 994def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global_32, i32>; 995def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_load_and_global_32, i32>; 996def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_load_max_global_32, i32>; 997def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_load_umax_global_32, i32>; 998def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i32>; 999def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; 1000def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; 1001def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; 1002def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1003def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; 1004 1005def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; 1006def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_load_sub_global_64, i64>; 1007def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global_64, i64>; 1008def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global_64, i64>; 1009def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_load_and_global_64, i64>; 1010def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_load_max_global_64, i64>; 1011def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_load_umax_global_64, i64>; 1012def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, i64>; 1013def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; 1014def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; 1015def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; 1016def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1017def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; 1018 1019def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 1020def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 1021 1022let OtherPredicates = [D16PreservesUnusedBits] in { 1023def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 1024def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 1025 1026def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 1027def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 1028def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 1029def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 1030def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 1031def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 1032 1033def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 1034def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 1035def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 1036def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 1037def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 1038def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 1039} 1040 1041} // End OtherPredicates = [HasFlatAddressSpace] 1042 1043 1044multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1045 def : FlatLoadSignedPat <inst, node, vt> { 1046 let AddedComplexity = 10; 1047 } 1048 1049 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1050 let AddedComplexity = 11; 1051 } 1052} 1053 1054multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1055 def : FlatSignedLoadPat_D16 <inst, node, vt> { 1056 let AddedComplexity = 10; 1057 } 1058 1059 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1060 let AddedComplexity = 11; 1061 } 1062} 1063 1064multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1065 ValueType vt> { 1066 def : FlatStoreSignedPat <inst, node, vt> { 1067 let AddedComplexity = 10; 1068 } 1069 1070 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1071 let AddedComplexity = 11; 1072 } 1073} 1074 1075// Deal with swapped operands for atomic_store vs. regular store 1076multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1077 def : FlatStoreSignedAtomicPat <inst, node, vt> { 1078 let AddedComplexity = 10; 1079 } 1080 1081 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1082 let AddedComplexity = 11; 1083 } 1084} 1085 1086multiclass GlobalFLATAtomicPats<string nortn_inst_name, SDPatternOperator node, 1087 ValueType vt, ValueType data_vt = vt> { 1088 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 1089 let AddedComplexity = 10; 1090 } 1091 1092 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 1093 let AddedComplexity = 11; 1094 } 1095} 1096 1097multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1098 ValueType vt> { 1099 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1100 let AddedComplexity = 10; 1101 } 1102 1103 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1104 let AddedComplexity = 11; 1105 } 1106} 1107 1108multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1109 def : ScratchLoadSignedPat <inst, node, vt> { 1110 let AddedComplexity = 25; 1111 } 1112 1113 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1114 let AddedComplexity = 26; 1115 } 1116} 1117 1118multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1119 ValueType vt> { 1120 def : ScratchStoreSignedPat <inst, node, vt> { 1121 let AddedComplexity = 25; 1122 } 1123 1124 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1125 let AddedComplexity = 26; 1126 } 1127} 1128 1129multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1130 def : ScratchLoadSignedPat_D16 <inst, node, vt> { 1131 let AddedComplexity = 25; 1132 } 1133 1134 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1135 let AddedComplexity = 26; 1136 } 1137} 1138 1139let OtherPredicates = [HasFlatGlobalInsts] in { 1140 1141defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1142defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1143defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1144defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1145defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1146defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1147defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1148defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1149defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1150defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1151 1152foreach vt = Reg32Types.types in { 1153defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1154defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1155} 1156 1157foreach vt = VReg_64.RegTypes in { 1158defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1159defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1160} 1161 1162defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1163 1164foreach vt = VReg_128.RegTypes in { 1165defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1166defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1167} 1168 1169// There is no distinction for atomic load lowering during selection; 1170// the memory legalizer will set the cache bits and insert the 1171// appropriate waits. 1172defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1173defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1174 1175defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1176defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1177defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1178defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1179defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1180 1181let OtherPredicates = [D16PreservesUnusedBits] in { 1182defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1183defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1184 1185defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1186defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1187defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1188defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1189defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1190defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1191 1192defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1193defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1194defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1195defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1196defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1197defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1198} 1199 1200defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; 1201defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64>; 1202 1203defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", atomic_load_add_global_32, i32>; 1204defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", atomic_load_sub_global_32, i32>; 1205defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", atomic_inc_global_32, i32>; 1206defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", atomic_dec_global_32, i32>; 1207defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", atomic_load_and_global_32, i32>; 1208defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", atomic_load_max_global_32, i32>; 1209defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", atomic_load_umax_global_32, i32>; 1210defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", atomic_load_min_global_32, i32>; 1211defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", atomic_load_umin_global_32, i32>; 1212defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", atomic_load_or_global_32, i32>; 1213defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", atomic_swap_global_32, i32>; 1214defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1215defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", atomic_load_xor_global_32, i32>; 1216defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1217 1218defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", atomic_load_add_global_64, i64>; 1219defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", atomic_load_sub_global_64, i64>; 1220defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", atomic_inc_global_64, i64>; 1221defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", atomic_dec_global_64, i64>; 1222defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", atomic_load_and_global_64, i64>; 1223defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", atomic_load_max_global_64, i64>; 1224defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", atomic_load_umax_global_64, i64>; 1225defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", atomic_load_min_global_64, i64>; 1226defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", atomic_load_umin_global_64, i64>; 1227defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", atomic_load_or_global_64, i64>; 1228defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", atomic_swap_global_64, i64>; 1229defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1230defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", atomic_load_xor_global_64, i64>; 1231 1232let OtherPredicates = [HasAtomicFaddInsts] in { 1233defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1234defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1235} 1236 1237let OtherPredicates = [isGFX90APlus] in { 1238defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F32", atomic_load_fadd_global_32, f32>; 1239defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", atomic_load_fadd_v2f16_global_32, v2f16>; 1240defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", atomic_load_fadd_global_64, f64>; 1241defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", atomic_load_fmin_global_64, f64>; 1242defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", atomic_load_fmax_global_64, f64>; 1243def : FlatSignedAtomicPat <FLAT_ATOMIC_ADD_F64_RTN, atomic_load_fadd_flat_64, f64>; 1244def : FlatSignedAtomicPat <FLAT_ATOMIC_MIN_F64_RTN, atomic_load_fmin_flat_64, f64>; 1245def : FlatSignedAtomicPat <FLAT_ATOMIC_MAX_F64_RTN, atomic_load_fmax_flat_64, f64>; 1246} 1247 1248} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1249 1250let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { 1251 1252defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; 1253defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; 1254defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; 1255defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; 1256defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; 1257defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; 1258defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; 1259defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; 1260defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; 1261defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; 1262 1263foreach vt = Reg32Types.types in { 1264defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; 1265defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; 1266} 1267 1268foreach vt = VReg_64.RegTypes in { 1269defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; 1270defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; 1271} 1272 1273defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; 1274 1275foreach vt = VReg_128.RegTypes in { 1276defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; 1277defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; 1278} 1279 1280defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; 1281defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; 1282defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; 1283defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; 1284defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; 1285 1286let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { 1287defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; 1288defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; 1289 1290defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; 1291defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; 1292defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; 1293defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; 1294defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; 1295defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; 1296 1297defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; 1298defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; 1299defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; 1300defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; 1301defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; 1302defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; 1303} 1304 1305} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] 1306 1307//===----------------------------------------------------------------------===// 1308// Target 1309//===----------------------------------------------------------------------===// 1310 1311//===----------------------------------------------------------------------===// 1312// CI 1313//===----------------------------------------------------------------------===// 1314 1315class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1316 FLAT_Real <op, ps>, 1317 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1318 let AssemblerPredicate = isGFX7Only; 1319 let DecoderNamespace="GFX7"; 1320} 1321 1322def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1323def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1324def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1325def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1326def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1327def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1328def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1329def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1330 1331def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1332def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1333def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1334def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1335def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1336def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1337 1338multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1339 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1340 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1341} 1342 1343defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1344defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1345defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1346defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1347defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1348defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1349defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1350defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1351defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1352defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1353defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1354defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1355defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1356defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1357defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1358defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1359defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1360defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1361defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1362defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1363defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1364defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1365defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1366defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1367defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1368defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1369 1370// CI Only flat instructions 1371defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1372defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1373defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1374defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1375defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1376defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1377 1378 1379//===----------------------------------------------------------------------===// 1380// VI 1381//===----------------------------------------------------------------------===// 1382 1383class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> : 1384 FLAT_Real <op, ps>, 1385 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1386 let AssemblerPredicate = isGFX8GFX9; 1387 let DecoderNamespace = "GFX8"; 1388 1389 let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1390 let AsmString = ps.Mnemonic # 1391 !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands); 1392} 1393 1394multiclass FLAT_Real_AllAddr_vi<bits<7> op, 1395 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1396 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>; 1397 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>; 1398} 1399 1400def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1401def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1402def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1403def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1404def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1405def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1406def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1407def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1408 1409def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1410def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1411def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1412def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1413def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1414def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1415def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1416def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1417 1418def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1419def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1420def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1421def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1422def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1423def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1424 1425multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps, 1426 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1427 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>; 1428 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>; 1429} 1430 1431multiclass FLAT_Global_Real_Atomics_vi<bits<7> op, 1432 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> : 1433 FLAT_Real_AllAddr_vi<op, has_sccb> { 1434 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>; 1435 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>; 1436} 1437 1438 1439defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1440defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1441defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1442defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1443defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1444defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1445defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1446defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1447defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1448defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1449defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1450defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1451defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1452defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1453defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1454defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1455defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1456defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1457defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1458defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1459defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1460defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1461defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1462defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1463defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1464defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1465 1466defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1467defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1468defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1469defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1470defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1471defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1472defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1473defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1474 1475defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1476defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1477defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1478defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1479defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1480defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1481 1482defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1483defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1484defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1485defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1486defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1487defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1488defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1489defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1490 1491 1492defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1493defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1494defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1495defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1496defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1497defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1498defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1499defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1500defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1501defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1502defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1503defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1504defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1505defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1506defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1507defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1508defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1509defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1510defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1511defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1512defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1513defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1514defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1515defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1516defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1517defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1518 1519defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1520defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1521defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1522defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1523defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1524defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1525defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1526defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1527defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1528defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1529defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1530defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1531defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1532defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1533defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1534defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1535defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1536defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1537defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1538defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1539defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1540defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1541 1542let SubtargetPredicate = isGFX908orGFX90A in { 1543defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>; 1544defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>; 1545} 1546 1547let SubtargetPredicate = isGFX90AOnly in { 1548 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64, 0>; 1549 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64, 0>; 1550 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64, 0>; 1551 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; 1552 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; 1553 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; 1554} // End SubtargetPredicate = isGFX90AOnly 1555 1556//===----------------------------------------------------------------------===// 1557// GFX10. 1558//===----------------------------------------------------------------------===// 1559 1560class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1561 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1562 let AssemblerPredicate = isGFX10Plus; 1563 let DecoderNamespace = "GFX10"; 1564 1565 let Inst{11-0} = offset{11-0}; 1566 let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); 1567 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1568 let Inst{55} = 0; 1569} 1570 1571 1572multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1573 def _gfx10 : 1574 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1575} 1576 1577multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1578 def _RTN_gfx10 : 1579 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1580} 1581 1582multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1583 def _SADDR_gfx10 : 1584 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1585} 1586 1587multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1588 def _SADDR_RTN_gfx10 : 1589 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1590} 1591 1592multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1593 def _ST_gfx10 : 1594 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1595 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1596 let OtherPredicates = [HasFlatScratchSTMode]; 1597 } 1598} 1599 1600multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1601 FLAT_Real_Base_gfx10<op>, 1602 FLAT_Real_SADDR_gfx10<op>; 1603 1604multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1605 FLAT_Real_Base_gfx10<op>, 1606 FLAT_Real_RTN_gfx10<op>; 1607 1608multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1609 FLAT_Real_AllAddr_gfx10<op>, 1610 FLAT_Real_RTN_gfx10<op>, 1611 FLAT_Real_SADDR_RTN_gfx10<op>; 1612 1613multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1614 FLAT_Real_RTN_gfx10<op>, 1615 FLAT_Real_SADDR_RTN_gfx10<op>; 1616 1617multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1618 FLAT_Real_Base_gfx10<op>, 1619 FLAT_Real_SADDR_gfx10<op>, 1620 FLAT_Real_ST_gfx10<op>; 1621 1622// ENC_FLAT. 1623defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1624defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1625defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1626defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1627defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1628defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1629defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1630defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1631defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1632defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1633defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1634defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1635defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1636defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1637defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1638defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1639defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1640defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1641defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1642defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1643defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1644defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1645defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1646defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1647defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1648defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1649defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1650defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1651defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1652defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1653defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1654defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1655defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1656defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1657defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1658defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1659defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1660defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1661defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1662defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1663defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1664defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1665defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1666defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1667defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1668defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1669defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1670defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1671defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1672defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1673defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1674defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1675defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1676defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1677 1678 1679// ENC_FLAT_GLBL. 1680defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1681defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1682defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1683defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1684defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1685defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1686defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1687defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1688defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1689defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1690defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1691defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1692defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1693defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1694defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1695defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1696defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1697defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1698defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1699defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1700defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1701defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1702defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 1703defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 1704defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 1705defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 1706defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 1707defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 1708defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 1709defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 1710defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 1711defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 1712defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 1713defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 1714defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 1715defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 1716defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 1717defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 1718defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 1719defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 1720defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 1721defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 1722defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 1723defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 1724defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 1725defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 1726defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 1727defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 1728defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 1729defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 1730defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 1731defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 1732defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 1733defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 1734defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 1735defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; 1736defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; 1737 1738// ENC_FLAT_SCRATCH. 1739defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 1740defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 1741defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 1742defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 1743defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 1744defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 1745defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 1746defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 1747defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 1748defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 1749defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 1750defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 1751defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 1752defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 1753defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 1754defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 1755defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 1756defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 1757defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 1758defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 1759defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 1760defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 1761