1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FLATOffset : ComplexPattern<i64, 2, "SelectFlatOffset<false>", [], [SDNPWantRoot], -10>; 10def FLATOffsetSigned : ComplexPattern<i64, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 11def ScratchOffset : ComplexPattern<i32, 2, "SelectFlatOffset<true>", [], [SDNPWantRoot], -10>; 12 13def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 14def ScratchSAddr : ComplexPattern<i32, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; 15 16//===----------------------------------------------------------------------===// 17// FLAT classes 18//===----------------------------------------------------------------------===// 19 20class FLAT_Pseudo<string opName, dag outs, dag ins, 21 string asmOps, list<dag> pattern=[]> : 22 InstSI<outs, ins, "", pattern>, 23 SIMCInstr<opName, SIEncodingFamily.NONE> { 24 25 let isPseudo = 1; 26 let isCodeGenOnly = 1; 27 28 let FLAT = 1; 29 30 let UseNamedOperandTable = 1; 31 let hasSideEffects = 0; 32 let SchedRW = [WriteVMEM]; 33 34 string Mnemonic = opName; 35 string AsmOperands = asmOps; 36 37 bits<1> is_flat_global = 0; 38 bits<1> is_flat_scratch = 0; 39 40 bits<1> has_vdst = 1; 41 42 // We need to distinguish having saddr and enabling saddr because 43 // saddr is only valid for scratch and global instructions. Pre-gfx9 44 // these bits were reserved, so we also don't necessarily want to 45 // set these bits to the disabled value for the original flat 46 // segment instructions. 47 bits<1> has_saddr = 0; 48 bits<1> enabled_saddr = 0; 49 bits<7> saddr_value = 0; 50 bits<1> has_vaddr = 1; 51 52 bits<1> has_data = 1; 53 bits<1> has_glc = 1; 54 bits<1> glcValue = 0; 55 bits<1> has_dlc = 1; 56 bits<1> dlcValue = 0; 57 bits<1> has_sccb = 1; 58 bits<1> sccbValue = 0; 59 60 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 61 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 62 63 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 64 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 65 66 // Internally, FLAT instruction are executed as both an LDS and a 67 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 68 // and are not considered done until both have been decremented. 69 let VM_CNT = 1; 70 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); 71 72 let IsFlatGlobal = is_flat_global; 73 74 let IsFlatScratch = is_flat_scratch; 75} 76 77class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 78 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 79 Enc64 { 80 81 let isPseudo = 0; 82 let isCodeGenOnly = 0; 83 84 // copy relevant pseudo op flags 85 let SubtargetPredicate = ps.SubtargetPredicate; 86 let AsmMatchConverter = ps.AsmMatchConverter; 87 let OtherPredicates = ps.OtherPredicates; 88 let TSFlags = ps.TSFlags; 89 let UseNamedOperandTable = ps.UseNamedOperandTable; 90 91 // encoding fields 92 bits<8> vaddr; 93 bits<10> vdata; 94 bits<7> saddr; 95 bits<10> vdst; 96 97 bits<1> slc; 98 bits<1> glc; 99 bits<1> dlc; 100 101 // Only valid on gfx90a+ 102 bits<1> sccb; 103 104 // Only valid on gfx9 105 bits<1> lds = 0; // XXX - What does this actually do? 106 107 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 108 bits<2> seg = !if(ps.is_flat_global, 0b10, 109 !if(ps.is_flat_scratch, 0b01, 0)); 110 111 // Signed offset. Highest bit ignored for flat and treated as 12-bit 112 // unsigned for flat accesses. 113 bits<13> offset; 114 // GFX90A+ only: instruction uses AccVGPR for data 115 bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); 116 117 // We don't use tfe right now, and it was removed in gfx9. 118 bits<1> tfe = 0; 119 120 // Only valid on GFX9+ 121 let Inst{12-0} = offset; 122 let Inst{13} = lds; 123 let Inst{15-14} = seg; 124 125 let Inst{16} = !if(ps.has_glc, glc, ps.glcValue); 126 let Inst{17} = slc; 127 let Inst{24-18} = op; 128 let Inst{31-26} = 0x37; // Encoding. 129 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 130 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); 131 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 132 133 // 54-48 is reserved. 134 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. 135 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); 136} 137 138class GlobalSaddrTable <bit is_saddr, string Name = ""> { 139 bit IsSaddr = is_saddr; 140 string SaddrOp = Name; 141} 142 143// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 144// same encoding value as exec_hi, so it isn't possible to use that if 145// saddr is 32-bit (which isn't handled here yet). 146class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 147 bit HasTiedOutput = 0, 148 bit HasSaddr = 0, bit EnableSaddr = 0, 149 RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< 150 opName, 151 (outs vdata_op:$vdst), 152 !con( 153 !con( 154 !if(EnableSaddr, 155 (ins SReg_64:$saddr, VGPR_32:$vaddr), 156 (ins VReg_64:$vaddr)), 157 (ins flat_offset:$offset)), 158 // FIXME: Operands with default values do not work with following non-optional operands. 159 !if(HasTiedOutput, (ins GLC:$glc, SLC:$slc, DLC:$dlc, SCCB:$sccb, vdata_op:$vdst_in), 160 (ins GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb))), 161 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc$sccb"> { 162 let has_data = 0; 163 let mayLoad = 1; 164 let has_saddr = HasSaddr; 165 let enabled_saddr = EnableSaddr; 166 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 167 let maybeAtomic = 1; 168 169 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 170 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 171} 172 173class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 174 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 175 opName, 176 (outs), 177 !con( 178 !if(EnableSaddr, 179 (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64:$saddr), 180 (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), 181 (ins flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb)), 182 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc$dlc$sccb"> { 183 let mayLoad = 0; 184 let mayStore = 1; 185 let has_vdst = 0; 186 let has_saddr = HasSaddr; 187 let enabled_saddr = EnableSaddr; 188 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 189 let maybeAtomic = 1; 190} 191 192multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 193 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 194 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 195 GlobalSaddrTable<0, opName>; 196 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 197 GlobalSaddrTable<1, opName>; 198 } 199} 200 201class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 202 bit HasTiedOutput = 0, bit HasSignedOffset = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 203 opName, 204 (outs regClass:$vdst), 205 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), 206 (ins flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb), 207 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 208 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc$sccb"> { 209 let is_flat_global = 1; 210 let has_data = 0; 211 let mayLoad = 1; 212 let has_vaddr = 0; 213 let has_saddr = 1; 214 let enabled_saddr = EnableSaddr; 215 let maybeAtomic = 1; 216 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 217 218 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 219 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 220} 221 222multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, 223 bit HasTiedOutput = 0, bit HasSignedOffset = 0> { 224 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, HasSignedOffset>, 225 GlobalSaddrTable<0, opName>; 226 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, HasSignedOffset, 1>, 227 GlobalSaddrTable<1, opName>; 228} 229 230multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 231 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 232 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 233 GlobalSaddrTable<0, opName>; 234 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 235 GlobalSaddrTable<1, opName>; 236 } 237} 238 239class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 240 bit HasSignedOffset = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 241 opName, 242 (outs), 243 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), 244 (ins flat_offset:$offset, GLC:$glc, SLC:$slc, DLC:$dlc, SCCB_0:$sccb)), 245 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc$sccb"> { 246 let is_flat_global = 1; 247 let mayLoad = 0; 248 let mayStore = 1; 249 let has_vdst = 0; 250 let has_vaddr = 0; 251 let has_saddr = 1; 252 let enabled_saddr = EnableSaddr; 253 let maybeAtomic = 1; 254 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 255} 256 257multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass, 258 bit HasSignedOffset = 0> { 259 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, HasSignedOffset>, 260 GlobalSaddrTable<0, opName>; 261 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, HasSignedOffset, 1>, 262 GlobalSaddrTable<1, opName>; 263} 264 265class FlatScratchInst <string sv_op, string mode> { 266 string SVOp = sv_op; 267 string Mode = mode; 268} 269 270class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 271 bit HasTiedOutput = 0, 272 bit EnableSaddr = 0, 273 bit EnableVaddr = !not(EnableSaddr)> 274 : FLAT_Pseudo< 275 opName, 276 (outs getLdStRegisterOperand<regClass>.ret:$vdst), 277 !con( 278 !if(EnableSaddr, 279 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 280 !if(EnableVaddr, 281 (ins VGPR_32:$vaddr, flat_offset:$offset), 282 (ins flat_offset:$offset))), 283 !if(HasTiedOutput, (ins GLC:$glc, SLC:$slc, DLC:$dlc, SCCB:$sccb, getLdStRegisterOperand<regClass>.ret:$vdst_in), 284 (ins GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb))), 285 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc$sccb"> { 286 let has_data = 0; 287 let mayLoad = 1; 288 let has_saddr = 1; 289 let enabled_saddr = EnableSaddr; 290 let has_vaddr = EnableVaddr; 291 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 292 let maybeAtomic = 1; 293 294 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 295 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 296} 297 298class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 299 bit EnableVaddr = !not(EnableSaddr), 300 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< 301 opName, 302 (outs), 303 !if(EnableSaddr, 304 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb), 305 !if(EnableVaddr, 306 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb), 307 (ins vdata_op:$vdata, flat_offset:$offset, GLC_0:$glc, SLC_0:$slc, DLC_0:$dlc, SCCB_0:$sccb))), 308 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc$dlc$sccb"> { 309 let mayLoad = 0; 310 let mayStore = 1; 311 let has_vdst = 0; 312 let has_saddr = 1; 313 let enabled_saddr = EnableSaddr; 314 let has_vaddr = EnableVaddr; 315 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")); 316 let maybeAtomic = 1; 317} 318 319multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 320 let is_flat_scratch = 1 in { 321 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, 322 FlatScratchInst<opName, "SV">; 323 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, 324 FlatScratchInst<opName, "SS">; 325 326 let SubtargetPredicate = HasFlatScratchSTMode in 327 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0>, 328 FlatScratchInst<opName, "ST">; 329 } 330} 331 332multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 333 let is_flat_scratch = 1 in { 334 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, 335 FlatScratchInst<opName, "SV">; 336 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, 337 FlatScratchInst<opName, "SS">; 338 339 let SubtargetPredicate = HasFlatScratchSTMode in 340 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0>, 341 FlatScratchInst<opName, "ST">; 342 } 343} 344 345class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 346 string asm, list<dag> pattern = []> : 347 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 348 let mayLoad = 1; 349 let mayStore = 1; 350 let has_glc = 0; 351 let glcValue = 0; 352 let has_dlc = 0; 353 let dlcValue = 0; 354 let has_vdst = 0; 355 let has_sccb = 1; 356 let sccbValue = 0; 357 let maybeAtomic = 1; 358 let IsAtomicNoRet = 1; 359} 360 361class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 362 string asm, list<dag> pattern = []> 363 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 364 let hasPostISelHook = 1; 365 let has_vdst = 1; 366 let glcValue = 1; 367 let dlcValue = 0; 368 let sccbValue = 0; 369 let IsAtomicNoRet = 0; 370 let IsAtomicRet = 1; 371 let PseudoInstr = NAME # "_RTN"; 372} 373 374multiclass FLAT_Atomic_Pseudo< 375 string opName, 376 RegisterClass vdst_rc, 377 ValueType vt, 378 SDPatternOperator atomic = null_frag, 379 ValueType data_vt = vt, 380 RegisterClass data_rc = vdst_rc, 381 bit isFP = isFloatType<data_vt>.ret, 382 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 383 def "" : FLAT_AtomicNoRet_Pseudo <opName, 384 (outs), 385 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, SLC_0:$slc, SCCB_0:$sccb), 386 " $vaddr, $vdata$offset$slc$sccb">, 387 GlobalSaddrTable<0, opName>, 388 AtomicNoRet <opName, 0> { 389 let PseudoInstr = NAME; 390 let FPAtomic = isFP; 391 let AddedComplexity = -1; // Prefer global atomics if available 392 } 393 394 def _RTN : FLAT_AtomicRet_Pseudo <opName, 395 (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), 396 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, GLC_1:$glc1, SLC_0:$slc, SCCB_0:$sccb), 397 " $vdst, $vaddr, $vdata$offset$glc1$slc$sccb", 398 [(set vt:$vdst, 399 (atomic (FLATOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 400 GlobalSaddrTable<0, opName#"_rtn">, 401 AtomicNoRet <opName, 1>{ 402 let FPAtomic = isFP; 403 let AddedComplexity = -1; // Prefer global atomics if available 404 } 405} 406 407multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 408 string opName, 409 RegisterClass vdst_rc, 410 ValueType vt, 411 SDPatternOperator atomic = null_frag, 412 ValueType data_vt = vt, 413 RegisterClass data_rc = vdst_rc, 414 bit isFP = isFloatType<data_vt>.ret, 415 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 416 417 def "" : FLAT_AtomicNoRet_Pseudo <opName, 418 (outs), 419 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, SLC_0:$slc, SCCB_0:$sccb), 420 " $vaddr, $vdata, off$offset$slc$sccb">, 421 GlobalSaddrTable<0, opName>, 422 AtomicNoRet <opName, 0> { 423 let has_saddr = 1; 424 let PseudoInstr = NAME; 425 let FPAtomic = isFP; 426 } 427 428 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 429 (outs), 430 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, SLC_0:$slc, SCCB_0:$sccb), 431 " $vaddr, $vdata, $saddr$offset$slc$sccb">, 432 GlobalSaddrTable<1, opName>, 433 AtomicNoRet <opName#"_saddr", 0> { 434 let has_saddr = 1; 435 let enabled_saddr = 1; 436 let PseudoInstr = NAME#"_SADDR"; 437 let FPAtomic = isFP; 438 } 439} 440 441multiclass FLAT_Global_Atomic_Pseudo_RTN< 442 string opName, 443 RegisterClass vdst_rc, 444 ValueType vt, 445 SDPatternOperator atomic = null_frag, 446 ValueType data_vt = vt, 447 RegisterClass data_rc = vdst_rc, 448 bit isFP = isFloatType<data_vt>.ret, 449 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, 450 RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { 451 452 def _RTN : FLAT_AtomicRet_Pseudo <opName, 453 (outs vdst_op:$vdst), 454 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, GLC_1:$glc1, SLC_0:$slc, SCCB_0:$sccb), 455 " $vdst, $vaddr, $vdata, off$offset$glc1$slc$sccb", 456 [(set vt:$vdst, 457 (atomic (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 458 GlobalSaddrTable<0, opName#"_rtn">, 459 AtomicNoRet <opName, 1> { 460 let has_saddr = 1; 461 let FPAtomic = isFP; 462 } 463 464 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 465 (outs vdst_op:$vdst), 466 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, GLC_1:$glc1, SLC_0:$slc, SCCB_0:$sccb), 467 " $vdst, $vaddr, $vdata, $saddr$offset$glc1$slc$sccb">, 468 GlobalSaddrTable<1, opName#"_rtn">, 469 AtomicNoRet <opName#"_saddr", 1> { 470 let has_saddr = 1; 471 let enabled_saddr = 1; 472 let PseudoInstr = NAME#"_SADDR_RTN"; 473 let FPAtomic = isFP; 474 } 475} 476 477multiclass FLAT_Global_Atomic_Pseudo< 478 string opName, 479 RegisterClass vdst_rc, 480 ValueType vt, 481 SDPatternOperator atomic_rtn = null_frag, 482 SDPatternOperator atomic_no_rtn = null_frag, 483 ValueType data_vt = vt, 484 RegisterClass data_rc = vdst_rc> { 485 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 486 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>; 487 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; 488 } 489} 490 491//===----------------------------------------------------------------------===// 492// Flat Instructions 493//===----------------------------------------------------------------------===// 494 495def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 496def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 497def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 498def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 499def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 500def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 501def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 502def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 503 504def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 505def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 506def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 507def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 508def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 509def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 510 511let SubtargetPredicate = HasD16LoadStore in { 512def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 513def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 514def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 515def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 516def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 517def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 518 519def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 520def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 521} 522 523defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 524 VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, 525 v2i32, VReg_64>; 526 527defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 528 VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, 529 v2i64, VReg_128>; 530 531defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 532 VGPR_32, i32, atomic_swap_flat_32>; 533 534defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 535 VReg_64, i64, atomic_swap_flat_64>; 536 537defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 538 VGPR_32, i32, atomic_load_add_flat_32>; 539 540defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 541 VGPR_32, i32, atomic_load_sub_flat_32>; 542 543defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 544 VGPR_32, i32, atomic_load_min_flat_32>; 545 546defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 547 VGPR_32, i32, atomic_load_umin_flat_32>; 548 549defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 550 VGPR_32, i32, atomic_load_max_flat_32>; 551 552defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 553 VGPR_32, i32, atomic_load_umax_flat_32>; 554 555defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 556 VGPR_32, i32, atomic_load_and_flat_32>; 557 558defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 559 VGPR_32, i32, atomic_load_or_flat_32>; 560 561defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 562 VGPR_32, i32, atomic_load_xor_flat_32>; 563 564defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 565 VGPR_32, i32, atomic_inc_flat_32>; 566 567defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 568 VGPR_32, i32, atomic_dec_flat_32>; 569 570defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 571 VReg_64, i64, atomic_load_add_flat_64>; 572 573defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 574 VReg_64, i64, atomic_load_sub_flat_64>; 575 576defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 577 VReg_64, i64, atomic_load_min_flat_64>; 578 579defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 580 VReg_64, i64, atomic_load_umin_flat_64>; 581 582defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 583 VReg_64, i64, atomic_load_max_flat_64>; 584 585defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 586 VReg_64, i64, atomic_load_umax_flat_64>; 587 588defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 589 VReg_64, i64, atomic_load_and_flat_64>; 590 591defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 592 VReg_64, i64, atomic_load_or_flat_64>; 593 594defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 595 VReg_64, i64, atomic_load_xor_flat_64>; 596 597defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 598 VReg_64, i64, atomic_inc_flat_64>; 599 600defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 601 VReg_64, i64, atomic_dec_flat_64>; 602 603// GFX7-, GFX10-only flat instructions. 604let SubtargetPredicate = isGFX7GFX10 in { 605 606defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 607 VGPR_32, f32, null_frag, v2f32, VReg_64>; 608 609defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 610 VReg_64, f64, null_frag, v2f64, VReg_128>; 611 612defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 613 VGPR_32, f32>; 614 615defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 616 VGPR_32, f32>; 617 618defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 619 VReg_64, f64>; 620 621defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 622 VReg_64, f64>; 623 624} // End SubtargetPredicate = isGFX7GFX10 625 626let SubtargetPredicate = isGFX90APlus in { 627 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64, int_amdgcn_flat_atomic_fadd>; 628 defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo<"flat_atomic_min_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmin>; 629 defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo<"flat_atomic_max_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmax>; 630 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 631 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 632 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 633} // End SubtargetPredicate = isGFX90APlus 634 635defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 636defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 637defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 638defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 639defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 640defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 641defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 642defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 643 644defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 645defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 646defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 647defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 648defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 649defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 650let OtherPredicates = [HasGFX10_BEncoding] in 651defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 652 653defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 654defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 655defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 656defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 657defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 658defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 659let OtherPredicates = [HasGFX10_BEncoding] in 660defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 661 662defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 663defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 664 665let is_flat_global = 1 in { 666defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 667 VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag, 668 v2i32, VReg_64>; 669 670defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 671 VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, 672 null_frag, 673 v2i64, VReg_128>; 674 675defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 676 VGPR_32, i32, atomic_swap_global_32>; 677 678defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 679 VReg_64, i64, atomic_swap_global_64>; 680 681defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 682 VGPR_32, i32, atomic_load_add_global_32>; 683 684defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 685 VGPR_32, i32, atomic_load_sub_global_32>; 686 687defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 688 VGPR_32, i32, atomic_load_min_global_32>; 689 690defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 691 VGPR_32, i32, atomic_load_umin_global_32>; 692 693defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 694 VGPR_32, i32, atomic_load_max_global_32>; 695 696defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 697 VGPR_32, i32, atomic_load_umax_global_32>; 698 699defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 700 VGPR_32, i32, atomic_load_and_global_32>; 701 702defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 703 VGPR_32, i32, atomic_load_or_global_32>; 704 705defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 706 VGPR_32, i32, atomic_load_xor_global_32>; 707 708defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 709 VGPR_32, i32, atomic_inc_global_32>; 710 711defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 712 VGPR_32, i32, atomic_dec_global_32>; 713 714defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 715 VReg_64, i64, atomic_load_add_global_64>; 716 717defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 718 VReg_64, i64, atomic_load_sub_global_64>; 719 720defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 721 VReg_64, i64, atomic_load_min_global_64>; 722 723defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 724 VReg_64, i64, atomic_load_umin_global_64>; 725 726defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 727 VReg_64, i64, atomic_load_max_global_64>; 728 729defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 730 VReg_64, i64, atomic_load_umax_global_64>; 731 732defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 733 VReg_64, i64, atomic_load_and_global_64>; 734 735defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 736 VReg_64, i64, atomic_load_or_global_64>; 737 738defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 739 VReg_64, i64, atomic_load_xor_global_64>; 740 741defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 742 VReg_64, i64, atomic_inc_global_64>; 743 744defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 745 VReg_64, i64, atomic_dec_global_64>; 746 747let SubtargetPredicate = HasGFX10_BEncoding in 748defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 749 VGPR_32, i32, int_amdgcn_global_atomic_csub>; 750} // End is_flat_global = 1 751 752 753 754let SubtargetPredicate = HasFlatScratchInsts in { 755defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 756defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 757defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 758defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 759defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 760defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 761defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 762defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 763 764defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 765defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 766defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 767defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 768defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 769defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 770 771defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 772defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 773defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 774defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 775defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 776defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 777 778defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 779defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 780 781} // End SubtargetPredicate = HasFlatScratchInsts 782 783let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 784 defm GLOBAL_ATOMIC_FCMPSWAP : 785 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32>; 786 defm GLOBAL_ATOMIC_FMIN : 787 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 788 defm GLOBAL_ATOMIC_FMAX : 789 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 790 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 791 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>; 792 defm GLOBAL_ATOMIC_FMIN_X2 : 793 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 794 defm GLOBAL_ATOMIC_FMAX_X2 : 795 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 796} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 797 798let is_flat_global = 1 in { 799let OtherPredicates = [HasAtomicFaddInsts] in { 800 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 801 "global_atomic_add_f32", VGPR_32, f32 802 >; 803 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 804 "global_atomic_pk_add_f16", VGPR_32, v2f16 805 >; 806} // End OtherPredicates = [HasAtomicFaddInsts] 807 808let OtherPredicates = [isGFX90APlus] in { 809 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < 810 "global_atomic_add_f32", VGPR_32, f32, int_amdgcn_global_atomic_fadd 811 >; 812 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < 813 "global_atomic_pk_add_f16", VGPR_32, v2f16, int_amdgcn_global_atomic_fadd 814 >; 815} // End OtherPredicates = [isGFX90APlus] 816} // End is_flat_global = 1 817 818//===----------------------------------------------------------------------===// 819// Flat Patterns 820//===----------------------------------------------------------------------===// 821 822// Patterns for global loads with no offset. 823class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 824 (vt (node (FLATOffset i64:$vaddr, i16:$offset))), 825 (inst $vaddr, $offset) 826>; 827 828class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 829 (node (FLATOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 830 (inst $vaddr, $offset, 0, 0, 0, 0, $in) 831>; 832 833class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 834 (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 835 (inst $vaddr, $offset, 0, 0, 0, 0, $in) 836>; 837 838class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 839 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 840 (inst $saddr, $voffset, $offset, 0, 0, 0, 0, $in) 841>; 842 843class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 844 (vt (node (FLATOffsetSigned (i64 VReg_64:$vaddr), i16:$offset))), 845 (inst $vaddr, $offset) 846>; 847 848class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 849 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 850 (inst $saddr, $voffset, $offset, 0, 0, 0) 851>; 852 853class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 854 ValueType vt> : GCNPat < 855 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 856 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 857>; 858 859class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 860 ValueType vt> : GCNPat < 861 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 862 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 863>; 864 865class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 866 ValueType vt, ValueType data_vt = vt> : GCNPat < 867 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 868 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 869>; 870 871class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 872 ValueType vt> : GCNPat < 873 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 874 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 875>; 876 877class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 878 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset)), 879 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 880>; 881 882class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 883 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset)), 884 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 885>; 886 887class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 888 // atomic store follows atomic binop convention so the address comes 889 // first. 890 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 891 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 892>; 893 894class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 895 ValueType vt, ValueType data_vt = vt> : GCNPat < 896 // atomic store follows atomic binop convention so the address comes 897 // first. 898 (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data), 899 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 900>; 901 902class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 903 ValueType data_vt = vt> : GCNPat < 904 (vt (node (FLATOffset i64:$vaddr, i16:$offset), data_vt:$data)), 905 (inst $vaddr, $data, $offset) 906>; 907 908class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 909 (node (FLATOffset i64:$vaddr, i16:$offset), vt:$data), 910 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 911>; 912 913class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 914 (node (FLATOffsetSigned i64:$vaddr, i16:$offset), vt:$data), 915 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 916>; 917 918class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 919 ValueType data_vt = vt> : GCNPat < 920 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset), data_vt:$data)), 921 (inst $vaddr, $data, $offset) 922>; 923 924class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 925 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset))), 926 (inst $vaddr, $offset) 927>; 928 929class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 930 (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset), vt:$in), 931 (inst $vaddr, $offset, 0, 0, 0, 0, $in) 932>; 933 934class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 935 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset)), 936 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) 937>; 938 939class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 940 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset))), 941 (inst $saddr, $offset) 942>; 943 944class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 945 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 946 (inst $saddr, $offset, 0, 0, 0, 0, $in) 947>; 948 949class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 950 ValueType vt> : GCNPat < 951 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)), 952 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 953>; 954 955let OtherPredicates = [HasFlatAddressSpace] in { 956 957def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 958def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 959def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 960def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 961def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 962def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 963def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 964def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 965def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 966def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 967def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 968 969def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 970def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 971 972def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 973def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 974 975foreach vt = Reg32Types.types in { 976def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 977def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 978} 979 980foreach vt = VReg_64.RegTypes in { 981def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 982def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 983} 984 985def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 986 987foreach vt = VReg_128.RegTypes in { 988def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 989def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 990} 991 992def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; 993def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64>; 994 995def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_load_add_global_32, i32>; 996def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_load_sub_global_32, i32>; 997def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global_32, i32>; 998def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global_32, i32>; 999def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_load_and_global_32, i32>; 1000def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_load_max_global_32, i32>; 1001def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_load_umax_global_32, i32>; 1002def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i32>; 1003def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; 1004def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; 1005def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; 1006def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1007def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; 1008 1009def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; 1010def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_load_sub_global_64, i64>; 1011def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global_64, i64>; 1012def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global_64, i64>; 1013def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_load_and_global_64, i64>; 1014def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_load_max_global_64, i64>; 1015def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_load_umax_global_64, i64>; 1016def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, i64>; 1017def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; 1018def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; 1019def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; 1020def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1021def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; 1022 1023def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 1024def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 1025 1026let OtherPredicates = [D16PreservesUnusedBits] in { 1027def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 1028def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 1029 1030def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 1031def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 1032def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 1033def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 1034def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 1035def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 1036 1037def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 1038def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 1039def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 1040def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 1041def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 1042def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 1043} 1044 1045} // End OtherPredicates = [HasFlatAddressSpace] 1046 1047 1048multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1049 def : FlatLoadSignedPat <inst, node, vt> { 1050 let AddedComplexity = 10; 1051 } 1052 1053 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1054 let AddedComplexity = 11; 1055 } 1056} 1057 1058multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1059 def : FlatSignedLoadPat_D16 <inst, node, vt> { 1060 let AddedComplexity = 10; 1061 } 1062 1063 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1064 let AddedComplexity = 11; 1065 } 1066} 1067 1068multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1069 ValueType vt> { 1070 def : FlatStoreSignedPat <inst, node, vt> { 1071 let AddedComplexity = 10; 1072 } 1073 1074 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1075 let AddedComplexity = 11; 1076 } 1077} 1078 1079// Deal with swapped operands for atomic_store vs. regular store 1080multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1081 def : FlatStoreSignedAtomicPat <inst, node, vt> { 1082 let AddedComplexity = 10; 1083 } 1084 1085 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1086 let AddedComplexity = 11; 1087 } 1088} 1089 1090multiclass GlobalFLATAtomicPats<string nortn_inst_name, SDPatternOperator node, 1091 ValueType vt, ValueType data_vt = vt> { 1092 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 1093 let AddedComplexity = 10; 1094 } 1095 1096 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 1097 let AddedComplexity = 11; 1098 } 1099} 1100 1101multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1102 ValueType vt> { 1103 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1104 let AddedComplexity = 10; 1105 } 1106 1107 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1108 let AddedComplexity = 11; 1109 } 1110} 1111 1112multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1113 def : ScratchLoadSignedPat <inst, node, vt> { 1114 let AddedComplexity = 25; 1115 } 1116 1117 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1118 let AddedComplexity = 26; 1119 } 1120} 1121 1122multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1123 ValueType vt> { 1124 def : ScratchStoreSignedPat <inst, node, vt> { 1125 let AddedComplexity = 25; 1126 } 1127 1128 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1129 let AddedComplexity = 26; 1130 } 1131} 1132 1133multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1134 def : ScratchLoadSignedPat_D16 <inst, node, vt> { 1135 let AddedComplexity = 25; 1136 } 1137 1138 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1139 let AddedComplexity = 26; 1140 } 1141} 1142 1143let OtherPredicates = [HasFlatGlobalInsts] in { 1144 1145defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1146defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1147defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1148defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1149defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1150defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1151defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1152defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1153defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1154defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1155 1156foreach vt = Reg32Types.types in { 1157defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1158defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1159} 1160 1161foreach vt = VReg_64.RegTypes in { 1162defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1163defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1164} 1165 1166defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1167 1168foreach vt = VReg_128.RegTypes in { 1169defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1170defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1171} 1172 1173// There is no distinction for atomic load lowering during selection; 1174// the memory legalizer will set the cache bits and insert the 1175// appropriate waits. 1176defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1177defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1178 1179defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1180defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1181defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1182defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1183defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1184 1185let OtherPredicates = [D16PreservesUnusedBits] in { 1186defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1187defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1188 1189defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1190defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1191defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1192defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1193defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1194defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1195 1196defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1197defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1198defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1199defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1200defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1201defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1202} 1203 1204defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; 1205defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64>; 1206 1207defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", atomic_load_add_global_32, i32>; 1208defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", atomic_load_sub_global_32, i32>; 1209defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", atomic_inc_global_32, i32>; 1210defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", atomic_dec_global_32, i32>; 1211defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", atomic_load_and_global_32, i32>; 1212defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", atomic_load_max_global_32, i32>; 1213defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", atomic_load_umax_global_32, i32>; 1214defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", atomic_load_min_global_32, i32>; 1215defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", atomic_load_umin_global_32, i32>; 1216defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", atomic_load_or_global_32, i32>; 1217defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", atomic_swap_global_32, i32>; 1218defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; 1219defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", atomic_load_xor_global_32, i32>; 1220defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1221 1222defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", atomic_load_add_global_64, i64>; 1223defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", atomic_load_sub_global_64, i64>; 1224defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", atomic_inc_global_64, i64>; 1225defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", atomic_dec_global_64, i64>; 1226defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", atomic_load_and_global_64, i64>; 1227defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", atomic_load_max_global_64, i64>; 1228defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", atomic_load_umax_global_64, i64>; 1229defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", atomic_load_min_global_64, i64>; 1230defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", atomic_load_umin_global_64, i64>; 1231defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", atomic_load_or_global_64, i64>; 1232defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", atomic_swap_global_64, i64>; 1233defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; 1234defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", atomic_load_xor_global_64, i64>; 1235 1236let OtherPredicates = [HasAtomicFaddInsts] in { 1237defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1238defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1239} 1240 1241let OtherPredicates = [isGFX90APlus] in { 1242defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F32", atomic_load_fadd_global_32, f32>; 1243defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", atomic_load_fadd_v2f16_global_32, v2f16>; 1244defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", atomic_load_fadd_global_64, f64>; 1245defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", atomic_load_fmin_global_64, f64>; 1246defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", atomic_load_fmax_global_64, f64>; 1247def : FlatSignedAtomicPat <FLAT_ATOMIC_ADD_F64_RTN, atomic_load_fadd_flat_64, f64>; 1248def : FlatSignedAtomicPat <FLAT_ATOMIC_MIN_F64_RTN, atomic_load_fmin_flat_64, f64>; 1249def : FlatSignedAtomicPat <FLAT_ATOMIC_MAX_F64_RTN, atomic_load_fmax_flat_64, f64>; 1250} 1251 1252} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1253 1254let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { 1255 1256defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; 1257defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; 1258defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; 1259defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; 1260defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; 1261defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; 1262defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; 1263defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; 1264defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; 1265defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; 1266 1267foreach vt = Reg32Types.types in { 1268defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; 1269defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; 1270} 1271 1272foreach vt = VReg_64.RegTypes in { 1273defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; 1274defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; 1275} 1276 1277defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; 1278 1279foreach vt = VReg_128.RegTypes in { 1280defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; 1281defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; 1282} 1283 1284defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; 1285defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; 1286defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; 1287defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; 1288defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; 1289 1290let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { 1291defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; 1292defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; 1293 1294defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; 1295defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; 1296defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; 1297defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; 1298defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; 1299defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; 1300 1301defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; 1302defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; 1303defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; 1304defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; 1305defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; 1306defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; 1307} 1308 1309} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] 1310 1311//===----------------------------------------------------------------------===// 1312// Target 1313//===----------------------------------------------------------------------===// 1314 1315//===----------------------------------------------------------------------===// 1316// CI 1317//===----------------------------------------------------------------------===// 1318 1319class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1320 FLAT_Real <op, ps>, 1321 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1322 let AssemblerPredicate = isGFX7Only; 1323 let DecoderNamespace="GFX7"; 1324} 1325 1326def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1327def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1328def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1329def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1330def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1331def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1332def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1333def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1334 1335def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1336def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1337def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1338def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1339def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1340def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1341 1342multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1343 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1344 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1345} 1346 1347defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1348defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1349defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1350defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1351defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1352defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1353defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1354defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1355defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1356defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1357defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1358defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1359defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1360defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1361defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1362defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1363defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1364defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1365defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1366defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1367defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1368defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1369defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1370defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1371defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1372defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1373 1374// CI Only flat instructions 1375defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1376defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1377defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1378defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1379defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1380defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1381 1382 1383//===----------------------------------------------------------------------===// 1384// VI 1385//===----------------------------------------------------------------------===// 1386 1387class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps> : 1388 FLAT_Real <op, ps>, 1389 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1390 let AssemblerPredicate = isGFX8GFX9; 1391 let DecoderNamespace = "GFX8"; 1392 1393 let Inst{25} = !if(ps.has_sccb, sccb, ps.sccbValue); 1394} 1395 1396multiclass FLAT_Real_AllAddr_vi<bits<7> op> { 1397 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)>; 1398 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1399} 1400 1401def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1402def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1403def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1404def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1405def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1406def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1407def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1408def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1409 1410def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1411def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1412def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1413def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1414def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1415def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1416def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1417def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1418 1419def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1420def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1421def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1422def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1423def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1424def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1425 1426multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> { 1427 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1428 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1429} 1430 1431multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> : 1432 FLAT_Real_AllAddr_vi<op> { 1433 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1434 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1435} 1436 1437 1438defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1439defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1440defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1441defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1442defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1443defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1444defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1445defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1446defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1447defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1448defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1449defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1450defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1451defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1452defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1453defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1454defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1455defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1456defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1457defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1458defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1459defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1460defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1461defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1462defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1463defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1464 1465defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1466defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1467defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1468defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1469defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1470defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1471defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1472defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1473 1474defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1475defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1476defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1477defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1478defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1479defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1480 1481defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1482defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1483defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1484defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1485defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1486defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1487defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1488defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1489 1490 1491defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1492defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1493defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1494defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1495defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1496defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1497defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1498defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1499defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1500defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1501defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1502defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1503defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1504defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1505defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1506defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1507defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1508defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1509defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1510defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1511defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1512defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1513defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1514defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1515defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1516defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1517 1518defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1519defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1520defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1521defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1522defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1523defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1524defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1525defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1526defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1527defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1528defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1529defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1530defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1531defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1532defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1533defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1534defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1535defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1536defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1537defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1538defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1539defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1540 1541let SubtargetPredicate = isGFX90APlus in { 1542 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64>; 1543 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64>; 1544 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64>; 1545 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f>; 1546 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50>; 1547 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51>; 1548} // End SubtargetPredicate = isGFX90APlus 1549 1550//===----------------------------------------------------------------------===// 1551// GFX10. 1552//===----------------------------------------------------------------------===// 1553 1554class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1555 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1556 let AssemblerPredicate = isGFX10Plus; 1557 let DecoderNamespace = "GFX10"; 1558 1559 let Inst{11-0} = offset{11-0}; 1560 let Inst{12} = !if(ps.has_dlc, dlc, ps.dlcValue); 1561 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1562 let Inst{55} = 0; 1563} 1564 1565 1566multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1567 def _gfx10 : 1568 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1569} 1570 1571multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1572 def _RTN_gfx10 : 1573 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1574} 1575 1576multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1577 def _SADDR_gfx10 : 1578 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1579} 1580 1581multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1582 def _SADDR_RTN_gfx10 : 1583 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1584} 1585 1586multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1587 def _ST_gfx10 : 1588 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1589 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1590 let OtherPredicates = [HasFlatScratchSTMode]; 1591 } 1592} 1593 1594multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1595 FLAT_Real_Base_gfx10<op>, 1596 FLAT_Real_SADDR_gfx10<op>; 1597 1598multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1599 FLAT_Real_Base_gfx10<op>, 1600 FLAT_Real_RTN_gfx10<op>; 1601 1602multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1603 FLAT_Real_AllAddr_gfx10<op>, 1604 FLAT_Real_RTN_gfx10<op>, 1605 FLAT_Real_SADDR_RTN_gfx10<op>; 1606 1607multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1608 FLAT_Real_RTN_gfx10<op>, 1609 FLAT_Real_SADDR_RTN_gfx10<op>; 1610 1611multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1612 FLAT_Real_Base_gfx10<op>, 1613 FLAT_Real_SADDR_gfx10<op>, 1614 FLAT_Real_ST_gfx10<op>; 1615 1616// ENC_FLAT. 1617defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1618defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1619defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1620defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1621defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1622defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1623defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1624defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1625defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1626defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1627defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1628defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1629defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1630defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1631defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1632defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1633defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1634defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1635defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1636defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1637defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1638defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1639defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1640defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1641defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1642defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1643defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1644defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1645defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1646defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1647defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1648defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1649defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1650defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1651defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1652defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1653defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1654defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1655defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1656defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1657defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1658defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1659defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1660defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1661defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1662defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1663defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1664defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1665defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1666defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1667defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1668defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1669defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1670defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1671 1672 1673// ENC_FLAT_GLBL. 1674defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1675defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1676defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1677defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1678defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1679defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1680defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1681defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1682defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1683defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1684defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1685defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1686defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1687defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1688defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1689defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1690defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1691defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1692defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1693defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1694defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1695defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1696defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 1697defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 1698defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 1699defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 1700defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 1701defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 1702defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 1703defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 1704defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 1705defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 1706defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 1707defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 1708defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 1709defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 1710defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 1711defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 1712defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 1713defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 1714defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 1715defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 1716defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 1717defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 1718defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 1719defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 1720defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 1721defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 1722defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 1723defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 1724defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 1725defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 1726defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 1727defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 1728defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 1729defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; 1730defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; 1731 1732// ENC_FLAT_SCRATCH. 1733defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 1734defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 1735defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 1736defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 1737defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 1738defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 1739defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 1740defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 1741defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 1742defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 1743defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 1744defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 1745defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 1746defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 1747defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 1748defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 1749defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 1750defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 1751defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 1752defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 1753defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 1754defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 1755 1756let SubtargetPredicate = HasAtomicFaddInsts in { 1757 1758defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d>; 1759defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e>; 1760 1761} // End SubtargetPredicate = HasAtomicFaddInsts 1762