1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>; 10def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>; 11def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>; 12 13def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 14def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; 15def ScratchSVAddr : ComplexPattern<iPTR, 3, "SelectScratchSVAddr", [], [SDNPWantRoot], -10>; 16 17//===----------------------------------------------------------------------===// 18// FLAT classes 19//===----------------------------------------------------------------------===// 20 21class FLAT_Pseudo<string opName, dag outs, dag ins, 22 string asmOps, list<dag> pattern=[]> : 23 InstSI<outs, ins, "", pattern>, 24 SIMCInstr<opName, SIEncodingFamily.NONE> { 25 26 let isPseudo = 1; 27 let isCodeGenOnly = 1; 28 29 let FLAT = 1; 30 31 let UseNamedOperandTable = 1; 32 let hasSideEffects = 0; 33 let SchedRW = [WriteVMEM]; 34 35 string Mnemonic = opName; 36 string AsmOperands = asmOps; 37 38 bits<1> is_flat_global = 0; 39 bits<1> is_flat_scratch = 0; 40 41 bits<1> has_vdst = 1; 42 43 // We need to distinguish having saddr and enabling saddr because 44 // saddr is only valid for scratch and global instructions. Pre-gfx9 45 // these bits were reserved, so we also don't necessarily want to 46 // set these bits to the disabled value for the original flat 47 // segment instructions. 48 bits<1> has_saddr = 0; 49 bits<1> enabled_saddr = 0; 50 bits<7> saddr_value = 0; 51 bits<1> has_vaddr = 1; 52 53 bits<1> has_data = 1; 54 bits<1> has_glc = 1; 55 bits<1> glcValue = 0; 56 bits<1> has_dlc = 1; 57 bits<1> dlcValue = 0; 58 bits<1> has_sccb = 1; 59 bits<1> sccbValue = 0; 60 bits<1> has_sve = 0; // Scratch VGPR Enable 61 bits<1> sve = 0; 62 63 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 64 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 65 66 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 67 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 68 69 // Internally, FLAT instruction are executed as both an LDS and a 70 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 71 // and are not considered done until both have been decremented. 72 let VM_CNT = 1; 73 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); 74 75 let FlatGlobal = is_flat_global; 76 77 let FlatScratch = is_flat_scratch; 78} 79 80class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 81 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 82 Enc64 { 83 84 let isPseudo = 0; 85 let isCodeGenOnly = 0; 86 87 let FLAT = 1; 88 89 // copy relevant pseudo op flags 90 let SubtargetPredicate = ps.SubtargetPredicate; 91 let AsmMatchConverter = ps.AsmMatchConverter; 92 let OtherPredicates = ps.OtherPredicates; 93 let TSFlags = ps.TSFlags; 94 let UseNamedOperandTable = ps.UseNamedOperandTable; 95 let SchedRW = ps.SchedRW; 96 let mayLoad = ps.mayLoad; 97 let mayStore = ps.mayStore; 98 let IsAtomicRet = ps.IsAtomicRet; 99 let IsAtomicNoRet = ps.IsAtomicNoRet; 100 let VM_CNT = ps.VM_CNT; 101 let LGKM_CNT = ps.LGKM_CNT; 102 103 // encoding fields 104 bits<8> vaddr; 105 bits<10> vdata; 106 bits<7> saddr; 107 bits<10> vdst; 108 109 bits<5> cpol; 110 111 // Only valid on gfx9 112 bits<1> lds = 0; // XXX - What does this actually do? 113 114 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 115 bits<2> seg = !if(ps.is_flat_global, 0b10, 116 !if(ps.is_flat_scratch, 0b01, 0)); 117 118 // Signed offset. Highest bit ignored for flat and treated as 12-bit 119 // unsigned for flat accesses. 120 bits<13> offset; 121 // GFX90A+ only: instruction uses AccVGPR for data 122 bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); 123 124 // We don't use tfe right now, and it was removed in gfx9. 125 bits<1> tfe = 0; 126 127 // Only valid on GFX9+ 128 let Inst{12-0} = offset; 129 let Inst{13} = !if(ps.has_sve, ps.sve, lds); 130 let Inst{15-14} = seg; 131 132 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); 133 let Inst{17} = cpol{CPolBit.SLC}; 134 let Inst{24-18} = op; 135 let Inst{31-26} = 0x37; // Encoding. 136 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 137 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); 138 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 139 140 // 54-48 is reserved. 141 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. 142 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); 143} 144 145class GlobalSaddrTable <bit is_saddr, string Name = ""> { 146 bit IsSaddr = is_saddr; 147 string SaddrOp = Name; 148} 149 150// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 151// same encoding value as exec_hi, so it isn't possible to use that if 152// saddr is 32-bit (which isn't handled here yet). 153class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 154 bit HasTiedOutput = 0, 155 bit HasSaddr = 0, bit EnableSaddr = 0, 156 RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< 157 opName, 158 (outs vdata_op:$vdst), 159 !con( 160 !con( 161 !if(EnableSaddr, 162 (ins SReg_64:$saddr, VGPR_32:$vaddr), 163 (ins VReg_64:$vaddr)), 164 (ins flat_offset:$offset)), 165 // FIXME: Operands with default values do not work with following non-optional operands. 166 !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in), 167 (ins CPol_0:$cpol))), 168 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 169 let has_data = 0; 170 let mayLoad = 1; 171 let has_saddr = HasSaddr; 172 let enabled_saddr = EnableSaddr; 173 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 174 let maybeAtomic = 1; 175 176 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 177 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 178} 179 180class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 181 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 182 opName, 183 (outs), 184 !con( 185 !if(EnableSaddr, 186 (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64:$saddr), 187 (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), 188 (ins flat_offset:$offset, CPol_0:$cpol)), 189 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 190 let mayLoad = 0; 191 let mayStore = 1; 192 let has_vdst = 0; 193 let has_saddr = HasSaddr; 194 let enabled_saddr = EnableSaddr; 195 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 196 let maybeAtomic = 1; 197} 198 199multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 200 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 201 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 202 GlobalSaddrTable<0, opName>; 203 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 204 GlobalSaddrTable<1, opName>; 205 } 206} 207 208class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 209 bit HasTiedOutput = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 210 opName, 211 (outs regClass:$vdst), 212 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), 213 (ins flat_offset:$offset, CPol_0:$cpol), 214 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 215 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 216 let is_flat_global = 1; 217 let has_data = 0; 218 let mayLoad = 1; 219 let has_vaddr = 0; 220 let has_saddr = 1; 221 let enabled_saddr = EnableSaddr; 222 let maybeAtomic = 1; 223 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 224 225 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 226 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 227} 228 229multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, 230 bit HasTiedOutput = 0> { 231 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput>, 232 GlobalSaddrTable<0, opName>; 233 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, 1>, 234 GlobalSaddrTable<1, opName>; 235} 236 237multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 238 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 239 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 240 GlobalSaddrTable<0, opName>; 241 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 242 GlobalSaddrTable<1, opName>; 243 } 244} 245 246class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo< 247 opName, 248 (outs ), 249 !con( 250 !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64:$vaddr)), 251 (ins flat_offset:$offset, CPol_0:$cpol)), 252 " $vaddr"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> { 253 let LGKM_CNT = 1; 254 let is_flat_global = 1; 255 let has_data = 0; 256 let has_vdst = 0; 257 let mayLoad = 1; 258 let mayStore = 1; 259 let has_saddr = 1; 260 let enabled_saddr = EnableSaddr; 261 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 262 let Uses = [M0, EXEC]; 263 let SchedRW = [WriteVMEM, WriteLDS]; 264} 265 266multiclass FLAT_Global_Load_LDS_Pseudo<string opName> { 267 def "" : FLAT_Global_Load_LDS_Pseudo<opName>, 268 GlobalSaddrTable<0, opName>; 269 def _SADDR : FLAT_Global_Load_LDS_Pseudo<opName, 1>, 270 GlobalSaddrTable<1, opName>; 271} 272 273class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 274 bit EnableSaddr = 0> : FLAT_Pseudo< 275 opName, 276 (outs), 277 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), 278 (ins flat_offset:$offset, CPol:$cpol)), 279 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 280 let is_flat_global = 1; 281 let mayLoad = 0; 282 let mayStore = 1; 283 let has_vdst = 0; 284 let has_vaddr = 0; 285 let has_saddr = 1; 286 let enabled_saddr = EnableSaddr; 287 let maybeAtomic = 1; 288 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 289} 290 291multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass> { 292 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass>, 293 GlobalSaddrTable<0, opName>; 294 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, 1>, 295 GlobalSaddrTable<1, opName>; 296} 297 298class FlatScratchInst <string sv_op, string mode> { 299 string SVOp = sv_op; 300 string Mode = mode; 301} 302 303class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 304 bit HasTiedOutput = 0, 305 bit EnableSaddr = 0, 306 bit EnableSVE = 0, 307 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> 308 : FLAT_Pseudo< 309 opName, 310 (outs getLdStRegisterOperand<regClass>.ret:$vdst), 311 !con( 312 !if(EnableSVE, 313 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 314 !if(EnableSaddr, 315 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 316 !if(EnableVaddr, 317 (ins VGPR_32:$vaddr, flat_offset:$offset), 318 (ins flat_offset:$offset)))), 319 !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in), 320 (ins CPol_0:$cpol))), 321 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 322 let has_data = 0; 323 let mayLoad = 1; 324 let has_saddr = 1; 325 let enabled_saddr = EnableSaddr; 326 let has_vaddr = EnableVaddr; 327 let has_sve = EnableSVE; 328 let sve = EnableVaddr; 329 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 330 let maybeAtomic = 1; 331 332 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 333 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 334} 335 336class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 337 bit EnableSVE = 0, 338 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr)), 339 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< 340 opName, 341 (outs), 342 !if(EnableSVE, 343 (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 344 !if(EnableSaddr, 345 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 346 !if(EnableVaddr, 347 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol), 348 (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))), 349 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 350 let mayLoad = 0; 351 let mayStore = 1; 352 let has_vdst = 0; 353 let has_saddr = 1; 354 let enabled_saddr = EnableSaddr; 355 let has_vaddr = EnableVaddr; 356 let has_sve = EnableSVE; 357 let sve = EnableVaddr; 358 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 359 let maybeAtomic = 1; 360} 361 362multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 363 let is_flat_scratch = 1 in { 364 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, 365 FlatScratchInst<opName, "SV">; 366 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, 367 FlatScratchInst<opName, "SS">; 368 369 let SubtargetPredicate = HasFlatScratchSVSMode in 370 def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>, 371 FlatScratchInst<opName, "SVS">; 372 373 let SubtargetPredicate = HasFlatScratchSTMode in 374 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>, 375 FlatScratchInst<opName, "ST">; 376 } 377} 378 379multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 380 let is_flat_scratch = 1 in { 381 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, 382 FlatScratchInst<opName, "SV">; 383 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, 384 FlatScratchInst<opName, "SS">; 385 386 let SubtargetPredicate = HasFlatScratchSVSMode in 387 def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>, 388 FlatScratchInst<opName, "SVS">; 389 390 let SubtargetPredicate = HasFlatScratchSTMode in 391 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>, 392 FlatScratchInst<opName, "ST">; 393 } 394} 395 396class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0, 397 bit EnableSVE = 0, 398 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo< 399 opName, 400 (outs ), 401 !if(EnableSVE, 402 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 403 !if(EnableSaddr, 404 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 405 !if(EnableVaddr, 406 (ins VGPR_32:$vaddr, flat_offset:$offset, CPol:$cpol), 407 (ins flat_offset:$offset, CPol:$cpol)))), 408 " "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 409 410 let LGKM_CNT = 1; 411 let is_flat_scratch = 1; 412 let has_data = 0; 413 let has_vdst = 0; 414 let mayLoad = 1; 415 let mayStore = 1; 416 let has_saddr = 1; 417 let enabled_saddr = EnableSaddr; 418 let has_vaddr = EnableVaddr; 419 let has_sve = EnableSVE; 420 let sve = EnableVaddr; 421 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 422 let Uses = [M0, EXEC]; 423 let SchedRW = [WriteVMEM, WriteLDS]; 424} 425 426multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> { 427 def "" : FLAT_Scratch_Load_LDS_Pseudo<opName>, 428 FlatScratchInst<opName, "SV">; 429 def _SADDR : FLAT_Scratch_Load_LDS_Pseudo<opName, 1>, 430 FlatScratchInst<opName, "SS">; 431 def _SVS : FLAT_Scratch_Load_LDS_Pseudo<opName, 1, 1>, 432 FlatScratchInst<opName, "SVS">; 433 def _ST : FLAT_Scratch_Load_LDS_Pseudo<opName, 0, 0, 0>, 434 FlatScratchInst<opName, "ST">; 435} 436 437class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 438 string asm, list<dag> pattern = []> : 439 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 440 let mayLoad = 1; 441 let mayStore = 1; 442 let has_glc = 0; 443 let glcValue = 0; 444 let has_vdst = 0; 445 let has_sccb = 1; 446 let sccbValue = 0; 447 let maybeAtomic = 1; 448 let IsAtomicNoRet = 1; 449} 450 451class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 452 string asm, list<dag> pattern = []> 453 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 454 let hasPostISelHook = 1; 455 let has_vdst = 1; 456 let glcValue = 1; 457 let sccbValue = 0; 458 let IsAtomicNoRet = 0; 459 let IsAtomicRet = 1; 460 let PseudoInstr = NAME # "_RTN"; 461} 462 463multiclass FLAT_Atomic_Pseudo< 464 string opName, 465 RegisterClass vdst_rc, 466 ValueType vt, 467 SDPatternOperator atomic = null_frag, 468 ValueType data_vt = vt, 469 RegisterClass data_rc = vdst_rc, 470 bit isFP = isFloatType<data_vt>.ret, 471 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 472 def "" : FLAT_AtomicNoRet_Pseudo <opName, 473 (outs), 474 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 475 " $vaddr, $vdata$offset$cpol">, 476 GlobalSaddrTable<0, opName>, 477 AtomicNoRet <opName, 0> { 478 let PseudoInstr = NAME; 479 let FPAtomic = isFP; 480 let AddedComplexity = -1; // Prefer global atomics if available 481 } 482 483 def _RTN : FLAT_AtomicRet_Pseudo <opName, 484 (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), 485 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 486 " $vdst, $vaddr, $vdata$offset$cpol", 487 [(set vt:$vdst, 488 (atomic (FlatOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 489 GlobalSaddrTable<0, opName#"_rtn">, 490 AtomicNoRet <opName, 1>{ 491 let FPAtomic = isFP; 492 let AddedComplexity = -1; // Prefer global atomics if available 493 } 494} 495 496multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 497 string opName, 498 RegisterClass vdst_rc, 499 ValueType vt, 500 ValueType data_vt = vt, 501 RegisterClass data_rc = vdst_rc, 502 bit isFP = isFloatType<data_vt>.ret, 503 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 504 505 def "" : FLAT_AtomicNoRet_Pseudo <opName, 506 (outs), 507 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 508 " $vaddr, $vdata, off$offset$cpol">, 509 GlobalSaddrTable<0, opName>, 510 AtomicNoRet <opName, 0> { 511 let has_saddr = 1; 512 let PseudoInstr = NAME; 513 let FPAtomic = isFP; 514 } 515 516 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 517 (outs), 518 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_0:$cpol), 519 " $vaddr, $vdata, $saddr$offset$cpol">, 520 GlobalSaddrTable<1, opName>, 521 AtomicNoRet <opName#"_saddr", 0> { 522 let has_saddr = 1; 523 let enabled_saddr = 1; 524 let PseudoInstr = NAME#"_SADDR"; 525 let FPAtomic = isFP; 526 } 527} 528 529multiclass FLAT_Global_Atomic_Pseudo_RTN< 530 string opName, 531 RegisterClass vdst_rc, 532 ValueType vt, 533 SDPatternOperator atomic = null_frag, 534 ValueType data_vt = vt, 535 RegisterClass data_rc = vdst_rc, 536 bit isFP = isFloatType<data_vt>.ret, 537 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, 538 RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { 539 540 def _RTN : FLAT_AtomicRet_Pseudo <opName, 541 (outs vdst_op:$vdst), 542 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 543 " $vdst, $vaddr, $vdata, off$offset$cpol", 544 [(set vt:$vdst, 545 (atomic (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$vdata))]>, 546 GlobalSaddrTable<0, opName#"_rtn">, 547 AtomicNoRet <opName, 1> { 548 let has_saddr = 1; 549 let FPAtomic = isFP; 550 } 551 552 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 553 (outs vdst_op:$vdst), 554 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_GLC1:$cpol), 555 " $vdst, $vaddr, $vdata, $saddr$offset$cpol">, 556 GlobalSaddrTable<1, opName#"_rtn">, 557 AtomicNoRet <opName#"_saddr", 1> { 558 let has_saddr = 1; 559 let enabled_saddr = 1; 560 let PseudoInstr = NAME#"_SADDR_RTN"; 561 let FPAtomic = isFP; 562 } 563} 564 565multiclass FLAT_Global_Atomic_Pseudo< 566 string opName, 567 RegisterClass vdst_rc, 568 ValueType vt, 569 SDPatternOperator atomic_rtn = null_frag, 570 ValueType data_vt = vt, 571 RegisterClass data_rc = vdst_rc> { 572 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 573 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc>; 574 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; 575 } 576} 577 578//===----------------------------------------------------------------------===// 579// Flat Instructions 580//===----------------------------------------------------------------------===// 581 582def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 583def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 584def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 585def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 586def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 587def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 588def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 589def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 590 591def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 592def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 593def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 594def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 595def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 596def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 597 598let SubtargetPredicate = HasD16LoadStore in { 599def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 600def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 601def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 602def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 603def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 604def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 605 606def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 607def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 608} 609 610defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 611 VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, 612 v2i32, VReg_64>; 613 614defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 615 VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, 616 v2i64, VReg_128>; 617 618defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 619 VGPR_32, i32, atomic_swap_flat_32>; 620 621defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 622 VReg_64, i64, atomic_swap_flat_64>; 623 624defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 625 VGPR_32, i32, atomic_load_add_flat_32>; 626 627defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 628 VGPR_32, i32, atomic_load_sub_flat_32>; 629 630defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 631 VGPR_32, i32, atomic_load_min_flat_32>; 632 633defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 634 VGPR_32, i32, atomic_load_umin_flat_32>; 635 636defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 637 VGPR_32, i32, atomic_load_max_flat_32>; 638 639defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 640 VGPR_32, i32, atomic_load_umax_flat_32>; 641 642defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 643 VGPR_32, i32, atomic_load_and_flat_32>; 644 645defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 646 VGPR_32, i32, atomic_load_or_flat_32>; 647 648defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 649 VGPR_32, i32, atomic_load_xor_flat_32>; 650 651defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 652 VGPR_32, i32, atomic_inc_flat_32>; 653 654defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 655 VGPR_32, i32, atomic_dec_flat_32>; 656 657defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 658 VReg_64, i64, atomic_load_add_flat_64>; 659 660defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 661 VReg_64, i64, atomic_load_sub_flat_64>; 662 663defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 664 VReg_64, i64, atomic_load_min_flat_64>; 665 666defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 667 VReg_64, i64, atomic_load_umin_flat_64>; 668 669defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 670 VReg_64, i64, atomic_load_max_flat_64>; 671 672defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 673 VReg_64, i64, atomic_load_umax_flat_64>; 674 675defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 676 VReg_64, i64, atomic_load_and_flat_64>; 677 678defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 679 VReg_64, i64, atomic_load_or_flat_64>; 680 681defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 682 VReg_64, i64, atomic_load_xor_flat_64>; 683 684defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 685 VReg_64, i64, atomic_inc_flat_64>; 686 687defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 688 VReg_64, i64, atomic_dec_flat_64>; 689 690// GFX7-, GFX10-only flat instructions. 691let SubtargetPredicate = isGFX7GFX10 in { 692 693defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 694 VGPR_32, f32, null_frag, v2f32, VReg_64>; 695 696defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 697 VReg_64, f64, null_frag, v2f64, VReg_128>; 698 699defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 700 VGPR_32, f32>; 701 702defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 703 VGPR_32, f32>; 704 705defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 706 VReg_64, f64>; 707 708defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 709 VReg_64, f64>; 710 711} // End SubtargetPredicate = isGFX7GFX10 712 713let SubtargetPredicate = isGFX90APlus in { 714 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64, int_amdgcn_flat_atomic_fadd>; 715 defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo<"flat_atomic_min_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmin>; 716 defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo<"flat_atomic_max_f64", VReg_64, f64, int_amdgcn_flat_atomic_fmax>; 717 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 718 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 719 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 720} // End SubtargetPredicate = isGFX90APlus 721 722let SubtargetPredicate = isGFX940Plus in { 723 defm FLAT_ATOMIC_ADD_F32 : FLAT_Atomic_Pseudo<"flat_atomic_add_f32", VGPR_32, f32, int_amdgcn_flat_atomic_fadd>; 724 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_f16", VGPR_32, v2f16, int_amdgcn_flat_atomic_fadd>; 725 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_bf16", VGPR_32, v2f16>; 726 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Atomic_Pseudo<"global_atomic_pk_add_bf16", VGPR_32, v2f16>; 727} // End SubtargetPredicate = isGFX940Plus 728 729defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 730defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 731defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 732defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 733defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 734defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 735defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 736defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 737 738defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 739defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 740defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 741defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 742defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 743defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 744let OtherPredicates = [HasGFX10_BEncoding] in 745defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 746 747defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 748defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 749defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 750defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 751defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 752defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 753let OtherPredicates = [HasGFX10_BEncoding] in 754defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 755 756defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 757defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 758 759let is_flat_global = 1 in { 760defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 761 VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, 762 v2i32, VReg_64>; 763 764defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 765 VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, 766 v2i64, VReg_128>; 767 768defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 769 VGPR_32, i32, atomic_swap_global_32>; 770 771defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 772 VReg_64, i64, atomic_swap_global_64>; 773 774defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 775 VGPR_32, i32, atomic_load_add_global_32>; 776 777defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 778 VGPR_32, i32, atomic_load_sub_global_32>; 779 780defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 781 VGPR_32, i32, atomic_load_min_global_32>; 782 783defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 784 VGPR_32, i32, atomic_load_umin_global_32>; 785 786defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 787 VGPR_32, i32, atomic_load_max_global_32>; 788 789defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 790 VGPR_32, i32, atomic_load_umax_global_32>; 791 792defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 793 VGPR_32, i32, atomic_load_and_global_32>; 794 795defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 796 VGPR_32, i32, atomic_load_or_global_32>; 797 798defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 799 VGPR_32, i32, atomic_load_xor_global_32>; 800 801defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 802 VGPR_32, i32, atomic_inc_global_32>; 803 804defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 805 VGPR_32, i32, atomic_dec_global_32>; 806 807defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 808 VReg_64, i64, atomic_load_add_global_64>; 809 810defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 811 VReg_64, i64, atomic_load_sub_global_64>; 812 813defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 814 VReg_64, i64, atomic_load_min_global_64>; 815 816defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 817 VReg_64, i64, atomic_load_umin_global_64>; 818 819defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 820 VReg_64, i64, atomic_load_max_global_64>; 821 822defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 823 VReg_64, i64, atomic_load_umax_global_64>; 824 825defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 826 VReg_64, i64, atomic_load_and_global_64>; 827 828defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 829 VReg_64, i64, atomic_load_or_global_64>; 830 831defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 832 VReg_64, i64, atomic_load_xor_global_64>; 833 834defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 835 VReg_64, i64, atomic_inc_global_64>; 836 837defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 838 VReg_64, i64, atomic_dec_global_64>; 839 840let SubtargetPredicate = HasGFX10_BEncoding in 841defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 842 VGPR_32, i32, int_amdgcn_global_atomic_csub>; 843 844let SubtargetPredicate = isGFX940Plus in { 845 846defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">; 847defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">; 848defm GLOBAL_LOAD_LDS_USHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ushort">; 849defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sshort">; 850defm GLOBAL_LOAD_LDS_DWORD : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dword">; 851 852} // End let SubtargetPredicate = isGFX940Plus 853} // End is_flat_global = 1 854 855 856 857let SubtargetPredicate = HasFlatScratchInsts in { 858defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 859defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 860defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 861defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 862defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 863defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 864defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 865defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 866 867defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 868defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 869defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 870defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 871defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 872defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 873 874defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 875defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 876defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 877defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 878defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 879defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 880 881defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 882defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 883 884let SubtargetPredicate = isGFX940Plus in { 885 886defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ubyte">; 887defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sbyte">; 888defm SCRATCH_LOAD_LDS_USHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ushort">; 889defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sshort">; 890defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_dword">; 891 892} // End let SubtargetPredicate = isGFX940Plus 893 894} // End SubtargetPredicate = HasFlatScratchInsts 895 896let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 897 defm GLOBAL_ATOMIC_FCMPSWAP : 898 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32, null_frag, v2f32, VReg_64>; 899 defm GLOBAL_ATOMIC_FMIN : 900 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32, int_amdgcn_global_atomic_fmin>; 901 defm GLOBAL_ATOMIC_FMAX : 902 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32, int_amdgcn_global_atomic_fmax>; 903 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 904 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64, null_frag, v2f64, VReg_128>; 905 defm GLOBAL_ATOMIC_FMIN_X2 : 906 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 907 defm GLOBAL_ATOMIC_FMAX_X2 : 908 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 909} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 910 911let is_flat_global = 1 in { 912let OtherPredicates = [HasAtomicFaddInsts] in { 913 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 914 "global_atomic_add_f32", VGPR_32, f32 915 >; 916 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 917 "global_atomic_pk_add_f16", VGPR_32, v2f16 918 >; 919} // End OtherPredicates = [HasAtomicFaddInsts] 920 921let OtherPredicates = [isGFX90APlus] in { 922 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < 923 "global_atomic_add_f32", VGPR_32, f32, int_amdgcn_global_atomic_fadd 924 >; 925 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < 926 "global_atomic_pk_add_f16", VGPR_32, v2f16, int_amdgcn_global_atomic_fadd 927 >; 928} // End OtherPredicates = [isGFX90APlus] 929} // End is_flat_global = 1 930 931//===----------------------------------------------------------------------===// 932// Flat Patterns 933//===----------------------------------------------------------------------===// 934 935// Patterns for global loads with no offset. 936class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 937 (vt (node (FlatOffset i64:$vaddr, i16:$offset))), 938 (inst $vaddr, $offset) 939>; 940 941class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 942 (node (FlatOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 943 (inst $vaddr, $offset, 0, $in) 944>; 945 946class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 947 (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 948 (inst $vaddr, $offset, 0, $in) 949>; 950 951class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 952 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 953 (inst $saddr, $voffset, $offset, 0, $in) 954>; 955 956class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 957 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset))), 958 (inst $vaddr, $offset) 959>; 960 961class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 962 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 963 (inst $saddr, $voffset, $offset, 0) 964>; 965 966class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 967 ValueType vt> : GCNPat < 968 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 969 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 970>; 971 972class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 973 ValueType vt> : GCNPat < 974 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 975 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 976>; 977 978class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 979 ValueType vt, ValueType data_vt = vt> : GCNPat < 980 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 981 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 982>; 983 984class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 985 ValueType vt> : GCNPat < 986 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 987 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 988>; 989 990class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 991 (node vt:$data, (FlatOffset i64:$vaddr, i16:$offset)), 992 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 993>; 994 995class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 996 (node vt:$data, (GlobalOffset i64:$vaddr, i16:$offset)), 997 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 998>; 999 1000class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1001 // atomic store follows atomic binop convention so the address comes 1002 // first. 1003 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 1004 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1005>; 1006 1007class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 1008 ValueType vt, ValueType data_vt = vt> : GCNPat < 1009 // atomic store follows atomic binop convention so the address comes 1010 // first. 1011 (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data), 1012 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1013>; 1014 1015class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1016 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 1017 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1018>; 1019 1020multiclass FlatAtomicPat <string inst, string node, ValueType vt, 1021 ValueType data_vt = vt> { 1022 defvar rtnNode = !cast<PatFrags>(node#"_ret_"#vt.Size); 1023 defvar noRtnNode = !cast<PatFrags>(node#"_noret_"#vt.Size); 1024 1025 def : GCNPat <(vt (rtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1026 (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1027 1028 def : GCNPat <(vt (noRtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1029 (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1030} 1031 1032class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1033 (node (GlobalOffset i64:$vaddr, i16:$offset), vt:$data), 1034 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1035>; 1036 1037class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 1038 ValueType data_vt = vt> : GCNPat < 1039 (vt (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1040 (inst VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1041>; 1042 1043class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1044 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset))), 1045 (inst $vaddr, $offset) 1046>; 1047 1048class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1049 (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset), vt:$in), 1050 (inst $vaddr, $offset, 0, $in) 1051>; 1052 1053class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1054 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset)), 1055 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) 1056>; 1057 1058class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1059 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset))), 1060 (inst $saddr, $offset) 1061>; 1062 1063class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1064 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1065 (inst $saddr, $offset, 0, $in) 1066>; 1067 1068class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1069 ValueType vt> : GCNPat < 1070 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)), 1071 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 1072>; 1073 1074class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1075 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset))), 1076 (inst $vaddr, $saddr, $offset, 0) 1077>; 1078 1079class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1080 ValueType vt> : GCNPat < 1081 (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset)), 1082 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset) 1083>; 1084 1085class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1086 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1087 (inst $vaddr, $saddr, $offset, 0, $in) 1088>; 1089 1090let OtherPredicates = [HasFlatAddressSpace] in { 1091 1092def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 1093def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 1094def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 1095def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 1096def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 1097def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 1098def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 1099def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 1100def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 1101def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 1102def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 1103 1104def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 1105def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 1106 1107def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 1108def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 1109 1110foreach vt = Reg32Types.types in { 1111def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 1112def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 1113} 1114 1115foreach vt = VReg_64.RegTypes in { 1116def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 1117def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 1118} 1119 1120def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 1121 1122foreach vt = VReg_128.RegTypes in { 1123def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 1124def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 1125} 1126 1127def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; 1128def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64>; 1129 1130foreach as = [ "flat", "global" ] in { 1131defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; 1132defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>; 1133defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_inc_"#as, i32>; 1134defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_dec_"#as, i32>; 1135defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>; 1136defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>; 1137defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>; 1138defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN", "atomic_load_min_"#as, i32>; 1139defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN", "atomic_load_umin_"#as, i32>; 1140defm : FlatAtomicPat <"FLAT_ATOMIC_OR", "atomic_load_or_"#as, i32>; 1141defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP", "atomic_swap_"#as, i32>; 1142defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_"#as, i32, v2i32>; 1143defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>; 1144 1145defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>; 1146defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>; 1147defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_inc_"#as, i64>; 1148defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_dec_"#as, i64>; 1149defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>; 1150defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>; 1151defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>; 1152defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN_X2", "atomic_load_min_"#as, i64>; 1153defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN_X2", "atomic_load_umin_"#as, i64>; 1154defm : FlatAtomicPat <"FLAT_ATOMIC_OR_X2", "atomic_load_or_"#as, i64>; 1155defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP_X2", "atomic_swap_"#as, i64>; 1156defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_"#as, i64, v2i64>; 1157defm : FlatAtomicPat <"FLAT_ATOMIC_XOR_X2", "atomic_load_xor_"#as, i64>; 1158} // end foreach as 1159 1160def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 1161def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 1162 1163let OtherPredicates = [D16PreservesUnusedBits] in { 1164def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 1165def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 1166 1167def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 1168def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 1169def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 1170def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 1171def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 1172def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 1173 1174def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 1175def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 1176def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 1177def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 1178def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 1179def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 1180} 1181 1182} // End OtherPredicates = [HasFlatAddressSpace] 1183 1184 1185multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1186 def : FlatLoadSignedPat <inst, node, vt> { 1187 let AddedComplexity = 10; 1188 } 1189 1190 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1191 let AddedComplexity = 11; 1192 } 1193} 1194 1195multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1196 def : FlatSignedLoadPat_D16 <inst, node, vt> { 1197 let AddedComplexity = 10; 1198 } 1199 1200 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1201 let AddedComplexity = 11; 1202 } 1203} 1204 1205multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1206 ValueType vt> { 1207 def : FlatStoreSignedPat <inst, node, vt> { 1208 let AddedComplexity = 10; 1209 } 1210 1211 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1212 let AddedComplexity = 11; 1213 } 1214} 1215 1216// Deal with swapped operands for atomic_store vs. regular store 1217multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1218 def : FlatStoreSignedAtomicPat <inst, node, vt> { 1219 let AddedComplexity = 10; 1220 } 1221 1222 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1223 let AddedComplexity = 11; 1224 } 1225} 1226 1227multiclass GlobalFLATAtomicPatsRtn<string nortn_inst_name, SDPatternOperator node, 1228 ValueType vt, ValueType data_vt = vt> { 1229 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 1230 let AddedComplexity = 10; 1231 } 1232 1233 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 1234 let AddedComplexity = 11; 1235 } 1236} 1237 1238multiclass GlobalFLATAtomicPats<string inst, string node, ValueType vt, 1239 ValueType data_vt = vt> { 1240 defvar rtnNode = !cast<PatFrags>(node#"_ret_"#vt.Size); 1241 defvar noRtnNode = !cast<PatFrags>(node#"_noret_"#vt.Size); 1242 1243 let AddedComplexity = 10 in { 1244 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(inst), noRtnNode, vt, data_vt>; 1245 def : FlatSignedAtomicPat <!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>; 1246 } 1247 1248 let AddedComplexity = 11 in { 1249 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR"), noRtnNode, vt, data_vt>; 1250 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_RTN"), rtnNode, vt, data_vt>; 1251 } 1252} 1253 1254multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1255 ValueType vt> { 1256 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1257 let AddedComplexity = 10; 1258 } 1259 1260 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1261 let AddedComplexity = 11; 1262 } 1263} 1264 1265multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1266 def : ScratchLoadSignedPat <inst, node, vt> { 1267 let AddedComplexity = 25; 1268 } 1269 1270 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1271 let AddedComplexity = 26; 1272 } 1273 1274 def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1275 let SubtargetPredicate = HasFlatScratchSVSMode; 1276 let AddedComplexity = 27; 1277 } 1278} 1279 1280multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1281 ValueType vt> { 1282 def : ScratchStoreSignedPat <inst, node, vt> { 1283 let AddedComplexity = 25; 1284 } 1285 1286 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1287 let AddedComplexity = 26; 1288 } 1289 1290 def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1291 let SubtargetPredicate = HasFlatScratchSVSMode; 1292 let AddedComplexity = 27; 1293 } 1294} 1295 1296multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1297 def : ScratchLoadSignedPat_D16 <inst, node, vt> { 1298 let AddedComplexity = 25; 1299 } 1300 1301 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1302 let AddedComplexity = 26; 1303 } 1304 1305 def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1306 let SubtargetPredicate = HasFlatScratchSVSMode; 1307 let AddedComplexity = 27; 1308 } 1309} 1310 1311let OtherPredicates = [HasFlatGlobalInsts] in { 1312 1313defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1314defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1315defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1316defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1317defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1318defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1319defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1320defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1321defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1322defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1323 1324foreach vt = Reg32Types.types in { 1325defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1326defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1327} 1328 1329foreach vt = VReg_64.RegTypes in { 1330defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1331defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1332} 1333 1334defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1335 1336foreach vt = VReg_128.RegTypes in { 1337defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1338defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1339} 1340 1341// There is no distinction for atomic load lowering during selection; 1342// the memory legalizer will set the cache bits and insert the 1343// appropriate waits. 1344defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1345defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1346 1347defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1348defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1349defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1350defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1351defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1352 1353let OtherPredicates = [D16PreservesUnusedBits] in { 1354defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1355defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1356 1357defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1358defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1359defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1360defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1361defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1362defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1363 1364defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1365defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1366defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1367defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1368defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1369defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1370} 1371 1372defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; 1373defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64>; 1374 1375defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; 1376defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; 1377defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_inc_global", i32>; 1378defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_dec_global", i32>; 1379defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>; 1380defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>; 1381defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>; 1382defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", "atomic_load_min_global", i32>; 1383defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", "atomic_load_umin_global", i32>; 1384defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", "atomic_load_or_global", i32>; 1385defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", "atomic_swap_global", i32>; 1386defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_global", i32, v2i32>; 1387defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>; 1388defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1389 1390defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>; 1391defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>; 1392defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_inc_global", i64>; 1393defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_dec_global", i64>; 1394defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>; 1395defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>; 1396defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>; 1397defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", "atomic_load_min_global", i64>; 1398defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", "atomic_load_umin_global", i64>; 1399defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", "atomic_load_or_global", i64>; 1400defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", "atomic_swap_global", i64>; 1401defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_global", i64, v2i64>; 1402defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", "atomic_load_xor_global", i64>; 1403 1404let OtherPredicates = [isGFX10Plus] in { 1405defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>; 1406defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>; 1407defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN_X2", "atomic_load_fmin_global", f64>; 1408defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX_X2", "atomic_load_fmax_global", f64>; 1409} 1410 1411let OtherPredicates = [HasAtomicFaddInsts] in { 1412defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1413defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1414} 1415 1416let OtherPredicates = [isGFX90APlus] in { 1417defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>; 1418defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_v2f16_global", v2f16>; 1419defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", "atomic_load_fadd_global", f64>; 1420defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global", f64>; 1421defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>; 1422def : FlatSignedAtomicPat <FLAT_ATOMIC_ADD_F64_RTN, atomic_load_fadd_flat_ret_64, f64>; 1423def : FlatSignedAtomicPat <FLAT_ATOMIC_ADD_F64, atomic_load_fadd_flat_noret_64, f64>; 1424def : FlatSignedAtomicPat <FLAT_ATOMIC_MIN_F64_RTN, atomic_load_fmin_flat_ret_64, f64>; 1425def : FlatSignedAtomicPat <FLAT_ATOMIC_MIN_F64, atomic_load_fmin_flat_noret_64, f64>; 1426def : FlatSignedAtomicPat <FLAT_ATOMIC_MAX_F64_RTN, atomic_load_fmax_flat_ret_64, f64>; 1427def : FlatSignedAtomicPat <FLAT_ATOMIC_MAX_F64, atomic_load_fmax_flat_noret_64, f64>; 1428} 1429 1430let OtherPredicates = [isGFX940Plus] in { 1431def : FlatSignedAtomicPat <FLAT_ATOMIC_ADD_F32_RTN, atomic_load_fadd_flat_32, f32>; 1432def : FlatSignedAtomicPat <FLAT_ATOMIC_PK_ADD_F16_RTN, atomic_load_fadd_v2f16_flat_32, v2f16>; 1433def : FlatSignedAtomicPat <FLAT_ATOMIC_PK_ADD_BF16_RTN, int_amdgcn_flat_atomic_fadd_v2bf16, v2i16>; 1434defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_PK_ADD_BF16", int_amdgcn_global_atomic_fadd_v2bf16, v2i16>; 1435} 1436 1437} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1438 1439let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { 1440 1441defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; 1442defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; 1443defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; 1444defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; 1445defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; 1446defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; 1447defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; 1448defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; 1449defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; 1450defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; 1451 1452foreach vt = Reg32Types.types in { 1453defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; 1454defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; 1455} 1456 1457foreach vt = VReg_64.RegTypes in { 1458defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; 1459defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; 1460} 1461 1462defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; 1463 1464foreach vt = VReg_128.RegTypes in { 1465defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; 1466defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; 1467} 1468 1469defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; 1470defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; 1471defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; 1472defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; 1473defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; 1474 1475let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { 1476defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; 1477defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; 1478 1479defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; 1480defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; 1481defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; 1482defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; 1483defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; 1484defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; 1485 1486defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; 1487defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; 1488defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; 1489defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; 1490defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; 1491defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; 1492} 1493 1494} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] 1495 1496//===----------------------------------------------------------------------===// 1497// Target 1498//===----------------------------------------------------------------------===// 1499 1500//===----------------------------------------------------------------------===// 1501// CI 1502//===----------------------------------------------------------------------===// 1503 1504class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1505 FLAT_Real <op, ps>, 1506 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1507 let AssemblerPredicate = isGFX7Only; 1508 let DecoderNamespace="GFX7"; 1509} 1510 1511def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1512def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1513def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1514def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1515def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1516def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1517def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1518def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1519 1520def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1521def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1522def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1523def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1524def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1525def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1526 1527multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1528 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1529 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1530} 1531 1532defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1533defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1534defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1535defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1536defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1537defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1538defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1539defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1540defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1541defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1542defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1543defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1544defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1545defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1546defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1547defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1548defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1549defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1550defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1551defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1552defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1553defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1554defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1555defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1556defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1557defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1558 1559// CI Only flat instructions 1560defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1561defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1562defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1563defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1564defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1565defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1566 1567 1568//===----------------------------------------------------------------------===// 1569// VI 1570//===----------------------------------------------------------------------===// 1571 1572class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> : 1573 FLAT_Real <op, ps>, 1574 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1575 let AssemblerPredicate = isGFX8GFX9; 1576 let DecoderNamespace = "GFX8"; 1577 1578 let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1579 let AsmString = ps.Mnemonic # 1580 !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands); 1581} 1582 1583multiclass FLAT_Real_AllAddr_vi<bits<7> op, 1584 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1585 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>; 1586 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>; 1587} 1588 1589class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> : 1590 FLAT_Real <op, ps>, 1591 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> { 1592 let AssemblerPredicate = isGFX940Plus; 1593 let DecoderNamespace = "GFX9"; 1594 let Inst{13} = ps.sve; 1595 let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1596} 1597 1598multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> { 1599 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> { 1600 let AssemblerPredicate = isGFX8GFX9NotGFX940; 1601 let OtherPredicates = [isGFX8GFX9NotGFX940]; 1602 } 1603 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> { 1604 let DecoderNamespace = "GFX9"; 1605 } 1606 let AssemblerPredicate = isGFX940Plus, SubtargetPredicate = isGFX940Plus in { 1607 def _VE_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1608 def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>; 1609 def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; 1610 } 1611} 1612 1613def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1614def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1615def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1616def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1617def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1618def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1619def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1620def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1621 1622def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1623def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1624def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1625def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1626def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1627def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1628def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1629def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1630 1631def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1632def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1633def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1634def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1635def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1636def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1637 1638multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps, 1639 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1640 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>; 1641 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>; 1642} 1643 1644multiclass FLAT_Global_Real_Atomics_vi<bits<7> op, 1645 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> : 1646 FLAT_Real_AllAddr_vi<op, has_sccb> { 1647 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>; 1648 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>; 1649} 1650 1651 1652defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1653defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1654defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1655defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1656defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1657defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1658defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1659defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1660defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1661defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1662defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1663defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1664defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1665defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1666defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1667defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1668defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1669defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1670defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1671defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1672defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1673defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1674defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1675defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1676defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1677defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1678 1679defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1680defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1681defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1682defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1683defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1684defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1685defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1686defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1687 1688defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1689defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1690defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1691defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1692defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1693defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1694 1695defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1696defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1697defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1698defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1699defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1700defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1701defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1702defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1703 1704let AssemblerPredicate = isGFX940Plus in { 1705defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_vi <0x026>; 1706defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_vi <0x027>; 1707defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_vi <0x028>; 1708defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_vi <0x029>; 1709defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_vi <0x02a>; 1710} // End let AssemblerPredicate = isGFX940Plus 1711 1712defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1713defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1714defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1715defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1716defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1717defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1718defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1719defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1720defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1721defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1722defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1723defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1724defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1725defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1726defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1727defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1728defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1729defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1730defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1731defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1732defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1733defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1734defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1735defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1736defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1737defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1738 1739let AssemblerPredicate = isGFX940Plus in { 1740defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x026>; 1741defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x027>; 1742defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_AllAddr_SVE_vi <0x028>; 1743defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x029>; 1744defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_AllAddr_SVE_vi <0x02a>; 1745} // End let AssemblerPredicate = isGFX940Plus 1746 1747defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x10>; 1748defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x11>; 1749defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_SVE_vi <0x12>; 1750defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x13>; 1751defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_SVE_vi <0x14>; 1752defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x15>; 1753defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x16>; 1754defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x17>; 1755defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_SVE_vi <0x18>; 1756defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x19>; 1757defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x20>; 1758defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x21>; 1759defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x22>; 1760defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x23>; 1761defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_SVE_vi <0x24>; 1762defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x25>; 1763defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_SVE_vi <0x1a>; 1764defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>; 1765defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_SVE_vi <0x1c>; 1766defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x1d>; 1767defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x1e>; 1768defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x1f>; 1769 1770let SubtargetPredicate = isGFX8GFX9NotGFX940 in { 1771 // These instructions are encoded differently on gfx90* and gfx940. 1772 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>; 1773 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>; 1774} 1775 1776let SubtargetPredicate = isGFX90AOnly in { 1777 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64, 0>; 1778 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64, 0>; 1779 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64, 0>; 1780 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; 1781 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; 1782 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; 1783} // End SubtargetPredicate = isGFX90AOnly 1784 1785multiclass FLAT_Real_AllAddr_gfx940<bits<7> op> { 1786 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1787 def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1788} 1789 1790multiclass FLAT_Real_Atomics_gfx940 <bits<7> op, FLAT_Pseudo ps> { 1791 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1792 def _RTN_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1793} 1794 1795multiclass FLAT_Global_Real_Atomics_gfx940<bits<7> op> : 1796 FLAT_Real_AllAddr_gfx940<op> { 1797 def _RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1798 def _SADDR_RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1799} 1800 1801let SubtargetPredicate = isGFX940Plus in { 1802 // These instructions are encoded differently on gfx90* and gfx940. 1803 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>; 1804 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>; 1805 1806 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f, FLAT_ATOMIC_ADD_F64>; 1807 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50, FLAT_ATOMIC_MIN_F64>; 1808 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51, FLAT_ATOMIC_MAX_F64>; 1809 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>; 1810 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>; 1811 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>; 1812 defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d, FLAT_ATOMIC_ADD_F32>; 1813 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e, FLAT_ATOMIC_PK_ADD_F16>; 1814 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52, FLAT_ATOMIC_PK_ADD_BF16>; 1815 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>; 1816} // End SubtargetPredicate = isGFX940Plus 1817 1818//===----------------------------------------------------------------------===// 1819// GFX10. 1820//===----------------------------------------------------------------------===// 1821 1822class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1823 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1824 let AssemblerPredicate = isGFX10Plus; 1825 let DecoderNamespace = "GFX10"; 1826 1827 let Inst{11-0} = offset{11-0}; 1828 let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); 1829 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1830 let Inst{55} = 0; 1831} 1832 1833 1834multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1835 def _gfx10 : 1836 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1837} 1838 1839multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1840 def _RTN_gfx10 : 1841 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1842} 1843 1844multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1845 def _SADDR_gfx10 : 1846 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1847} 1848 1849multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1850 def _SADDR_RTN_gfx10 : 1851 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1852} 1853 1854multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1855 def _ST_gfx10 : 1856 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1857 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1858 let OtherPredicates = [HasFlatScratchSTMode]; 1859 } 1860} 1861 1862multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1863 FLAT_Real_Base_gfx10<op>, 1864 FLAT_Real_SADDR_gfx10<op>; 1865 1866multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1867 FLAT_Real_Base_gfx10<op>, 1868 FLAT_Real_RTN_gfx10<op>; 1869 1870multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1871 FLAT_Real_AllAddr_gfx10<op>, 1872 FLAT_Real_RTN_gfx10<op>, 1873 FLAT_Real_SADDR_RTN_gfx10<op>; 1874 1875multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1876 FLAT_Real_RTN_gfx10<op>, 1877 FLAT_Real_SADDR_RTN_gfx10<op>; 1878 1879multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1880 FLAT_Real_Base_gfx10<op>, 1881 FLAT_Real_SADDR_gfx10<op>, 1882 FLAT_Real_ST_gfx10<op>; 1883 1884// ENC_FLAT. 1885defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1886defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1887defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1888defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1889defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1890defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1891defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1892defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1893defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1894defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1895defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1896defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1897defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1898defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1899defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1900defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1901defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1902defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1903defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1904defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1905defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1906defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1907defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1908defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1909defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1910defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1911defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1912defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1913defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1914defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1915defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1916defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1917defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1918defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1919defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1920defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1921defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1922defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1923defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1924defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1925defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1926defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1927defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1928defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1929defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1930defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1931defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1932defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1933defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1934defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1935defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1936defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1937defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1938defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1939 1940 1941// ENC_FLAT_GLBL. 1942defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1943defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1944defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1945defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1946defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1947defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1948defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1949defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1950defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1951defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1952defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1953defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1954defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 1955defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 1956defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 1957defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 1958defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 1959defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 1960defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 1961defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 1962defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 1963defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 1964defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 1965defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 1966defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 1967defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 1968defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 1969defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 1970defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 1971defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 1972defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 1973defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 1974defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 1975defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 1976defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 1977defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 1978defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 1979defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 1980defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 1981defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 1982defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 1983defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 1984defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 1985defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 1986defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 1987defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 1988defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 1989defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 1990defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 1991defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 1992defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 1993defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 1994defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 1995defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 1996defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 1997defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; 1998defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; 1999 2000// ENC_FLAT_SCRATCH. 2001defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 2002defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 2003defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 2004defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 2005defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 2006defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 2007defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 2008defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 2009defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 2010defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 2011defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 2012defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 2013defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 2014defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 2015defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 2016defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 2017defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 2018defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 2019defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 2020defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 2021defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 2022defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 2023