1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>; 10def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>; 11def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>; 12 13def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 14def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; 15def ScratchSVAddr : ComplexPattern<iPTR, 3, "SelectScratchSVAddr", [], [SDNPWantRoot], -10>; 16 17//===----------------------------------------------------------------------===// 18// FLAT classes 19//===----------------------------------------------------------------------===// 20 21class FLAT_Pseudo<string opName, dag outs, dag ins, 22 string asmOps, list<dag> pattern=[]> : 23 InstSI<outs, ins, "", pattern>, 24 SIMCInstr<opName, SIEncodingFamily.NONE> { 25 26 let isPseudo = 1; 27 let isCodeGenOnly = 1; 28 29 let FLAT = 1; 30 31 let UseNamedOperandTable = 1; 32 let hasSideEffects = 0; 33 let SchedRW = [WriteVMEM]; 34 35 string Mnemonic = opName; 36 string AsmOperands = asmOps; 37 38 bits<1> is_flat_global = 0; 39 bits<1> is_flat_scratch = 0; 40 41 bits<1> has_vdst = 1; 42 43 // We need to distinguish having saddr and enabling saddr because 44 // saddr is only valid for scratch and global instructions. Pre-gfx9 45 // these bits were reserved, so we also don't necessarily want to 46 // set these bits to the disabled value for the original flat 47 // segment instructions. 48 bits<1> has_saddr = 0; 49 bits<1> enabled_saddr = 0; 50 bits<7> saddr_value = 0; 51 bits<1> has_vaddr = 1; 52 53 bits<1> has_data = 1; 54 bits<1> has_glc = 1; 55 bits<1> glcValue = 0; 56 bits<1> has_dlc = 1; 57 bits<1> dlcValue = 0; 58 bits<1> has_sccb = 1; 59 bits<1> sccbValue = 0; 60 bits<1> has_sve = 0; // Scratch VGPR Enable 61 bits<1> sve = 0; 62 63 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 64 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 65 66 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 67 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 68 69 // Internally, FLAT instruction are executed as both an LDS and a 70 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 71 // and are not considered done until both have been decremented. 72 let VM_CNT = 1; 73 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); 74 75 let FlatGlobal = is_flat_global; 76 77 let FlatScratch = is_flat_scratch; 78} 79 80class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 81 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 82 Enc64 { 83 84 let isPseudo = 0; 85 let isCodeGenOnly = 0; 86 87 let FLAT = 1; 88 89 // copy relevant pseudo op flags 90 let SubtargetPredicate = ps.SubtargetPredicate; 91 let AsmMatchConverter = ps.AsmMatchConverter; 92 let OtherPredicates = ps.OtherPredicates; 93 let TSFlags = ps.TSFlags; 94 let UseNamedOperandTable = ps.UseNamedOperandTable; 95 let SchedRW = ps.SchedRW; 96 let mayLoad = ps.mayLoad; 97 let mayStore = ps.mayStore; 98 let IsAtomicRet = ps.IsAtomicRet; 99 let IsAtomicNoRet = ps.IsAtomicNoRet; 100 let VM_CNT = ps.VM_CNT; 101 let LGKM_CNT = ps.LGKM_CNT; 102 let VALU = ps.VALU; 103 104 // encoding fields 105 bits<8> vaddr; 106 bits<10> vdata; 107 bits<7> saddr; 108 bits<10> vdst; 109 110 bits<5> cpol; 111 112 // Only valid on gfx9 113 bits<1> lds = 0; // XXX - What does this actually do? 114 115 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 116 bits<2> seg = !if(ps.is_flat_global, 0b10, 117 !if(ps.is_flat_scratch, 0b01, 0)); 118 119 // Signed offset. Highest bit ignored for flat and treated as 12-bit 120 // unsigned for flat accesses. 121 bits<13> offset; 122 // GFX90A+ only: instruction uses AccVGPR for data 123 bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); 124 125 // We don't use tfe right now, and it was removed in gfx9. 126 bits<1> tfe = 0; 127 128 // Only valid on GFX9+ 129 let Inst{12-0} = offset; 130 let Inst{13} = !if(ps.has_sve, ps.sve, lds); 131 let Inst{15-14} = seg; 132 133 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); 134 let Inst{17} = cpol{CPolBit.SLC}; 135 let Inst{24-18} = op; 136 let Inst{31-26} = 0x37; // Encoding. 137 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 138 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); 139 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 140 141 // 54-48 is reserved. 142 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. 143 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); 144} 145 146class GlobalSaddrTable <bit is_saddr, string Name = ""> { 147 bit IsSaddr = is_saddr; 148 string SaddrOp = Name; 149} 150 151// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 152// same encoding value as exec_hi, so it isn't possible to use that if 153// saddr is 32-bit (which isn't handled here yet). 154class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 155 bit HasTiedOutput = 0, 156 bit HasSaddr = 0, bit EnableSaddr = 0, 157 RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< 158 opName, 159 (outs vdata_op:$vdst), 160 !con( 161 !con( 162 !if(EnableSaddr, 163 (ins SReg_64:$saddr, VGPR_32:$vaddr), 164 (ins VReg_64:$vaddr)), 165 (ins flat_offset:$offset)), 166 // FIXME: Operands with default values do not work with following non-optional operands. 167 !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in), 168 (ins CPol_0:$cpol))), 169 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 170 let has_data = 0; 171 let mayLoad = 1; 172 let has_saddr = HasSaddr; 173 let enabled_saddr = EnableSaddr; 174 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 175 let maybeAtomic = 1; 176 177 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 178 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 179} 180 181class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 182 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 183 opName, 184 (outs), 185 !con( 186 !if(EnableSaddr, 187 (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64:$saddr), 188 (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), 189 (ins flat_offset:$offset, CPol_0:$cpol)), 190 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 191 let mayLoad = 0; 192 let mayStore = 1; 193 let has_vdst = 0; 194 let has_saddr = HasSaddr; 195 let enabled_saddr = EnableSaddr; 196 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 197 let maybeAtomic = 1; 198} 199 200multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 201 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 202 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 203 GlobalSaddrTable<0, opName>; 204 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 205 GlobalSaddrTable<1, opName>; 206 } 207} 208 209class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 210 bit HasTiedOutput = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 211 opName, 212 (outs regClass:$vdst), 213 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), 214 (ins flat_offset:$offset, CPol_0:$cpol), 215 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 216 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 217 let is_flat_global = 1; 218 let has_data = 0; 219 let mayLoad = 1; 220 let has_vaddr = 0; 221 let has_saddr = 1; 222 let enabled_saddr = EnableSaddr; 223 let maybeAtomic = 1; 224 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 225 226 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 227 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 228} 229 230multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, 231 bit HasTiedOutput = 0> { 232 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput>, 233 GlobalSaddrTable<0, opName>; 234 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, 1>, 235 GlobalSaddrTable<1, opName>; 236} 237 238multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 239 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 240 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 241 GlobalSaddrTable<0, opName>; 242 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 243 GlobalSaddrTable<1, opName>; 244 } 245} 246 247class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo< 248 opName, 249 (outs ), 250 !con( 251 !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64:$vaddr)), 252 (ins flat_offset:$offset, CPol_0:$cpol)), 253 " $vaddr"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> { 254 let LGKM_CNT = 1; 255 let is_flat_global = 1; 256 let has_data = 0; 257 let has_vdst = 0; 258 let mayLoad = 1; 259 let mayStore = 1; 260 let has_saddr = 1; 261 let enabled_saddr = EnableSaddr; 262 let VALU = 1; 263 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 264 let Uses = [M0, EXEC]; 265 let SchedRW = [WriteVMEM, WriteLDS]; 266} 267 268multiclass FLAT_Global_Load_LDS_Pseudo<string opName> { 269 def "" : FLAT_Global_Load_LDS_Pseudo<opName>, 270 GlobalSaddrTable<0, opName>; 271 def _SADDR : FLAT_Global_Load_LDS_Pseudo<opName, 1>, 272 GlobalSaddrTable<1, opName>; 273} 274 275class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 276 bit EnableSaddr = 0> : FLAT_Pseudo< 277 opName, 278 (outs), 279 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), 280 (ins flat_offset:$offset, CPol:$cpol)), 281 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 282 let is_flat_global = 1; 283 let mayLoad = 0; 284 let mayStore = 1; 285 let has_vdst = 0; 286 let has_vaddr = 0; 287 let has_saddr = 1; 288 let enabled_saddr = EnableSaddr; 289 let maybeAtomic = 1; 290 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 291} 292 293multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass> { 294 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass>, 295 GlobalSaddrTable<0, opName>; 296 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, 1>, 297 GlobalSaddrTable<1, opName>; 298} 299 300class FlatScratchInst <string sv_op, string mode> { 301 string SVOp = sv_op; 302 string Mode = mode; 303} 304 305class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 306 bit HasTiedOutput = 0, 307 bit EnableSaddr = 0, 308 bit EnableSVE = 0, 309 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> 310 : FLAT_Pseudo< 311 opName, 312 (outs getLdStRegisterOperand<regClass>.ret:$vdst), 313 !con( 314 !if(EnableSVE, 315 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 316 !if(EnableSaddr, 317 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 318 !if(EnableVaddr, 319 (ins VGPR_32:$vaddr, flat_offset:$offset), 320 (ins flat_offset:$offset)))), 321 !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in), 322 (ins CPol_0:$cpol))), 323 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 324 let has_data = 0; 325 let mayLoad = 1; 326 let has_saddr = 1; 327 let enabled_saddr = EnableSaddr; 328 let has_vaddr = EnableVaddr; 329 let has_sve = EnableSVE; 330 let sve = EnableVaddr; 331 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 332 let maybeAtomic = 1; 333 334 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 335 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 336} 337 338class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 339 bit EnableSVE = 0, 340 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr)), 341 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< 342 opName, 343 (outs), 344 !if(EnableSVE, 345 (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 346 !if(EnableSaddr, 347 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 348 !if(EnableVaddr, 349 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol), 350 (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))), 351 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 352 let mayLoad = 0; 353 let mayStore = 1; 354 let has_vdst = 0; 355 let has_saddr = 1; 356 let enabled_saddr = EnableSaddr; 357 let has_vaddr = EnableVaddr; 358 let has_sve = EnableSVE; 359 let sve = EnableVaddr; 360 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 361 let maybeAtomic = 1; 362} 363 364multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 365 let is_flat_scratch = 1 in { 366 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, 367 FlatScratchInst<opName, "SV">; 368 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, 369 FlatScratchInst<opName, "SS">; 370 371 let SubtargetPredicate = HasFlatScratchSVSMode in 372 def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>, 373 FlatScratchInst<opName, "SVS">; 374 375 let SubtargetPredicate = HasFlatScratchSTMode in 376 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>, 377 FlatScratchInst<opName, "ST">; 378 } 379} 380 381multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 382 let is_flat_scratch = 1 in { 383 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, 384 FlatScratchInst<opName, "SV">; 385 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, 386 FlatScratchInst<opName, "SS">; 387 388 let SubtargetPredicate = HasFlatScratchSVSMode in 389 def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>, 390 FlatScratchInst<opName, "SVS">; 391 392 let SubtargetPredicate = HasFlatScratchSTMode in 393 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>, 394 FlatScratchInst<opName, "ST">; 395 } 396} 397 398class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0, 399 bit EnableSVE = 0, 400 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo< 401 opName, 402 (outs ), 403 !if(EnableSVE, 404 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 405 !if(EnableSaddr, 406 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 407 !if(EnableVaddr, 408 (ins VGPR_32:$vaddr, flat_offset:$offset, CPol:$cpol), 409 (ins flat_offset:$offset, CPol:$cpol)))), 410 " "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 411 412 let LGKM_CNT = 1; 413 let is_flat_scratch = 1; 414 let has_data = 0; 415 let has_vdst = 0; 416 let mayLoad = 1; 417 let mayStore = 1; 418 let has_saddr = 1; 419 let enabled_saddr = EnableSaddr; 420 let has_vaddr = EnableVaddr; 421 let has_sve = EnableSVE; 422 let sve = EnableVaddr; 423 let VALU = 1; 424 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 425 let Uses = [M0, EXEC]; 426 let SchedRW = [WriteVMEM, WriteLDS]; 427} 428 429multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> { 430 def "" : FLAT_Scratch_Load_LDS_Pseudo<opName>, 431 FlatScratchInst<opName, "SV">; 432 def _SADDR : FLAT_Scratch_Load_LDS_Pseudo<opName, 1>, 433 FlatScratchInst<opName, "SS">; 434 def _SVS : FLAT_Scratch_Load_LDS_Pseudo<opName, 1, 1>, 435 FlatScratchInst<opName, "SVS">; 436 def _ST : FLAT_Scratch_Load_LDS_Pseudo<opName, 0, 0, 0>, 437 FlatScratchInst<opName, "ST">; 438} 439 440class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 441 string asm, list<dag> pattern = []> : 442 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 443 let mayLoad = 1; 444 let mayStore = 1; 445 let has_glc = 0; 446 let glcValue = 0; 447 let has_vdst = 0; 448 let has_sccb = 1; 449 let sccbValue = 0; 450 let maybeAtomic = 1; 451 let IsAtomicNoRet = 1; 452} 453 454class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 455 string asm, list<dag> pattern = []> 456 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 457 let hasPostISelHook = 1; 458 let has_vdst = 1; 459 let glcValue = 1; 460 let sccbValue = 0; 461 let IsAtomicNoRet = 0; 462 let IsAtomicRet = 1; 463 let PseudoInstr = NAME # "_RTN"; 464} 465 466multiclass FLAT_Atomic_Pseudo< 467 string opName, 468 RegisterClass vdst_rc, 469 ValueType vt, 470 ValueType data_vt = vt, 471 RegisterClass data_rc = vdst_rc, 472 bit isFP = isFloatType<data_vt>.ret, 473 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 474 def "" : FLAT_AtomicNoRet_Pseudo <opName, 475 (outs), 476 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 477 " $vaddr, $vdata$offset$cpol">, 478 GlobalSaddrTable<0, opName>, 479 AtomicNoRet <opName, 0> { 480 let PseudoInstr = NAME; 481 let FPAtomic = isFP; 482 let AddedComplexity = -1; // Prefer global atomics if available 483 } 484 485 def _RTN : FLAT_AtomicRet_Pseudo <opName, 486 (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), 487 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 488 " $vdst, $vaddr, $vdata$offset$cpol">, 489 GlobalSaddrTable<0, opName#"_rtn">, 490 AtomicNoRet <opName, 1> { 491 let FPAtomic = isFP; 492 let AddedComplexity = -1; // Prefer global atomics if available 493 } 494} 495 496multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 497 string opName, 498 RegisterClass vdst_rc, 499 ValueType vt, 500 ValueType data_vt = vt, 501 RegisterClass data_rc = vdst_rc, 502 bit isFP = isFloatType<data_vt>.ret, 503 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 504 505 def "" : FLAT_AtomicNoRet_Pseudo <opName, 506 (outs), 507 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 508 " $vaddr, $vdata, off$offset$cpol">, 509 GlobalSaddrTable<0, opName>, 510 AtomicNoRet <opName, 0> { 511 let has_saddr = 1; 512 let PseudoInstr = NAME; 513 let FPAtomic = isFP; 514 } 515 516 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 517 (outs), 518 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_0:$cpol), 519 " $vaddr, $vdata, $saddr$offset$cpol">, 520 GlobalSaddrTable<1, opName>, 521 AtomicNoRet <opName#"_saddr", 0> { 522 let has_saddr = 1; 523 let enabled_saddr = 1; 524 let PseudoInstr = NAME#"_SADDR"; 525 let FPAtomic = isFP; 526 } 527} 528 529multiclass FLAT_Global_Atomic_Pseudo_RTN< 530 string opName, 531 RegisterClass vdst_rc, 532 ValueType vt, 533 ValueType data_vt = vt, 534 RegisterClass data_rc = vdst_rc, 535 bit isFP = isFloatType<data_vt>.ret, 536 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, 537 RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { 538 539 def _RTN : FLAT_AtomicRet_Pseudo <opName, 540 (outs vdst_op:$vdst), 541 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 542 " $vdst, $vaddr, $vdata, off$offset$cpol">, 543 GlobalSaddrTable<0, opName#"_rtn">, 544 AtomicNoRet <opName, 1> { 545 let has_saddr = 1; 546 let FPAtomic = isFP; 547 } 548 549 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 550 (outs vdst_op:$vdst), 551 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_GLC1:$cpol), 552 " $vdst, $vaddr, $vdata, $saddr$offset$cpol">, 553 GlobalSaddrTable<1, opName#"_rtn">, 554 AtomicNoRet <opName#"_saddr", 1> { 555 let has_saddr = 1; 556 let enabled_saddr = 1; 557 let PseudoInstr = NAME#"_SADDR_RTN"; 558 let FPAtomic = isFP; 559 } 560} 561 562multiclass FLAT_Global_Atomic_Pseudo< 563 string opName, 564 RegisterClass vdst_rc, 565 ValueType vt, 566 ValueType data_vt = vt, 567 RegisterClass data_rc = vdst_rc> { 568 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 569 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc>; 570 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, data_vt, data_rc>; 571 } 572} 573 574//===----------------------------------------------------------------------===// 575// Flat Instructions 576//===----------------------------------------------------------------------===// 577 578def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 579def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 580def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 581def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 582def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 583def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 584def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 585def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 586 587def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 588def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 589def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 590def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 591def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 592def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 593 594let SubtargetPredicate = HasD16LoadStore in { 595def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 596def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 597def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 598def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 599def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 600def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 601 602def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 603def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 604} 605 606defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 607 VGPR_32, i32, v2i32, VReg_64>; 608 609defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 610 VReg_64, i64, v2i64, VReg_128>; 611 612defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 613 VGPR_32, i32>; 614 615defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 616 VReg_64, i64>; 617 618defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 619 VGPR_32, i32>; 620 621defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 622 VGPR_32, i32>; 623 624defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 625 VGPR_32, i32>; 626 627defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 628 VGPR_32, i32>; 629 630defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 631 VGPR_32, i32>; 632 633defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 634 VGPR_32, i32>; 635 636defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 637 VGPR_32, i32>; 638 639defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 640 VGPR_32, i32>; 641 642defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 643 VGPR_32, i32>; 644 645defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 646 VGPR_32, i32>; 647 648defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 649 VGPR_32, i32>; 650 651defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 652 VReg_64, i64>; 653 654defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 655 VReg_64, i64>; 656 657defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 658 VReg_64, i64>; 659 660defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 661 VReg_64, i64>; 662 663defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 664 VReg_64, i64>; 665 666defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 667 VReg_64, i64>; 668 669defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 670 VReg_64, i64>; 671 672defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 673 VReg_64, i64>; 674 675defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 676 VReg_64, i64>; 677 678defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 679 VReg_64, i64>; 680 681defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 682 VReg_64, i64>; 683 684// GFX7-, GFX10-only flat instructions. 685let SubtargetPredicate = isGFX7GFX10 in { 686 687defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 688 VGPR_32, f32, v2f32, VReg_64>; 689 690defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 691 VReg_64, f64, v2f64, VReg_128>; 692 693defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 694 VGPR_32, f32>; 695 696defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 697 VGPR_32, f32>; 698 699defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 700 VReg_64, f64>; 701 702defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 703 VReg_64, f64>; 704 705} // End SubtargetPredicate = isGFX7GFX10 706 707let SubtargetPredicate = isGFX90APlus in { 708 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64>; 709 defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo<"flat_atomic_min_f64", VReg_64, f64>; 710 defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo<"flat_atomic_max_f64", VReg_64, f64>; 711 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64>; 712 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64>; 713 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64>; 714} // End SubtargetPredicate = isGFX90APlus 715 716let SubtargetPredicate = isGFX940Plus in { 717 defm FLAT_ATOMIC_ADD_F32 : FLAT_Atomic_Pseudo<"flat_atomic_add_f32", VGPR_32, f32>; 718 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_f16", VGPR_32, v2f16>; 719 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_bf16", VGPR_32, v2f16>; 720 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Atomic_Pseudo<"global_atomic_pk_add_bf16", VGPR_32, v2f16>; 721} // End SubtargetPredicate = isGFX940Plus 722 723defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 724defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 725defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 726defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 727defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 728defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 729defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 730defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 731 732defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 733defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 734defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 735defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 736defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 737defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 738let OtherPredicates = [HasGFX10_BEncoding] in 739defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 740 741defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 742defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 743defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 744defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 745defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 746defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 747let OtherPredicates = [HasGFX10_BEncoding] in 748defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 749 750defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 751defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 752 753let is_flat_global = 1 in { 754defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 755 VGPR_32, i32, v2i32, VReg_64>; 756 757defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 758 VReg_64, i64, v2i64, VReg_128>; 759 760defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 761 VGPR_32, i32>; 762 763defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 764 VReg_64, i64>; 765 766defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 767 VGPR_32, i32>; 768 769defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 770 VGPR_32, i32>; 771 772defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 773 VGPR_32, i32>; 774 775defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 776 VGPR_32, i32>; 777 778defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 779 VGPR_32, i32>; 780 781defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 782 VGPR_32, i32>; 783 784defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 785 VGPR_32, i32>; 786 787defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 788 VGPR_32, i32>; 789 790defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 791 VGPR_32, i32>; 792 793defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 794 VGPR_32, i32>; 795 796defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 797 VGPR_32, i32>; 798 799defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 800 VReg_64, i64>; 801 802defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 803 VReg_64, i64>; 804 805defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 806 VReg_64, i64>; 807 808defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 809 VReg_64, i64>; 810 811defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 812 VReg_64, i64>; 813 814defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 815 VReg_64, i64>; 816 817defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 818 VReg_64, i64>; 819 820defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 821 VReg_64, i64>; 822 823defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 824 VReg_64, i64>; 825 826defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 827 VReg_64, i64>; 828 829defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 830 VReg_64, i64>; 831 832let SubtargetPredicate = HasGFX10_BEncoding in 833defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 834 VGPR_32, i32>; 835 836let SubtargetPredicate = isGFX940Plus in { 837 838defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">; 839defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">; 840defm GLOBAL_LOAD_LDS_USHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ushort">; 841defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sshort">; 842defm GLOBAL_LOAD_LDS_DWORD : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dword">; 843 844} // End let SubtargetPredicate = isGFX940Plus 845} // End is_flat_global = 1 846 847 848 849let SubtargetPredicate = HasFlatScratchInsts in { 850defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 851defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 852defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 853defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 854defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 855defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 856defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 857defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 858 859defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 860defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 861defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 862defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 863defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 864defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 865 866defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 867defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 868defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 869defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 870defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 871defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 872 873defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 874defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 875 876let SubtargetPredicate = isGFX940Plus in { 877 878defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ubyte">; 879defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sbyte">; 880defm SCRATCH_LOAD_LDS_USHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ushort">; 881defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sshort">; 882defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_dword">; 883 884} // End let SubtargetPredicate = isGFX940Plus 885 886} // End SubtargetPredicate = HasFlatScratchInsts 887 888let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 889 defm GLOBAL_ATOMIC_FCMPSWAP : 890 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32, v2f32, VReg_64>; 891 defm GLOBAL_ATOMIC_FMIN : 892 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 893 defm GLOBAL_ATOMIC_FMAX : 894 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 895 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 896 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64, v2f64, VReg_128>; 897 defm GLOBAL_ATOMIC_FMIN_X2 : 898 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 899 defm GLOBAL_ATOMIC_FMAX_X2 : 900 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 901} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 902 903let is_flat_global = 1 in { 904let OtherPredicates = [HasAtomicFaddNoRtnInsts] in 905 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 906 "global_atomic_add_f32", VGPR_32, f32 907 >; 908let OtherPredicates = [HasAtomicPkFaddNoRtnInsts] in 909 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 910 "global_atomic_pk_add_f16", VGPR_32, v2f16 911 >; 912let OtherPredicates = [HasAtomicFaddRtnInsts] in 913 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < 914 "global_atomic_add_f32", VGPR_32, f32 915 >; 916let OtherPredicates = [isGFX90APlus] in 917 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < 918 "global_atomic_pk_add_f16", VGPR_32, v2f16 919 >; 920} // End is_flat_global = 1 921 922//===----------------------------------------------------------------------===// 923// Flat Patterns 924//===----------------------------------------------------------------------===// 925 926// Patterns for global loads with no offset. 927class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 928 (vt (node (FlatOffset i64:$vaddr, i16:$offset))), 929 (inst $vaddr, $offset) 930>; 931 932class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 933 (node (FlatOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 934 (inst $vaddr, $offset, 0, $in) 935>; 936 937class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 938 (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 939 (inst $vaddr, $offset, 0, $in) 940>; 941 942class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 943 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 944 (inst $saddr, $voffset, $offset, 0, $in) 945>; 946 947class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 948 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset))), 949 (inst $vaddr, $offset) 950>; 951 952class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 953 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 954 (inst $saddr, $voffset, $offset, 0) 955>; 956 957class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 958 ValueType vt> : GCNPat < 959 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 960 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 961>; 962 963class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 964 ValueType vt> : GCNPat < 965 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 966 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 967>; 968 969class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 970 ValueType vt, ValueType data_vt = vt> : GCNPat < 971 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 972 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 973>; 974 975class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 976 ValueType vt> : GCNPat < 977 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 978 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 979>; 980 981class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 982 (node vt:$data, (FlatOffset i64:$vaddr, i16:$offset)), 983 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 984>; 985 986class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 987 (node vt:$data, (GlobalOffset i64:$vaddr, i16:$offset)), 988 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 989>; 990 991class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 992 // atomic store follows atomic binop convention so the address comes 993 // first. 994 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 995 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 996>; 997 998class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 999 ValueType vt, ValueType data_vt = vt> : GCNPat < 1000 // atomic store follows atomic binop convention so the address comes 1001 // first. 1002 (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data), 1003 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1004>; 1005 1006class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1007 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 1008 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1009>; 1010 1011multiclass FlatAtomicPat <string inst, string node, ValueType vt, 1012 ValueType data_vt = vt> { 1013 defvar rtnNode = !cast<PatFrags>(node#"_ret_"#vt.Size); 1014 defvar noRtnNode = !cast<PatFrags>(node#"_noret_"#vt.Size); 1015 1016 def : GCNPat <(vt (rtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1017 (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1018 1019 def : GCNPat <(vt (noRtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1020 (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1021} 1022 1023multiclass FlatSignedAtomicPat <string inst, string node, ValueType vt, 1024 ValueType data_vt = vt, bit isIntr = 0> { 1025 defvar rtnNode = !cast<PatFrags>(node # "_ret" # !if(isIntr, "", "_" # vt.Size)); 1026 defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt.Size)); 1027 1028 def : GCNPat <(vt (rtnNode (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1029 (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1030 1031 def : GCNPat <(vt (noRtnNode (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1032 (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1033} 1034 1035multiclass FlatSignedAtomicIntrPat <string inst, string node, ValueType vt, 1036 ValueType data_vt = vt> { 1037 defm : FlatSignedAtomicPat<inst, node, vt, data_vt, /* isIntr */ 1>; 1038} 1039 1040class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1041 (node (GlobalOffset i64:$vaddr, i16:$offset), vt:$data), 1042 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1043>; 1044 1045class FlatSignedAtomicPatRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 1046 ValueType data_vt = vt> : GCNPat < 1047 (vt (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1048 (inst VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1049>; 1050 1051class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1052 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset))), 1053 (inst $vaddr, $offset) 1054>; 1055 1056class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1057 (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset), vt:$in), 1058 (inst $vaddr, $offset, 0, $in) 1059>; 1060 1061class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1062 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset)), 1063 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) 1064>; 1065 1066class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1067 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset))), 1068 (inst $saddr, $offset) 1069>; 1070 1071class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1072 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1073 (inst $saddr, $offset, 0, $in) 1074>; 1075 1076class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1077 ValueType vt> : GCNPat < 1078 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)), 1079 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 1080>; 1081 1082class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1083 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset))), 1084 (inst $vaddr, $saddr, $offset, 0) 1085>; 1086 1087class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1088 ValueType vt> : GCNPat < 1089 (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset)), 1090 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset) 1091>; 1092 1093class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1094 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1095 (inst $vaddr, $saddr, $offset, 0, $in) 1096>; 1097 1098let OtherPredicates = [HasFlatAddressSpace] in { 1099 1100def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i32>; 1101def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i16>; 1102def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i32>; 1103def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i16>; 1104def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 1105def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 1106def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 1107def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 1108def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 1109def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 1110def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 1111def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 1112def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 1113def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 1114def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 1115 1116def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 1117def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 1118 1119def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 1120def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 1121 1122foreach vt = Reg32Types.types in { 1123def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 1124def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 1125} 1126 1127foreach vt = VReg_64.RegTypes in { 1128def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 1129def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 1130} 1131 1132def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 1133 1134foreach vt = VReg_128.RegTypes in { 1135def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 1136def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 1137} 1138 1139def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>; 1140def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>; 1141def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>; 1142def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>; 1143def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>; 1144def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; 1145 1146foreach as = [ "flat", "global" ] in { 1147defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; 1148defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>; 1149defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_inc_"#as, i32>; 1150defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_dec_"#as, i32>; 1151defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>; 1152defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>; 1153defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>; 1154defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN", "atomic_load_min_"#as, i32>; 1155defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN", "atomic_load_umin_"#as, i32>; 1156defm : FlatAtomicPat <"FLAT_ATOMIC_OR", "atomic_load_or_"#as, i32>; 1157defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP", "atomic_swap_"#as, i32>; 1158defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_"#as, i32, v2i32>; 1159defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>; 1160 1161defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>; 1162defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>; 1163defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_inc_"#as, i64>; 1164defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_dec_"#as, i64>; 1165defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>; 1166defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>; 1167defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>; 1168defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN_X2", "atomic_load_min_"#as, i64>; 1169defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN_X2", "atomic_load_umin_"#as, i64>; 1170defm : FlatAtomicPat <"FLAT_ATOMIC_OR_X2", "atomic_load_or_"#as, i64>; 1171defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP_X2", "atomic_swap_"#as, i64>; 1172defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_"#as, i64, v2i64>; 1173defm : FlatAtomicPat <"FLAT_ATOMIC_XOR_X2", "atomic_load_xor_"#as, i64>; 1174} // end foreach as 1175 1176def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 1177def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 1178 1179let OtherPredicates = [HasD16LoadStore] in { 1180def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 1181def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 1182} 1183 1184let OtherPredicates = [D16PreservesUnusedBits] in { 1185def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 1186def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 1187def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 1188def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 1189def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 1190def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 1191 1192def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 1193def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 1194def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 1195def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 1196def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 1197def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 1198} 1199 1200} // End OtherPredicates = [HasFlatAddressSpace] 1201 1202 1203multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1204 def : FlatLoadSignedPat <inst, node, vt> { 1205 let AddedComplexity = 10; 1206 } 1207 1208 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1209 let AddedComplexity = 11; 1210 } 1211} 1212 1213multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1214 def : FlatSignedLoadPat_D16 <inst, node, vt> { 1215 let AddedComplexity = 10; 1216 } 1217 1218 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1219 let AddedComplexity = 11; 1220 } 1221} 1222 1223multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1224 ValueType vt> { 1225 def : FlatStoreSignedPat <inst, node, vt> { 1226 let AddedComplexity = 10; 1227 } 1228 1229 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1230 let AddedComplexity = 11; 1231 } 1232} 1233 1234// Deal with swapped operands for atomic_store vs. regular store 1235multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1236 def : FlatStoreSignedAtomicPat <inst, node, vt> { 1237 let AddedComplexity = 10; 1238 } 1239 1240 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1241 let AddedComplexity = 11; 1242 } 1243} 1244 1245multiclass GlobalFLATAtomicPatsRtn<string nortn_inst_name, SDPatternOperator node, 1246 ValueType vt, ValueType data_vt = vt> { 1247 def : FlatSignedAtomicPatRtn <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 1248 let AddedComplexity = 10; 1249 } 1250 1251 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 1252 let AddedComplexity = 11; 1253 } 1254} 1255 1256multiclass GlobalFLATAtomicPats<string inst, string node, ValueType vt, 1257 ValueType data_vt = vt, bit isIntr = 0> { 1258 defvar rtnNode = !cast<PatFrags>(node # "_ret" # !if(isIntr, "", "_" # vt.Size)); 1259 defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt.Size)); 1260 1261 let AddedComplexity = 10 in { 1262 defm : FlatSignedAtomicPat <inst, node, vt, data_vt, isIntr>; 1263 } 1264 1265 let AddedComplexity = 11 in { 1266 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR"), noRtnNode, vt, data_vt>; 1267 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_RTN"), rtnNode, vt, data_vt>; 1268 } 1269} 1270 1271multiclass GlobalFLATAtomicIntrPats<string inst, string node, ValueType vt, 1272 ValueType data_vt = vt> { 1273 defm : GlobalFLATAtomicPats<inst, node, vt, data_vt, /* isIntr */ 1>; 1274} 1275 1276multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1277 ValueType vt> { 1278 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1279 let AddedComplexity = 10; 1280 } 1281 1282 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1283 let AddedComplexity = 11; 1284 } 1285} 1286 1287multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1288 def : ScratchLoadSignedPat <inst, node, vt> { 1289 let AddedComplexity = 25; 1290 } 1291 1292 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1293 let AddedComplexity = 26; 1294 } 1295 1296 def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1297 let SubtargetPredicate = HasFlatScratchSVSMode; 1298 let AddedComplexity = 27; 1299 } 1300} 1301 1302multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1303 ValueType vt> { 1304 def : ScratchStoreSignedPat <inst, node, vt> { 1305 let AddedComplexity = 25; 1306 } 1307 1308 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1309 let AddedComplexity = 26; 1310 } 1311 1312 def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1313 let SubtargetPredicate = HasFlatScratchSVSMode; 1314 let AddedComplexity = 27; 1315 } 1316} 1317 1318multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1319 def : ScratchLoadSignedPat_D16 <inst, node, vt> { 1320 let AddedComplexity = 25; 1321 } 1322 1323 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1324 let AddedComplexity = 26; 1325 } 1326 1327 def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1328 let SubtargetPredicate = HasFlatScratchSVSMode; 1329 let AddedComplexity = 27; 1330 } 1331} 1332 1333let OtherPredicates = [HasFlatGlobalInsts] in { 1334 1335defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i32>; 1336defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i16>; 1337defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i32>; 1338defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i16>; 1339defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1340defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1341defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1342defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1343defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1344defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1345defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1346defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1347defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1348defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1349 1350foreach vt = Reg32Types.types in { 1351defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1352defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1353} 1354 1355foreach vt = VReg_64.RegTypes in { 1356defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1357defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1358} 1359 1360defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1361 1362foreach vt = VReg_128.RegTypes in { 1363defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1364defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1365} 1366 1367// There is no distinction for atomic load lowering during selection; 1368// the memory legalizer will set the cache bits and insert the 1369// appropriate waits. 1370defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1371defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1372 1373defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1374defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1375defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1376defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1377defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1378 1379let OtherPredicates = [HasD16LoadStore] in { 1380defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1381defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1382} 1383 1384let OtherPredicates = [D16PreservesUnusedBits] in { 1385defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1386defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1387defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1388defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1389defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1390defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1391 1392defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1393defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1394defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1395defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1396defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1397defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1398} 1399 1400defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>; 1401defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>; 1402defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>; 1403defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>; 1404defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>; 1405defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>; 1406 1407defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; 1408defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; 1409defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_inc_global", i32>; 1410defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_dec_global", i32>; 1411defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>; 1412defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>; 1413defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>; 1414defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", "atomic_load_min_global", i32>; 1415defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", "atomic_load_umin_global", i32>; 1416defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", "atomic_load_or_global", i32>; 1417defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", "atomic_swap_global", i32>; 1418defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_global", i32, v2i32>; 1419defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>; 1420defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1421 1422defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>; 1423defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>; 1424defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_inc_global", i64>; 1425defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_dec_global", i64>; 1426defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>; 1427defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>; 1428defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>; 1429defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", "atomic_load_min_global", i64>; 1430defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", "atomic_load_umin_global", i64>; 1431defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", "atomic_load_or_global", i64>; 1432defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", "atomic_swap_global", i64>; 1433defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_global", i64, v2i64>; 1434defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", "atomic_load_xor_global", i64>; 1435 1436let OtherPredicates = [isGFX10Plus] in { 1437defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>; 1438defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>; 1439defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN_X2", "atomic_load_fmin_global", f64>; 1440defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX_X2", "atomic_load_fmax_global", f64>; 1441defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin", f32>; 1442defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax", f32>; 1443defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN_X2", "int_amdgcn_global_atomic_fmin", f64>; 1444defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX_X2", "int_amdgcn_global_atomic_fmax", f64>; 1445} 1446 1447let OtherPredicates = [HasAtomicFaddNoRtnInsts] in 1448defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1449let OtherPredicates = [HasAtomicPkFaddNoRtnInsts] in 1450defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1451 1452let OtherPredicates = [isGFX90APlus] in { 1453defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>; 1454defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_v2f16_global", v2f16>; 1455defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", "atomic_load_fadd_global", f64>; 1456defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global", f64>; 1457defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>; 1458defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_ADD_F32", "int_amdgcn_global_atomic_fadd", f32>; 1459defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_ADD_F64", "int_amdgcn_global_atomic_fadd", f64>; 1460defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_PK_ADD_F16", "int_amdgcn_global_atomic_fadd", v2f16>; 1461defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MIN_F64", "int_amdgcn_global_atomic_fmin", f64>; 1462defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MAX_F64", "int_amdgcn_global_atomic_fmax", f64>; 1463defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>; 1464defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>; 1465defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>; 1466defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_ADD_F64", "int_amdgcn_flat_atomic_fadd", f64>; 1467defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>; 1468defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>; 1469} 1470 1471let OtherPredicates = [isGFX940Plus] in { 1472defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>; 1473defm : FlatSignedAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_v2f16_flat", v2f16>; 1474defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_ADD_F32", "int_amdgcn_flat_atomic_fadd", f32>; 1475defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_F16", "int_amdgcn_flat_atomic_fadd", v2f16>; 1476defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_BF16", "int_amdgcn_flat_atomic_fadd_v2bf16", v2i16>; 1477defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "int_amdgcn_global_atomic_fadd_v2bf16", v2i16>; 1478} 1479 1480} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1481 1482let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { 1483 1484defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; 1485defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; 1486defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; 1487defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; 1488defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; 1489defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; 1490defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; 1491defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; 1492defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; 1493defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; 1494 1495foreach vt = Reg32Types.types in { 1496defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; 1497defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; 1498} 1499 1500foreach vt = VReg_64.RegTypes in { 1501defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; 1502defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; 1503} 1504 1505defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; 1506 1507foreach vt = VReg_128.RegTypes in { 1508defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; 1509defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; 1510} 1511 1512defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; 1513defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; 1514defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; 1515defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; 1516defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; 1517 1518let OtherPredicates = [HasD16LoadStore, HasFlatScratchInsts, EnableFlatScratch] in { 1519defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; 1520defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; 1521} 1522 1523let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { 1524defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; 1525defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; 1526defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; 1527defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; 1528defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; 1529defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; 1530 1531defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; 1532defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; 1533defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; 1534defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; 1535defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; 1536defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; 1537} 1538 1539} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] 1540 1541//===----------------------------------------------------------------------===// 1542// Target 1543//===----------------------------------------------------------------------===// 1544 1545//===----------------------------------------------------------------------===// 1546// CI 1547//===----------------------------------------------------------------------===// 1548 1549class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1550 FLAT_Real <op, ps>, 1551 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1552 let AssemblerPredicate = isGFX7Only; 1553 let DecoderNamespace="GFX7"; 1554} 1555 1556def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1557def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1558def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1559def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1560def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1561def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1562def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1563def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1564 1565def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1566def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1567def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1568def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1569def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1570def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1571 1572multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1573 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1574 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1575} 1576 1577defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1578defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1579defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1580defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1581defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1582defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1583defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1584defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1585defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1586defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1587defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1588defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1589defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1590defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1591defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1592defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1593defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1594defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1595defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1596defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1597defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1598defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1599defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1600defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1601defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1602defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1603 1604// CI Only flat instructions 1605defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1606defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1607defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1608defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1609defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1610defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1611 1612 1613//===----------------------------------------------------------------------===// 1614// VI 1615//===----------------------------------------------------------------------===// 1616 1617class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> : 1618 FLAT_Real <op, ps>, 1619 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1620 let AssemblerPredicate = isGFX8GFX9; 1621 let DecoderNamespace = "GFX8"; 1622 1623 let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1624 let AsmString = ps.Mnemonic # 1625 !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands); 1626} 1627 1628multiclass FLAT_Real_AllAddr_vi<bits<7> op, 1629 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1630 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>; 1631 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>; 1632} 1633 1634class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> : 1635 FLAT_Real <op, ps>, 1636 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> { 1637 let AssemblerPredicate = isGFX940Plus; 1638 let DecoderNamespace = "GFX9"; 1639 let Inst{13} = ps.sve; 1640 let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1641} 1642 1643multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> { 1644 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> { 1645 let AssemblerPredicate = isGFX8GFX9NotGFX940; 1646 let OtherPredicates = [isGFX8GFX9NotGFX940]; 1647 } 1648 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> { 1649 let DecoderNamespace = "GFX9"; 1650 } 1651 let AssemblerPredicate = isGFX940Plus, SubtargetPredicate = isGFX940Plus in { 1652 def _VE_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1653 def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>; 1654 def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; 1655 } 1656} 1657 1658def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1659def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1660def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1661def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1662def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1663def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1664def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1665def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1666 1667def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1668def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1669def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1670def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1671def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1672def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1673def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1674def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1675 1676def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1677def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1678def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1679def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1680def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1681def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1682 1683multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps, 1684 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1685 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>; 1686 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>; 1687} 1688 1689multiclass FLAT_Global_Real_Atomics_vi<bits<7> op, 1690 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> : 1691 FLAT_Real_AllAddr_vi<op, has_sccb> { 1692 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>; 1693 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>; 1694} 1695 1696 1697defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1698defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1699defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1700defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1701defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1702defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1703defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1704defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1705defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1706defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1707defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1708defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1709defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1710defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1711defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1712defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1713defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1714defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1715defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1716defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1717defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1718defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1719defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1720defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1721defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1722defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1723 1724defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1725defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1726defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1727defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1728defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1729defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1730defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1731defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1732 1733defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1734defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1735defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1736defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1737defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1738defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1739 1740defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1741defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1742defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1743defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1744defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1745defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1746defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1747defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1748 1749let AssemblerPredicate = isGFX940Plus in { 1750defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_vi <0x026>; 1751defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_vi <0x027>; 1752defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_vi <0x028>; 1753defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_vi <0x029>; 1754defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_vi <0x02a>; 1755} // End let AssemblerPredicate = isGFX940Plus 1756 1757defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1758defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1759defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1760defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1761defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1762defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1763defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1764defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1765defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1766defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1767defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1768defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1769defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1770defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1771defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1772defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1773defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1774defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1775defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1776defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1777defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1778defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1779defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1780defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1781defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1782defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1783 1784let AssemblerPredicate = isGFX940Plus in { 1785defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x026>; 1786defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x027>; 1787defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_AllAddr_SVE_vi <0x028>; 1788defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x029>; 1789defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_AllAddr_SVE_vi <0x02a>; 1790} // End let AssemblerPredicate = isGFX940Plus 1791 1792defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x10>; 1793defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x11>; 1794defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_SVE_vi <0x12>; 1795defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x13>; 1796defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_SVE_vi <0x14>; 1797defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x15>; 1798defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x16>; 1799defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x17>; 1800defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_SVE_vi <0x18>; 1801defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x19>; 1802defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x20>; 1803defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x21>; 1804defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x22>; 1805defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x23>; 1806defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_SVE_vi <0x24>; 1807defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x25>; 1808defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_SVE_vi <0x1a>; 1809defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>; 1810defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_SVE_vi <0x1c>; 1811defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x1d>; 1812defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x1e>; 1813defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x1f>; 1814 1815let SubtargetPredicate = isGFX8GFX9NotGFX940 in { 1816 // These instructions are encoded differently on gfx90* and gfx940. 1817 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>; 1818 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>; 1819} 1820 1821let SubtargetPredicate = isGFX90AOnly in { 1822 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64, 0>; 1823 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64, 0>; 1824 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64, 0>; 1825 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; 1826 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; 1827 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; 1828} // End SubtargetPredicate = isGFX90AOnly 1829 1830multiclass FLAT_Real_AllAddr_gfx940<bits<7> op> { 1831 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1832 def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1833} 1834 1835multiclass FLAT_Real_Atomics_gfx940 <bits<7> op, FLAT_Pseudo ps> { 1836 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1837 def _RTN_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1838} 1839 1840multiclass FLAT_Global_Real_Atomics_gfx940<bits<7> op> : 1841 FLAT_Real_AllAddr_gfx940<op> { 1842 def _RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1843 def _SADDR_RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1844} 1845 1846let SubtargetPredicate = isGFX940Plus in { 1847 // These instructions are encoded differently on gfx90* and gfx940. 1848 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>; 1849 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>; 1850 1851 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f, FLAT_ATOMIC_ADD_F64>; 1852 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50, FLAT_ATOMIC_MIN_F64>; 1853 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51, FLAT_ATOMIC_MAX_F64>; 1854 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>; 1855 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>; 1856 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>; 1857 defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d, FLAT_ATOMIC_ADD_F32>; 1858 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e, FLAT_ATOMIC_PK_ADD_F16>; 1859 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52, FLAT_ATOMIC_PK_ADD_BF16>; 1860 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>; 1861} // End SubtargetPredicate = isGFX940Plus 1862 1863//===----------------------------------------------------------------------===// 1864// GFX10. 1865//===----------------------------------------------------------------------===// 1866 1867class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1868 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1869 let AssemblerPredicate = isGFX10Plus; 1870 let DecoderNamespace = "GFX10"; 1871 1872 let Inst{11-0} = offset{11-0}; 1873 let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); 1874 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1875 let Inst{55} = 0; 1876} 1877 1878 1879multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1880 def _gfx10 : 1881 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1882} 1883 1884multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1885 def _RTN_gfx10 : 1886 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1887} 1888 1889multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1890 def _SADDR_gfx10 : 1891 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1892} 1893 1894multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1895 def _SADDR_RTN_gfx10 : 1896 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1897} 1898 1899multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1900 def _ST_gfx10 : 1901 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1902 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1903 let OtherPredicates = [HasFlatScratchSTMode]; 1904 } 1905} 1906 1907multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1908 FLAT_Real_Base_gfx10<op>, 1909 FLAT_Real_SADDR_gfx10<op>; 1910 1911multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1912 FLAT_Real_Base_gfx10<op>, 1913 FLAT_Real_RTN_gfx10<op>; 1914 1915multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1916 FLAT_Real_AllAddr_gfx10<op>, 1917 FLAT_Real_RTN_gfx10<op>, 1918 FLAT_Real_SADDR_RTN_gfx10<op>; 1919 1920multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1921 FLAT_Real_RTN_gfx10<op>, 1922 FLAT_Real_SADDR_RTN_gfx10<op>; 1923 1924multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1925 FLAT_Real_Base_gfx10<op>, 1926 FLAT_Real_SADDR_gfx10<op>, 1927 FLAT_Real_ST_gfx10<op>; 1928 1929// ENC_FLAT. 1930defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1931defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1932defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1933defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1934defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1935defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1936defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1937defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1938defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1939defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1940defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1941defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1942defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1943defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1944defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1945defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1946defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1947defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1948defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1949defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1950defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1951defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1952defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1953defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1954defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1955defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1956defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1957defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1958defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1959defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1960defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1961defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1962defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1963defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1964defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1965defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1966defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1967defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1968defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1969defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1970defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1971defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1972defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1973defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1974defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1975defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1976defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1977defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1978defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1979defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1980defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1981defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1982defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1983defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1984 1985 1986// ENC_FLAT_GLBL. 1987defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1988defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1989defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1990defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1991defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1992defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1993defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1994defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1995defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1996defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1997defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1998defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 1999defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 2000defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 2001defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 2002defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 2003defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 2004defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 2005defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 2006defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 2007defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 2008defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 2009defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 2010defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 2011defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 2012defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 2013defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 2014defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 2015defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 2016defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 2017defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 2018defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 2019defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 2020defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 2021defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 2022defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 2023defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 2024defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 2025defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 2026defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 2027defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 2028defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 2029defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 2030defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 2031defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 2032defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 2033defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 2034defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 2035defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 2036defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 2037defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 2038defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 2039defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 2040defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 2041defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 2042defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; 2043defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; 2044 2045// ENC_FLAT_SCRATCH. 2046defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 2047defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 2048defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 2049defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 2050defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 2051defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 2052defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 2053defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 2054defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 2055defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 2056defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 2057defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 2058defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 2059defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 2060defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 2061defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 2062defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 2063defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 2064defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 2065defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 2066defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 2067defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 2068