1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>; 10def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>; 11def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>; 12 13def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; 14def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; 15def ScratchSVAddr : ComplexPattern<iPTR, 3, "SelectScratchSVAddr", [], [SDNPWantRoot], -10>; 16 17//===----------------------------------------------------------------------===// 18// FLAT classes 19//===----------------------------------------------------------------------===// 20 21class FLAT_Pseudo<string opName, dag outs, dag ins, 22 string asmOps, list<dag> pattern=[]> : 23 InstSI<outs, ins, "", pattern>, 24 SIMCInstr<opName, SIEncodingFamily.NONE> { 25 26 let isPseudo = 1; 27 let isCodeGenOnly = 1; 28 29 let FLAT = 1; 30 31 let UseNamedOperandTable = 1; 32 let hasSideEffects = 0; 33 let SchedRW = [WriteVMEM]; 34 35 string Mnemonic = opName; 36 string AsmOperands = asmOps; 37 38 bits<1> is_flat_global = 0; 39 bits<1> is_flat_scratch = 0; 40 41 bits<1> has_vdst = 1; 42 43 // We need to distinguish having saddr and enabling saddr because 44 // saddr is only valid for scratch and global instructions. Pre-gfx9 45 // these bits were reserved, so we also don't necessarily want to 46 // set these bits to the disabled value for the original flat 47 // segment instructions. 48 bits<1> has_saddr = 0; 49 bits<1> enabled_saddr = 0; 50 bits<7> saddr_value = 0; 51 bits<1> has_vaddr = 1; 52 53 bits<1> has_data = 1; 54 bits<1> has_glc = 1; 55 bits<1> glcValue = 0; 56 bits<1> has_dlc = 1; 57 bits<1> dlcValue = 0; 58 bits<1> has_sccb = 1; 59 bits<1> sccbValue = 0; 60 bits<1> has_sve = 0; // Scratch VGPR Enable 61 bits<1> sve = 0; 62 63 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, 64 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); 65 66 // TODO: M0 if it could possibly access LDS (before gfx9? only)? 67 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); 68 69 // Internally, FLAT instruction are executed as both an LDS and a 70 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT 71 // and are not considered done until both have been decremented. 72 let VM_CNT = 1; 73 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); 74 75 let FlatGlobal = is_flat_global; 76 77 let FlatScratch = is_flat_scratch; 78} 79 80class FLAT_Real <bits<7> op, FLAT_Pseudo ps> : 81 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 82 Enc64 { 83 84 let isPseudo = 0; 85 let isCodeGenOnly = 0; 86 87 let FLAT = 1; 88 89 // copy relevant pseudo op flags 90 let SubtargetPredicate = ps.SubtargetPredicate; 91 let AsmMatchConverter = ps.AsmMatchConverter; 92 let OtherPredicates = ps.OtherPredicates; 93 let TSFlags = ps.TSFlags; 94 let UseNamedOperandTable = ps.UseNamedOperandTable; 95 let SchedRW = ps.SchedRW; 96 let mayLoad = ps.mayLoad; 97 let mayStore = ps.mayStore; 98 let IsAtomicRet = ps.IsAtomicRet; 99 let IsAtomicNoRet = ps.IsAtomicNoRet; 100 let VM_CNT = ps.VM_CNT; 101 let LGKM_CNT = ps.LGKM_CNT; 102 let VALU = ps.VALU; 103 104 // encoding fields 105 bits<8> vaddr; 106 bits<10> vdata; 107 bits<7> saddr; 108 bits<10> vdst; 109 110 bits<5> cpol; 111 112 // Only valid on gfx9 113 bits<1> lds = 0; // XXX - What does this actually do? 114 115 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved 116 bits<2> seg = !if(ps.is_flat_global, 0b10, 117 !if(ps.is_flat_scratch, 0b01, 0)); 118 119 // Signed offset. Highest bit ignored for flat and treated as 12-bit 120 // unsigned for flat accesses. 121 bits<13> offset; 122 // GFX90A+ only: instruction uses AccVGPR for data 123 bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); 124 125 // We don't use tfe right now, and it was removed in gfx9. 126 bits<1> tfe = 0; 127 128 // Only valid on GFX9+ 129 let Inst{12-0} = offset; 130 let Inst{13} = !if(ps.has_sve, ps.sve, lds); 131 let Inst{15-14} = seg; 132 133 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); 134 let Inst{17} = cpol{CPolBit.SLC}; 135 let Inst{24-18} = op; 136 let Inst{31-26} = 0x37; // Encoding. 137 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 138 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); 139 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); 140 141 // 54-48 is reserved. 142 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. 143 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); 144} 145 146class GlobalSaddrTable <bit is_saddr, string Name = ""> { 147 bit IsSaddr = is_saddr; 148 string SaddrOp = Name; 149} 150 151// TODO: Is exec allowed for saddr? The disabled value 0x7f is the 152// same encoding value as exec_hi, so it isn't possible to use that if 153// saddr is 32-bit (which isn't handled here yet). 154class FLAT_Load_Pseudo <string opName, RegisterClass regClass, 155 bit HasTiedOutput = 0, 156 bit HasSaddr = 0, bit EnableSaddr = 0, 157 RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< 158 opName, 159 (outs vdata_op:$vdst), 160 !con( 161 !con( 162 !if(EnableSaddr, 163 (ins SReg_64:$saddr, VGPR_32:$vaddr), 164 (ins VReg_64:$vaddr)), 165 (ins flat_offset:$offset)), 166 // FIXME: Operands with default values do not work with following non-optional operands. 167 !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in), 168 (ins CPol_0:$cpol))), 169 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 170 let has_data = 0; 171 let mayLoad = 1; 172 let has_saddr = HasSaddr; 173 let enabled_saddr = EnableSaddr; 174 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 175 let maybeAtomic = 1; 176 177 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 178 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 179} 180 181class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, 182 bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 183 opName, 184 (outs), 185 !con( 186 !if(EnableSaddr, 187 (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64:$saddr), 188 (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), 189 (ins flat_offset:$offset, CPol_0:$cpol)), 190 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { 191 let mayLoad = 0; 192 let mayStore = 1; 193 let has_vdst = 0; 194 let has_saddr = HasSaddr; 195 let enabled_saddr = EnableSaddr; 196 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", ""); 197 let maybeAtomic = 1; 198} 199 200multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { 201 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 202 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, 203 GlobalSaddrTable<0, opName>; 204 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, 205 GlobalSaddrTable<1, opName>; 206 } 207} 208 209class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, 210 bit HasTiedOutput = 0, bit EnableSaddr = 0> : FLAT_Pseudo< 211 opName, 212 (outs regClass:$vdst), 213 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), 214 (ins flat_offset:$offset, CPol_0:$cpol), 215 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), 216 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 217 let is_flat_global = 1; 218 let has_data = 0; 219 let mayLoad = 1; 220 let has_vaddr = 0; 221 let has_saddr = 1; 222 let enabled_saddr = EnableSaddr; 223 let maybeAtomic = 1; 224 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 225 226 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 227 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 228} 229 230multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, 231 bit HasTiedOutput = 0> { 232 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput>, 233 GlobalSaddrTable<0, opName>; 234 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, 1>, 235 GlobalSaddrTable<1, opName>; 236} 237 238multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { 239 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 240 def "" : FLAT_Store_Pseudo<opName, regClass, 1>, 241 GlobalSaddrTable<0, opName>; 242 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, 243 GlobalSaddrTable<1, opName>; 244 } 245} 246 247class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo< 248 opName, 249 (outs ), 250 !con( 251 !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64:$vaddr)), 252 (ins flat_offset:$offset, CPol_0:$cpol)), 253 " $vaddr"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> { 254 let LGKM_CNT = 1; 255 let is_flat_global = 1; 256 let has_data = 0; 257 let has_vdst = 0; 258 let mayLoad = 1; 259 let mayStore = 1; 260 let has_saddr = 1; 261 let enabled_saddr = EnableSaddr; 262 let VALU = 1; 263 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 264 let Uses = [M0, EXEC]; 265 let SchedRW = [WriteVMEM, WriteLDS]; 266} 267 268multiclass FLAT_Global_Load_LDS_Pseudo<string opName> { 269 def "" : FLAT_Global_Load_LDS_Pseudo<opName>, 270 GlobalSaddrTable<0, opName>; 271 def _SADDR : FLAT_Global_Load_LDS_Pseudo<opName, 1>, 272 GlobalSaddrTable<1, opName>; 273} 274 275class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, 276 bit EnableSaddr = 0> : FLAT_Pseudo< 277 opName, 278 (outs), 279 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), 280 (ins flat_offset:$offset, CPol:$cpol)), 281 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 282 let is_flat_global = 1; 283 let mayLoad = 0; 284 let mayStore = 1; 285 let has_vdst = 0; 286 let has_vaddr = 0; 287 let has_saddr = 1; 288 let enabled_saddr = EnableSaddr; 289 let maybeAtomic = 1; 290 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", ""); 291} 292 293multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass> { 294 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass>, 295 GlobalSaddrTable<0, opName>; 296 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, 1>, 297 GlobalSaddrTable<1, opName>; 298} 299 300class FlatScratchInst <string sv_op, string mode> { 301 string SVOp = sv_op; 302 string Mode = mode; 303} 304 305class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, 306 bit HasTiedOutput = 0, 307 bit EnableSaddr = 0, 308 bit EnableSVE = 0, 309 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> 310 : FLAT_Pseudo< 311 opName, 312 (outs getLdStRegisterOperand<regClass>.ret:$vdst), 313 !con( 314 !if(EnableSVE, 315 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 316 !if(EnableSaddr, 317 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), 318 !if(EnableVaddr, 319 (ins VGPR_32:$vaddr, flat_offset:$offset), 320 (ins flat_offset:$offset)))), 321 !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in), 322 (ins CPol_0:$cpol))), 323 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 324 let has_data = 0; 325 let mayLoad = 1; 326 let has_saddr = 1; 327 let enabled_saddr = EnableSaddr; 328 let has_vaddr = EnableVaddr; 329 let has_sve = EnableSVE; 330 let sve = EnableVaddr; 331 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 332 let maybeAtomic = 1; 333 334 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); 335 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); 336} 337 338class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, 339 bit EnableSVE = 0, 340 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr)), 341 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< 342 opName, 343 (outs), 344 !if(EnableSVE, 345 (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 346 !if(EnableSaddr, 347 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), 348 !if(EnableVaddr, 349 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol), 350 (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))), 351 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 352 let mayLoad = 0; 353 let mayStore = 1; 354 let has_vdst = 0; 355 let has_saddr = 1; 356 let enabled_saddr = EnableSaddr; 357 let has_vaddr = EnableVaddr; 358 let has_sve = EnableSVE; 359 let sve = EnableVaddr; 360 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 361 let maybeAtomic = 1; 362} 363 364multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { 365 let is_flat_scratch = 1 in { 366 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, 367 FlatScratchInst<opName, "SV">; 368 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, 369 FlatScratchInst<opName, "SS">; 370 371 let SubtargetPredicate = HasFlatScratchSVSMode in 372 def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>, 373 FlatScratchInst<opName, "SVS">; 374 375 let SubtargetPredicate = HasFlatScratchSTMode in 376 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>, 377 FlatScratchInst<opName, "ST">; 378 } 379} 380 381multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { 382 let is_flat_scratch = 1 in { 383 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, 384 FlatScratchInst<opName, "SV">; 385 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, 386 FlatScratchInst<opName, "SS">; 387 388 let SubtargetPredicate = HasFlatScratchSVSMode in 389 def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>, 390 FlatScratchInst<opName, "SVS">; 391 392 let SubtargetPredicate = HasFlatScratchSTMode in 393 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>, 394 FlatScratchInst<opName, "ST">; 395 } 396} 397 398class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0, 399 bit EnableSVE = 0, 400 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo< 401 opName, 402 (outs ), 403 !if(EnableSVE, 404 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 405 !if(EnableSaddr, 406 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), 407 !if(EnableVaddr, 408 (ins VGPR_32:$vaddr, flat_offset:$offset, CPol:$cpol), 409 (ins flat_offset:$offset, CPol:$cpol)))), 410 " "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { 411 412 let LGKM_CNT = 1; 413 let is_flat_scratch = 1; 414 let has_data = 0; 415 let has_vdst = 0; 416 let mayLoad = 1; 417 let mayStore = 1; 418 let has_saddr = 1; 419 let enabled_saddr = EnableSaddr; 420 let has_vaddr = EnableVaddr; 421 let has_sve = EnableSVE; 422 let sve = EnableVaddr; 423 let VALU = 1; 424 let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"))); 425 let Uses = [M0, EXEC]; 426 let SchedRW = [WriteVMEM, WriteLDS]; 427} 428 429multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> { 430 def "" : FLAT_Scratch_Load_LDS_Pseudo<opName>, 431 FlatScratchInst<opName, "SV">; 432 def _SADDR : FLAT_Scratch_Load_LDS_Pseudo<opName, 1>, 433 FlatScratchInst<opName, "SS">; 434 def _SVS : FLAT_Scratch_Load_LDS_Pseudo<opName, 1, 1>, 435 FlatScratchInst<opName, "SVS">; 436 def _ST : FLAT_Scratch_Load_LDS_Pseudo<opName, 0, 0, 0>, 437 FlatScratchInst<opName, "ST">; 438} 439 440class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, 441 string asm, list<dag> pattern = []> : 442 FLAT_Pseudo<opName, outs, ins, asm, pattern> { 443 let mayLoad = 1; 444 let mayStore = 1; 445 let has_glc = 0; 446 let glcValue = 0; 447 let has_vdst = 0; 448 let has_sccb = 1; 449 let sccbValue = 0; 450 let maybeAtomic = 1; 451 let IsAtomicNoRet = 1; 452} 453 454class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, 455 string asm, list<dag> pattern = []> 456 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { 457 let hasPostISelHook = 1; 458 let has_vdst = 1; 459 let glcValue = 1; 460 let sccbValue = 0; 461 let IsAtomicNoRet = 0; 462 let IsAtomicRet = 1; 463 let PseudoInstr = NAME # "_RTN"; 464} 465 466multiclass FLAT_Atomic_Pseudo< 467 string opName, 468 RegisterClass vdst_rc, 469 ValueType vt, 470 ValueType data_vt = vt, 471 RegisterClass data_rc = vdst_rc, 472 bit isFP = isFloatType<data_vt>.ret, 473 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 474 def "" : FLAT_AtomicNoRet_Pseudo <opName, 475 (outs), 476 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 477 " $vaddr, $vdata$offset$cpol">, 478 GlobalSaddrTable<0, opName>, 479 AtomicNoRet <opName, 0> { 480 let PseudoInstr = NAME; 481 let FPAtomic = isFP; 482 let AddedComplexity = -1; // Prefer global atomics if available 483 } 484 485 def _RTN : FLAT_AtomicRet_Pseudo <opName, 486 (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), 487 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 488 " $vdst, $vaddr, $vdata$offset$cpol">, 489 GlobalSaddrTable<0, opName#"_rtn">, 490 AtomicNoRet <opName, 1> { 491 let FPAtomic = isFP; 492 let AddedComplexity = -1; // Prefer global atomics if available 493 } 494} 495 496multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< 497 string opName, 498 RegisterClass vdst_rc, 499 ValueType vt, 500 ValueType data_vt = vt, 501 RegisterClass data_rc = vdst_rc, 502 bit isFP = isFloatType<data_vt>.ret, 503 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { 504 505 def "" : FLAT_AtomicNoRet_Pseudo <opName, 506 (outs), 507 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), 508 " $vaddr, $vdata, off$offset$cpol">, 509 GlobalSaddrTable<0, opName>, 510 AtomicNoRet <opName, 0> { 511 let has_saddr = 1; 512 let PseudoInstr = NAME; 513 let FPAtomic = isFP; 514 } 515 516 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, 517 (outs), 518 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_0:$cpol), 519 " $vaddr, $vdata, $saddr$offset$cpol">, 520 GlobalSaddrTable<1, opName>, 521 AtomicNoRet <opName#"_saddr", 0> { 522 let has_saddr = 1; 523 let enabled_saddr = 1; 524 let PseudoInstr = NAME#"_SADDR"; 525 let FPAtomic = isFP; 526 } 527} 528 529multiclass FLAT_Global_Atomic_Pseudo_RTN< 530 string opName, 531 RegisterClass vdst_rc, 532 ValueType vt, 533 ValueType data_vt = vt, 534 RegisterClass data_rc = vdst_rc, 535 bit isFP = isFloatType<data_vt>.ret, 536 RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, 537 RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { 538 539 def _RTN : FLAT_AtomicRet_Pseudo <opName, 540 (outs vdst_op:$vdst), 541 (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), 542 " $vdst, $vaddr, $vdata, off$offset$cpol">, 543 GlobalSaddrTable<0, opName#"_rtn">, 544 AtomicNoRet <opName, 1> { 545 let has_saddr = 1; 546 let FPAtomic = isFP; 547 } 548 549 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, 550 (outs vdst_op:$vdst), 551 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_GLC1:$cpol), 552 " $vdst, $vaddr, $vdata, $saddr$offset$cpol">, 553 GlobalSaddrTable<1, opName#"_rtn">, 554 AtomicNoRet <opName#"_saddr", 1> { 555 let has_saddr = 1; 556 let enabled_saddr = 1; 557 let PseudoInstr = NAME#"_SADDR_RTN"; 558 let FPAtomic = isFP; 559 } 560} 561 562multiclass FLAT_Global_Atomic_Pseudo< 563 string opName, 564 RegisterClass vdst_rc, 565 ValueType vt, 566 ValueType data_vt = vt, 567 RegisterClass data_rc = vdst_rc> { 568 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in { 569 defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc>; 570 defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, data_vt, data_rc>; 571 } 572} 573 574//===----------------------------------------------------------------------===// 575// Flat Instructions 576//===----------------------------------------------------------------------===// 577 578def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; 579def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; 580def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; 581def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; 582def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; 583def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; 584def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; 585def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; 586 587def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>; 588def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>; 589def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; 590def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; 591def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; 592def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; 593 594let SubtargetPredicate = HasD16LoadStore in { 595def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; 596def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; 597def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; 598def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; 599def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; 600def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; 601 602def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; 603def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; 604} 605 606defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", 607 VGPR_32, i32, v2i32, VReg_64>; 608 609defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", 610 VReg_64, i64, v2i64, VReg_128>; 611 612defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", 613 VGPR_32, i32>; 614 615defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", 616 VReg_64, i64>; 617 618defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", 619 VGPR_32, i32>; 620 621defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", 622 VGPR_32, i32>; 623 624defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", 625 VGPR_32, i32>; 626 627defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", 628 VGPR_32, i32>; 629 630defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", 631 VGPR_32, i32>; 632 633defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", 634 VGPR_32, i32>; 635 636defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", 637 VGPR_32, i32>; 638 639defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", 640 VGPR_32, i32>; 641 642defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", 643 VGPR_32, i32>; 644 645defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", 646 VGPR_32, i32>; 647 648defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", 649 VGPR_32, i32>; 650 651defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", 652 VReg_64, i64>; 653 654defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", 655 VReg_64, i64>; 656 657defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", 658 VReg_64, i64>; 659 660defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", 661 VReg_64, i64>; 662 663defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", 664 VReg_64, i64>; 665 666defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", 667 VReg_64, i64>; 668 669defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", 670 VReg_64, i64>; 671 672defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", 673 VReg_64, i64>; 674 675defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", 676 VReg_64, i64>; 677 678defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", 679 VReg_64, i64>; 680 681defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", 682 VReg_64, i64>; 683 684// GFX7-, GFX10-only flat instructions. 685let SubtargetPredicate = isGFX7GFX10 in { 686 687defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", 688 VGPR_32, f32, v2f32, VReg_64>; 689 690defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", 691 VReg_64, f64, v2f64, VReg_128>; 692 693defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", 694 VGPR_32, f32>; 695 696defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", 697 VGPR_32, f32>; 698 699defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2", 700 VReg_64, f64>; 701 702defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2", 703 VReg_64, f64>; 704 705} // End SubtargetPredicate = isGFX7GFX10 706 707let SubtargetPredicate = isGFX90APlus in { 708 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64>; 709 defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo<"flat_atomic_min_f64", VReg_64, f64>; 710 defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo<"flat_atomic_max_f64", VReg_64, f64>; 711 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64>; 712 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64>; 713 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64>; 714} // End SubtargetPredicate = isGFX90APlus 715 716let SubtargetPredicate = isGFX940Plus in { 717 defm FLAT_ATOMIC_ADD_F32 : FLAT_Atomic_Pseudo<"flat_atomic_add_f32", VGPR_32, f32>; 718 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_f16", VGPR_32, v2f16>; 719 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_bf16", VGPR_32, v2f16>; 720 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Atomic_Pseudo<"global_atomic_pk_add_bf16", VGPR_32, v2f16>; 721} // End SubtargetPredicate = isGFX940Plus 722 723defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; 724defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; 725defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; 726defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; 727defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; 728defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; 729defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; 730defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; 731 732defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; 733defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; 734defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; 735defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; 736defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; 737defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; 738let OtherPredicates = [HasGFX10_BEncoding] in 739defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; 740 741defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; 742defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; 743defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; 744defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; 745defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; 746defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; 747let OtherPredicates = [HasGFX10_BEncoding] in 748defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; 749 750defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; 751defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; 752 753let is_flat_global = 1 in { 754defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", 755 VGPR_32, i32, v2i32, VReg_64>; 756 757defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", 758 VReg_64, i64, v2i64, VReg_128>; 759 760defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", 761 VGPR_32, i32>; 762 763defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", 764 VReg_64, i64>; 765 766defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", 767 VGPR_32, i32>; 768 769defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", 770 VGPR_32, i32>; 771 772defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", 773 VGPR_32, i32>; 774 775defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", 776 VGPR_32, i32>; 777 778defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", 779 VGPR_32, i32>; 780 781defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", 782 VGPR_32, i32>; 783 784defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", 785 VGPR_32, i32>; 786 787defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", 788 VGPR_32, i32>; 789 790defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", 791 VGPR_32, i32>; 792 793defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", 794 VGPR_32, i32>; 795 796defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", 797 VGPR_32, i32>; 798 799defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", 800 VReg_64, i64>; 801 802defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", 803 VReg_64, i64>; 804 805defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", 806 VReg_64, i64>; 807 808defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", 809 VReg_64, i64>; 810 811defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", 812 VReg_64, i64>; 813 814defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", 815 VReg_64, i64>; 816 817defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", 818 VReg_64, i64>; 819 820defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", 821 VReg_64, i64>; 822 823defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", 824 VReg_64, i64>; 825 826defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", 827 VReg_64, i64>; 828 829defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", 830 VReg_64, i64>; 831 832let SubtargetPredicate = HasGFX10_BEncoding in 833defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub", 834 VGPR_32, i32>; 835 836let SubtargetPredicate = isGFX940Plus in { 837 838defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">; 839defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">; 840defm GLOBAL_LOAD_LDS_USHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ushort">; 841defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sshort">; 842defm GLOBAL_LOAD_LDS_DWORD : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dword">; 843 844} // End let SubtargetPredicate = isGFX940Plus 845} // End is_flat_global = 1 846 847 848 849let SubtargetPredicate = HasFlatScratchInsts in { 850defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; 851defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; 852defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; 853defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; 854defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; 855defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; 856defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; 857defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; 858 859defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32, 1>; 860defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; 861defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32, 1>; 862defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; 863defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32, 1>; 864defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; 865 866defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; 867defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; 868defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; 869defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; 870defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; 871defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; 872 873defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; 874defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; 875 876let SubtargetPredicate = isGFX940Plus in { 877 878defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ubyte">; 879defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sbyte">; 880defm SCRATCH_LOAD_LDS_USHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ushort">; 881defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sshort">; 882defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_dword">; 883 884} // End let SubtargetPredicate = isGFX940Plus 885 886} // End SubtargetPredicate = HasFlatScratchInsts 887 888let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { 889 defm GLOBAL_ATOMIC_FCMPSWAP : 890 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32, v2f32, VReg_64>; 891 defm GLOBAL_ATOMIC_FMIN : 892 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; 893 defm GLOBAL_ATOMIC_FMAX : 894 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; 895 defm GLOBAL_ATOMIC_FCMPSWAP_X2 : 896 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64, v2f64, VReg_128>; 897 defm GLOBAL_ATOMIC_FMIN_X2 : 898 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; 899 defm GLOBAL_ATOMIC_FMAX_X2 : 900 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; 901} // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 902 903let is_flat_global = 1 in { 904let OtherPredicates = [HasAtomicFaddInsts] in { 905 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < 906 "global_atomic_add_f32", VGPR_32, f32 907 >; 908 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < 909 "global_atomic_pk_add_f16", VGPR_32, v2f16 910 >; 911} // End OtherPredicates = [HasAtomicFaddInsts] 912 913let OtherPredicates = [isGFX90APlus] in { 914 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < 915 "global_atomic_add_f32", VGPR_32, f32 916 >; 917 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < 918 "global_atomic_pk_add_f16", VGPR_32, v2f16 919 >; 920} // End OtherPredicates = [isGFX90APlus] 921} // End is_flat_global = 1 922 923//===----------------------------------------------------------------------===// 924// Flat Patterns 925//===----------------------------------------------------------------------===// 926 927// Patterns for global loads with no offset. 928class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 929 (vt (node (FlatOffset i64:$vaddr, i16:$offset))), 930 (inst $vaddr, $offset) 931>; 932 933class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 934 (node (FlatOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 935 (inst $vaddr, $offset, 0, $in) 936>; 937 938class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 939 (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset), vt:$in), 940 (inst $vaddr, $offset, 0, $in) 941>; 942 943class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 944 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$in)), 945 (inst $saddr, $voffset, $offset, 0, $in) 946>; 947 948class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 949 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i16:$offset))), 950 (inst $vaddr, $offset) 951>; 952 953class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 954 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset))), 955 (inst $saddr, $voffset, $offset, 0) 956>; 957 958class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 959 ValueType vt> : GCNPat < 960 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset)), 961 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 962>; 963 964class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 965 ValueType vt> : GCNPat < 966 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 967 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 968>; 969 970class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 971 ValueType vt, ValueType data_vt = vt> : GCNPat < 972 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), data_vt:$data)), 973 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) 974>; 975 976class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 977 ValueType vt> : GCNPat < 978 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i16:$offset), vt:$data), 979 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 980>; 981 982class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 983 (node vt:$data, (FlatOffset i64:$vaddr, i16:$offset)), 984 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 985>; 986 987class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 988 (node vt:$data, (GlobalOffset i64:$vaddr, i16:$offset)), 989 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 990>; 991 992class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 993 // atomic store follows atomic binop convention so the address comes 994 // first. 995 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 996 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 997>; 998 999class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, 1000 ValueType vt, ValueType data_vt = vt> : GCNPat < 1001 // atomic store follows atomic binop convention so the address comes 1002 // first. 1003 (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data), 1004 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1005>; 1006 1007class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1008 (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), 1009 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1010>; 1011 1012multiclass FlatAtomicPat <string inst, string node, ValueType vt, 1013 ValueType data_vt = vt> { 1014 defvar rtnNode = !cast<PatFrags>(node#"_ret_"#vt.Size); 1015 defvar noRtnNode = !cast<PatFrags>(node#"_noret_"#vt.Size); 1016 1017 def : GCNPat <(vt (rtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1018 (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1019 1020 def : GCNPat <(vt (noRtnNode (FlatOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1021 (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1022} 1023 1024multiclass FlatSignedAtomicPat <string inst, string node, ValueType vt, 1025 ValueType data_vt = vt, bit isIntr = 0> { 1026 defvar rtnNode = !cast<PatFrags>(node # "_ret" # !if(isIntr, "", "_" # vt.Size)); 1027 defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt.Size)); 1028 1029 def : GCNPat <(vt (rtnNode (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1030 (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1031 1032 def : GCNPat <(vt (noRtnNode (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1033 (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; 1034} 1035 1036multiclass FlatSignedAtomicIntrPat <string inst, string node, ValueType vt, 1037 ValueType data_vt = vt> { 1038 defm : FlatSignedAtomicPat<inst, node, vt, data_vt, /* isIntr */ 1>; 1039} 1040 1041class FlatSignedAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1042 (node (GlobalOffset i64:$vaddr, i16:$offset), vt:$data), 1043 (inst VReg_64:$vaddr, getVregSrcForVT<vt>.ret:$data, $offset) 1044>; 1045 1046class FlatSignedAtomicPatRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt, 1047 ValueType data_vt = vt> : GCNPat < 1048 (vt (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data)), 1049 (inst VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) 1050>; 1051 1052class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1053 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset))), 1054 (inst $vaddr, $offset) 1055>; 1056 1057class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1058 (node (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset), vt:$in), 1059 (inst $vaddr, $offset, 0, $in) 1060>; 1061 1062class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1063 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i16:$offset)), 1064 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) 1065>; 1066 1067class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1068 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset))), 1069 (inst $saddr, $offset) 1070>; 1071 1072class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1073 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1074 (inst $saddr, $offset, 0, $in) 1075>; 1076 1077class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1078 ValueType vt> : GCNPat < 1079 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)), 1080 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) 1081>; 1082 1083class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1084 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset))), 1085 (inst $vaddr, $saddr, $offset, 0) 1086>; 1087 1088class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, 1089 ValueType vt> : GCNPat < 1090 (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset)), 1091 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset) 1092>; 1093 1094class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < 1095 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset), vt:$in)), 1096 (inst $vaddr, $saddr, $offset, 0, $in) 1097>; 1098 1099let OtherPredicates = [HasFlatAddressSpace] in { 1100 1101def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i32>; 1102def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i16>; 1103def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i32>; 1104def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i16>; 1105def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; 1106def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; 1107def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; 1108def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; 1109def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; 1110def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; 1111def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; 1112def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; 1113def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; 1114def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; 1115def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; 1116 1117def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; 1118def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; 1119 1120def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; 1121def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; 1122 1123foreach vt = Reg32Types.types in { 1124def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; 1125def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; 1126} 1127 1128foreach vt = VReg_64.RegTypes in { 1129def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; 1130def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; 1131} 1132 1133def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; 1134 1135foreach vt = VReg_128.RegTypes in { 1136def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; 1137def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; 1138} 1139 1140def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>; 1141def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>; 1142def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>; 1143def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>; 1144def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>; 1145def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; 1146 1147foreach as = [ "flat", "global" ] in { 1148defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; 1149defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>; 1150defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_inc_"#as, i32>; 1151defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_dec_"#as, i32>; 1152defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>; 1153defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>; 1154defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>; 1155defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN", "atomic_load_min_"#as, i32>; 1156defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN", "atomic_load_umin_"#as, i32>; 1157defm : FlatAtomicPat <"FLAT_ATOMIC_OR", "atomic_load_or_"#as, i32>; 1158defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP", "atomic_swap_"#as, i32>; 1159defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_"#as, i32, v2i32>; 1160defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>; 1161 1162defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>; 1163defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>; 1164defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_inc_"#as, i64>; 1165defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_dec_"#as, i64>; 1166defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>; 1167defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>; 1168defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>; 1169defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN_X2", "atomic_load_min_"#as, i64>; 1170defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN_X2", "atomic_load_umin_"#as, i64>; 1171defm : FlatAtomicPat <"FLAT_ATOMIC_OR_X2", "atomic_load_or_"#as, i64>; 1172defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP_X2", "atomic_swap_"#as, i64>; 1173defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_"#as, i64, v2i64>; 1174defm : FlatAtomicPat <"FLAT_ATOMIC_XOR_X2", "atomic_load_xor_"#as, i64>; 1175} // end foreach as 1176 1177def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; 1178def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; 1179 1180let OtherPredicates = [HasD16LoadStore] in { 1181def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; 1182def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; 1183} 1184 1185let OtherPredicates = [D16PreservesUnusedBits] in { 1186def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; 1187def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; 1188def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; 1189def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; 1190def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; 1191def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; 1192 1193def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; 1194def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; 1195def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; 1196def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; 1197def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; 1198def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; 1199} 1200 1201} // End OtherPredicates = [HasFlatAddressSpace] 1202 1203 1204multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1205 def : FlatLoadSignedPat <inst, node, vt> { 1206 let AddedComplexity = 10; 1207 } 1208 1209 def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1210 let AddedComplexity = 11; 1211 } 1212} 1213 1214multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1215 def : FlatSignedLoadPat_D16 <inst, node, vt> { 1216 let AddedComplexity = 10; 1217 } 1218 1219 def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1220 let AddedComplexity = 11; 1221 } 1222} 1223 1224multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1225 ValueType vt> { 1226 def : FlatStoreSignedPat <inst, node, vt> { 1227 let AddedComplexity = 10; 1228 } 1229 1230 def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1231 let AddedComplexity = 11; 1232 } 1233} 1234 1235// Deal with swapped operands for atomic_store vs. regular store 1236multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1237 def : FlatStoreSignedAtomicPat <inst, node, vt> { 1238 let AddedComplexity = 10; 1239 } 1240 1241 def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1242 let AddedComplexity = 11; 1243 } 1244} 1245 1246multiclass GlobalFLATAtomicPatsRtn<string nortn_inst_name, SDPatternOperator node, 1247 ValueType vt, ValueType data_vt = vt> { 1248 def : FlatSignedAtomicPatRtn <!cast<FLAT_Pseudo>(nortn_inst_name#"_RTN"), node, vt, data_vt> { 1249 let AddedComplexity = 10; 1250 } 1251 1252 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(nortn_inst_name#"_SADDR_RTN"), node, vt, data_vt> { 1253 let AddedComplexity = 11; 1254 } 1255} 1256 1257multiclass GlobalFLATAtomicPats<string inst, string node, ValueType vt, 1258 ValueType data_vt = vt, bit isIntr = 0> { 1259 defvar rtnNode = !cast<PatFrags>(node # "_ret" # !if(isIntr, "", "_" # vt.Size)); 1260 defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt.Size)); 1261 1262 let AddedComplexity = 10 in { 1263 defm : FlatSignedAtomicPat <inst, node, vt, data_vt, isIntr>; 1264 } 1265 1266 let AddedComplexity = 11 in { 1267 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR"), noRtnNode, vt, data_vt>; 1268 def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_RTN"), rtnNode, vt, data_vt>; 1269 } 1270} 1271 1272multiclass GlobalFLATAtomicIntrPats<string inst, string node, ValueType vt, 1273 ValueType data_vt = vt> { 1274 defm : GlobalFLATAtomicPats<inst, node, vt, data_vt, /* isIntr */ 1>; 1275} 1276 1277multiclass GlobalFLATNoRtnAtomicPats<FLAT_Pseudo inst, SDPatternOperator node, 1278 ValueType vt> { 1279 def : FlatSignedAtomicPatNoRtn <inst, node, vt> { 1280 let AddedComplexity = 10; 1281 } 1282 1283 def : GlobalAtomicNoRtnSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1284 let AddedComplexity = 11; 1285 } 1286} 1287 1288multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1289 def : ScratchLoadSignedPat <inst, node, vt> { 1290 let AddedComplexity = 25; 1291 } 1292 1293 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1294 let AddedComplexity = 26; 1295 } 1296 1297 def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1298 let SubtargetPredicate = HasFlatScratchSVSMode; 1299 let AddedComplexity = 27; 1300 } 1301} 1302 1303multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, 1304 ValueType vt> { 1305 def : ScratchStoreSignedPat <inst, node, vt> { 1306 let AddedComplexity = 25; 1307 } 1308 1309 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1310 let AddedComplexity = 26; 1311 } 1312 1313 def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1314 let SubtargetPredicate = HasFlatScratchSVSMode; 1315 let AddedComplexity = 27; 1316 } 1317} 1318 1319multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { 1320 def : ScratchLoadSignedPat_D16 <inst, node, vt> { 1321 let AddedComplexity = 25; 1322 } 1323 1324 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { 1325 let AddedComplexity = 26; 1326 } 1327 1328 def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { 1329 let SubtargetPredicate = HasFlatScratchSVSMode; 1330 let AddedComplexity = 27; 1331 } 1332} 1333 1334let OtherPredicates = [HasFlatGlobalInsts] in { 1335 1336defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i32>; 1337defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i16>; 1338defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i32>; 1339defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i16>; 1340defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; 1341defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; 1342defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; 1343defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; 1344defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; 1345defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; 1346defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; 1347defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; 1348defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; 1349defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; 1350 1351foreach vt = Reg32Types.types in { 1352defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; 1353defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; 1354} 1355 1356foreach vt = VReg_64.RegTypes in { 1357defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; 1358defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; 1359} 1360 1361defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; 1362 1363foreach vt = VReg_128.RegTypes in { 1364defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; 1365defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; 1366} 1367 1368// There is no distinction for atomic load lowering during selection; 1369// the memory legalizer will set the cache bits and insert the 1370// appropriate waits. 1371defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; 1372defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; 1373 1374defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; 1375defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; 1376defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; 1377defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; 1378defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; 1379 1380let OtherPredicates = [HasD16LoadStore] in { 1381defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; 1382defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; 1383} 1384 1385let OtherPredicates = [D16PreservesUnusedBits] in { 1386defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; 1387defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; 1388defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; 1389defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; 1390defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; 1391defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; 1392 1393defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; 1394defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; 1395defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; 1396defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; 1397defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; 1398defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; 1399} 1400 1401defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>; 1402defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>; 1403defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>; 1404defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>; 1405defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>; 1406defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>; 1407 1408defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; 1409defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; 1410defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_inc_global", i32>; 1411defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_dec_global", i32>; 1412defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>; 1413defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>; 1414defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>; 1415defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", "atomic_load_min_global", i32>; 1416defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", "atomic_load_umin_global", i32>; 1417defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", "atomic_load_or_global", i32>; 1418defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", "atomic_swap_global", i32>; 1419defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_global", i32, v2i32>; 1420defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>; 1421defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", int_amdgcn_global_atomic_csub, i32>; 1422 1423defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>; 1424defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>; 1425defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_inc_global", i64>; 1426defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_dec_global", i64>; 1427defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>; 1428defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>; 1429defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>; 1430defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", "atomic_load_min_global", i64>; 1431defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", "atomic_load_umin_global", i64>; 1432defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", "atomic_load_or_global", i64>; 1433defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", "atomic_swap_global", i64>; 1434defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_global", i64, v2i64>; 1435defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", "atomic_load_xor_global", i64>; 1436 1437let OtherPredicates = [isGFX10Plus] in { 1438defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>; 1439defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>; 1440defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN_X2", "atomic_load_fmin_global", f64>; 1441defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX_X2", "atomic_load_fmax_global", f64>; 1442defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin", f32>; 1443defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax", f32>; 1444defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN_X2", "int_amdgcn_global_atomic_fmin", f64>; 1445defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX_X2", "int_amdgcn_global_atomic_fmax", f64>; 1446} 1447 1448let OtherPredicates = [HasAtomicFaddInsts] in { 1449defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_ADD_F32, atomic_load_fadd_global_noret_32, f32>; 1450defm : GlobalFLATNoRtnAtomicPats <GLOBAL_ATOMIC_PK_ADD_F16, atomic_load_fadd_v2f16_global_noret_32, v2f16>; 1451} 1452 1453let OtherPredicates = [isGFX90APlus] in { 1454defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>; 1455defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_v2f16_global", v2f16>; 1456defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", "atomic_load_fadd_global", f64>; 1457defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global", f64>; 1458defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>; 1459defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_ADD_F32", "int_amdgcn_global_atomic_fadd", f32>; 1460defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_ADD_F64", "int_amdgcn_global_atomic_fadd", f64>; 1461defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_PK_ADD_F16", "int_amdgcn_global_atomic_fadd", v2f16>; 1462defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MIN_F64", "int_amdgcn_global_atomic_fmin", f64>; 1463defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MAX_F64", "int_amdgcn_global_atomic_fmax", f64>; 1464defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>; 1465defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>; 1466defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>; 1467defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_ADD_F64", "int_amdgcn_flat_atomic_fadd", f64>; 1468defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>; 1469defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>; 1470} 1471 1472let OtherPredicates = [isGFX940Plus] in { 1473defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>; 1474defm : FlatSignedAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_v2f16_flat", v2f16>; 1475defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_ADD_F32", "int_amdgcn_flat_atomic_fadd", f32>; 1476defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_F16", "int_amdgcn_flat_atomic_fadd", v2f16>; 1477defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_BF16", "int_amdgcn_flat_atomic_fadd_v2bf16", v2i16>; 1478defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "int_amdgcn_global_atomic_fadd_v2bf16", v2i16>; 1479} 1480 1481} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 1482 1483let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { 1484 1485defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; 1486defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; 1487defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; 1488defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; 1489defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; 1490defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; 1491defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; 1492defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; 1493defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; 1494defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; 1495 1496foreach vt = Reg32Types.types in { 1497defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; 1498defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; 1499} 1500 1501foreach vt = VReg_64.RegTypes in { 1502defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; 1503defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; 1504} 1505 1506defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; 1507 1508foreach vt = VReg_128.RegTypes in { 1509defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; 1510defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; 1511} 1512 1513defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; 1514defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; 1515defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; 1516defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; 1517defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; 1518 1519let OtherPredicates = [HasD16LoadStore, HasFlatScratchInsts, EnableFlatScratch] in { 1520defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; 1521defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; 1522} 1523 1524let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { 1525defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; 1526defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; 1527defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; 1528defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; 1529defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; 1530defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; 1531 1532defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; 1533defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; 1534defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; 1535defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; 1536defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; 1537defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; 1538} 1539 1540} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] 1541 1542//===----------------------------------------------------------------------===// 1543// Target 1544//===----------------------------------------------------------------------===// 1545 1546//===----------------------------------------------------------------------===// 1547// CI 1548//===----------------------------------------------------------------------===// 1549 1550class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> : 1551 FLAT_Real <op, ps>, 1552 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { 1553 let AssemblerPredicate = isGFX7Only; 1554 let DecoderNamespace="GFX7"; 1555} 1556 1557def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; 1558def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; 1559def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; 1560def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; 1561def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; 1562def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; 1563def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; 1564def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; 1565 1566def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; 1567def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; 1568def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; 1569def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; 1570def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; 1571def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; 1572 1573multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> { 1574 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1575 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1576} 1577 1578defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; 1579defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; 1580defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; 1581defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; 1582defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; 1583defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; 1584defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; 1585defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; 1586defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; 1587defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; 1588defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; 1589defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; 1590defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; 1591defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; 1592defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; 1593defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; 1594defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; 1595defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; 1596defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; 1597defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; 1598defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; 1599defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; 1600defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; 1601defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; 1602defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; 1603defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; 1604 1605// CI Only flat instructions 1606defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; 1607defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; 1608defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; 1609defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; 1610defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; 1611defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; 1612 1613 1614//===----------------------------------------------------------------------===// 1615// VI 1616//===----------------------------------------------------------------------===// 1617 1618class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> : 1619 FLAT_Real <op, ps>, 1620 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { 1621 let AssemblerPredicate = isGFX8GFX9; 1622 let DecoderNamespace = "GFX8"; 1623 1624 let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1625 let AsmString = ps.Mnemonic # 1626 !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands); 1627} 1628 1629multiclass FLAT_Real_AllAddr_vi<bits<7> op, 1630 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1631 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>; 1632 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>; 1633} 1634 1635class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> : 1636 FLAT_Real <op, ps>, 1637 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> { 1638 let AssemblerPredicate = isGFX940Plus; 1639 let DecoderNamespace = "GFX9"; 1640 let Inst{13} = ps.sve; 1641 let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); 1642} 1643 1644multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> { 1645 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> { 1646 let AssemblerPredicate = isGFX8GFX9NotGFX940; 1647 let OtherPredicates = [isGFX8GFX9NotGFX940]; 1648 } 1649 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> { 1650 let DecoderNamespace = "GFX9"; 1651 } 1652 let AssemblerPredicate = isGFX940Plus, SubtargetPredicate = isGFX940Plus in { 1653 def _VE_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1654 def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>; 1655 def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; 1656 } 1657} 1658 1659def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; 1660def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; 1661def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; 1662def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; 1663def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; 1664def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; 1665def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; 1666def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; 1667 1668def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; 1669def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; 1670def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; 1671def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; 1672def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; 1673def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; 1674def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; 1675def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; 1676 1677def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; 1678def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; 1679def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; 1680def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; 1681def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; 1682def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; 1683 1684multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps, 1685 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { 1686 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>; 1687 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>; 1688} 1689 1690multiclass FLAT_Global_Real_Atomics_vi<bits<7> op, 1691 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> : 1692 FLAT_Real_AllAddr_vi<op, has_sccb> { 1693 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>; 1694 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>; 1695} 1696 1697 1698defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; 1699defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; 1700defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; 1701defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; 1702defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; 1703defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; 1704defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; 1705defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; 1706defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; 1707defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; 1708defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; 1709defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; 1710defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; 1711defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; 1712defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; 1713defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; 1714defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; 1715defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; 1716defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; 1717defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; 1718defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; 1719defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; 1720defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; 1721defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; 1722defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; 1723defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; 1724 1725defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; 1726defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; 1727defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; 1728defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; 1729defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; 1730defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; 1731defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; 1732defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; 1733 1734defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; 1735defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; 1736defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; 1737defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; 1738defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; 1739defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; 1740 1741defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; 1742defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; 1743defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; 1744defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; 1745defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; 1746defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; 1747defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; 1748defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; 1749 1750let AssemblerPredicate = isGFX940Plus in { 1751defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_vi <0x026>; 1752defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_vi <0x027>; 1753defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_vi <0x028>; 1754defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_vi <0x029>; 1755defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_vi <0x02a>; 1756} // End let AssemblerPredicate = isGFX940Plus 1757 1758defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; 1759defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; 1760defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; 1761defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; 1762defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; 1763defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; 1764defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; 1765defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; 1766defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; 1767defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; 1768defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; 1769defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; 1770defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; 1771defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; 1772defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; 1773defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; 1774defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; 1775defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; 1776defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; 1777defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; 1778defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; 1779defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; 1780defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; 1781defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; 1782defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; 1783defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; 1784 1785let AssemblerPredicate = isGFX940Plus in { 1786defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x026>; 1787defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x027>; 1788defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_AllAddr_SVE_vi <0x028>; 1789defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x029>; 1790defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_AllAddr_SVE_vi <0x02a>; 1791} // End let AssemblerPredicate = isGFX940Plus 1792 1793defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x10>; 1794defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x11>; 1795defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_SVE_vi <0x12>; 1796defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x13>; 1797defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_SVE_vi <0x14>; 1798defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x15>; 1799defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x16>; 1800defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x17>; 1801defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_SVE_vi <0x18>; 1802defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x19>; 1803defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x20>; 1804defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x21>; 1805defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x22>; 1806defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x23>; 1807defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_SVE_vi <0x24>; 1808defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x25>; 1809defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_SVE_vi <0x1a>; 1810defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>; 1811defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_SVE_vi <0x1c>; 1812defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x1d>; 1813defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x1e>; 1814defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x1f>; 1815 1816let SubtargetPredicate = isGFX8GFX9NotGFX940 in { 1817 // These instructions are encoded differently on gfx90* and gfx940. 1818 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>; 1819 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>; 1820} 1821 1822let SubtargetPredicate = isGFX90AOnly in { 1823 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64, 0>; 1824 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64, 0>; 1825 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64, 0>; 1826 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; 1827 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; 1828 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; 1829} // End SubtargetPredicate = isGFX90AOnly 1830 1831multiclass FLAT_Real_AllAddr_gfx940<bits<7> op> { 1832 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; 1833 def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1834} 1835 1836multiclass FLAT_Real_Atomics_gfx940 <bits<7> op, FLAT_Pseudo ps> { 1837 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; 1838 def _RTN_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; 1839} 1840 1841multiclass FLAT_Global_Real_Atomics_gfx940<bits<7> op> : 1842 FLAT_Real_AllAddr_gfx940<op> { 1843 def _RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1844 def _SADDR_RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1845} 1846 1847let SubtargetPredicate = isGFX940Plus in { 1848 // These instructions are encoded differently on gfx90* and gfx940. 1849 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>; 1850 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>; 1851 1852 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f, FLAT_ATOMIC_ADD_F64>; 1853 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50, FLAT_ATOMIC_MIN_F64>; 1854 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51, FLAT_ATOMIC_MAX_F64>; 1855 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>; 1856 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>; 1857 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>; 1858 defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d, FLAT_ATOMIC_ADD_F32>; 1859 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e, FLAT_ATOMIC_PK_ADD_F16>; 1860 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52, FLAT_ATOMIC_PK_ADD_BF16>; 1861 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>; 1862} // End SubtargetPredicate = isGFX940Plus 1863 1864//===----------------------------------------------------------------------===// 1865// GFX10. 1866//===----------------------------------------------------------------------===// 1867 1868class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps> : 1869 FLAT_Real<op, ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { 1870 let AssemblerPredicate = isGFX10Plus; 1871 let DecoderNamespace = "GFX10"; 1872 1873 let Inst{11-0} = offset{11-0}; 1874 let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); 1875 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); 1876 let Inst{55} = 0; 1877} 1878 1879 1880multiclass FLAT_Real_Base_gfx10<bits<7> op> { 1881 def _gfx10 : 1882 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME)>; 1883} 1884 1885multiclass FLAT_Real_RTN_gfx10<bits<7> op> { 1886 def _RTN_gfx10 : 1887 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; 1888} 1889 1890multiclass FLAT_Real_SADDR_gfx10<bits<7> op> { 1891 def _SADDR_gfx10 : 1892 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; 1893} 1894 1895multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op> { 1896 def _SADDR_RTN_gfx10 : 1897 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; 1898} 1899 1900multiclass FLAT_Real_ST_gfx10<bits<7> op> { 1901 def _ST_gfx10 : 1902 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")> { 1903 let Inst{54-48} = !cast<int>(EXEC_HI.HWEncoding); 1904 let OtherPredicates = [HasFlatScratchSTMode]; 1905 } 1906} 1907 1908multiclass FLAT_Real_AllAddr_gfx10<bits<7> op> : 1909 FLAT_Real_Base_gfx10<op>, 1910 FLAT_Real_SADDR_gfx10<op>; 1911 1912multiclass FLAT_Real_Atomics_gfx10<bits<7> op> : 1913 FLAT_Real_Base_gfx10<op>, 1914 FLAT_Real_RTN_gfx10<op>; 1915 1916multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op> : 1917 FLAT_Real_AllAddr_gfx10<op>, 1918 FLAT_Real_RTN_gfx10<op>, 1919 FLAT_Real_SADDR_RTN_gfx10<op>; 1920 1921multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op> : 1922 FLAT_Real_RTN_gfx10<op>, 1923 FLAT_Real_SADDR_RTN_gfx10<op>; 1924 1925multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : 1926 FLAT_Real_Base_gfx10<op>, 1927 FLAT_Real_SADDR_gfx10<op>, 1928 FLAT_Real_ST_gfx10<op>; 1929 1930// ENC_FLAT. 1931defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; 1932defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; 1933defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; 1934defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; 1935defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; 1936defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; 1937defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; 1938defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; 1939defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; 1940defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; 1941defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; 1942defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; 1943defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; 1944defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; 1945defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; 1946defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; 1947defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; 1948defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; 1949defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; 1950defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; 1951defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; 1952defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; 1953defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; 1954defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; 1955defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; 1956defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; 1957defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; 1958defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; 1959defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; 1960defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; 1961defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; 1962defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; 1963defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; 1964defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; 1965defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; 1966defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; 1967defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; 1968defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; 1969defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; 1970defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; 1971defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; 1972defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; 1973defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; 1974defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; 1975defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; 1976defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; 1977defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; 1978defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; 1979defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; 1980defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; 1981defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; 1982defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; 1983defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f>; 1984defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060>; 1985 1986 1987// ENC_FLAT_GLBL. 1988defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; 1989defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; 1990defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; 1991defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; 1992defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; 1993defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; 1994defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; 1995defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; 1996defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; 1997defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; 1998defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; 1999defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; 2000defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; 2001defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; 2002defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; 2003defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; 2004defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; 2005defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; 2006defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; 2007defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; 2008defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; 2009defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; 2010defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; 2011defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; 2012defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; 2013defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; 2014defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>; 2015defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; 2016defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; 2017defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; 2018defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; 2019defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; 2020defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; 2021defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; 2022defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; 2023defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; 2024defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; 2025defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; 2026defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; 2027defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; 2028defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; 2029defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; 2030defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; 2031defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; 2032defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; 2033defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; 2034defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; 2035defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; 2036defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; 2037defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; 2038defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; 2039defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; 2040defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; 2041defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f>; 2042defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060>; 2043defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; 2044defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; 2045 2046// ENC_FLAT_SCRATCH. 2047defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; 2048defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; 2049defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; 2050defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; 2051defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; 2052defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; 2053defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; 2054defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; 2055defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; 2056defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; 2057defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; 2058defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; 2059defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; 2060defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; 2061defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; 2062defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; 2063defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; 2064defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; 2065defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; 2066defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; 2067defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; 2068defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; 2069