1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MTBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43//===----------------------------------------------------------------------===// 44// MTBUF classes 45//===----------------------------------------------------------------------===// 46 47class MTBUFGetBaseOpcode<string Op> { 48 string ret = !subst("FORMAT_XY", "FORMAT_X", 49 !subst("FORMAT_XYZ", "FORMAT_X", 50 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 51} 52 53 54class MTBUF_Pseudo <string opName, dag outs, dag ins, 55 string asmOps, list<dag> pattern=[]> : 56 InstSI<outs, ins, "", pattern>, 57 SIMCInstr<opName, SIEncodingFamily.NONE> { 58 59 let isPseudo = 1; 60 let isCodeGenOnly = 1; 61 let Size = 8; 62 let UseNamedOperandTable = 1; 63 64 string Mnemonic = opName; 65 string AsmOperands = asmOps; 66 67 Instruction Opcode = !cast<Instruction>(NAME); 68 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 69 70 let VM_CNT = 1; 71 let EXP_CNT = 1; 72 let MTBUF = 1; 73 let Uses = [EXEC]; 74 let hasSideEffects = 0; 75 let SchedRW = [WriteVMEM]; 76 77 let AsmMatchConverter = "cvtMtbuf"; 78 79 bits<1> offen = 0; 80 bits<1> idxen = 0; 81 bits<1> addr64 = 0; 82 bits<1> has_vdata = 1; 83 bits<1> has_vaddr = 1; 84 bits<1> has_glc = 1; 85 bits<1> has_dlc = 1; 86 bits<1> glc_value = 0; // the value for glc if no such operand 87 bits<1> dlc_value = 0; // the value for dlc if no such operand 88 bits<1> has_srsrc = 1; 89 bits<1> has_soffset = 1; 90 bits<1> has_offset = 1; 91 bits<1> has_slc = 1; 92 bits<1> has_tfe = 1; 93 bits<4> elements = 0; 94 bits<1> has_sccb = 1; 95 bits<1> sccb_value = 0; 96} 97 98class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 99 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 100 101 let isPseudo = 0; 102 let isCodeGenOnly = 0; 103 104 let VM_CNT = 1; 105 let EXP_CNT = 1; 106 let MTBUF = 1; 107 108 // copy relevant pseudo op flags 109 let UseNamedOperandTable = ps.UseNamedOperandTable; 110 let SubtargetPredicate = ps.SubtargetPredicate; 111 let AsmMatchConverter = ps.AsmMatchConverter; 112 let Constraints = ps.Constraints; 113 let DisableEncoding = ps.DisableEncoding; 114 let TSFlags = ps.TSFlags; 115 let SchedRW = ps.SchedRW; 116 let mayLoad = ps.mayLoad; 117 let mayStore = ps.mayStore; 118 let IsAtomicRet = ps.IsAtomicRet; 119 let IsAtomicNoRet = ps.IsAtomicNoRet; 120 121 bits<12> offset; 122 bits<5> cpol; 123 bits<7> format; 124 bits<8> vaddr; 125 bits<10> vdata; 126 bits<7> srsrc; 127 bits<1> tfe; 128 bits<8> soffset; 129 130 bits<4> dfmt = format{3-0}; 131 bits<3> nfmt = format{6-4}; 132 133 // GFX90A+ only: instruction uses AccVGPR for data 134 // Bit supersedes tfe. 135 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdata_op:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 152 TFE:$tfe, SWZ:$swz), 153 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 155 TFE:$tfe, SWZ:$swz) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc,$format $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc,$format $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc,$format $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc,$format $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 187 !eq(addrKind, BUFAddrKind.BothEn)); 188 189 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 190 !eq(addrKind, BUFAddrKind.BothEn)); 191 192 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 193 194 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 int elems, 201 list<dag> pattern=[], 202 // Workaround bug bz30254 203 int addrKindCopy = addrKind> 204 : MTBUF_Pseudo<opName, 205 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 206 getMTBUFIns<addrKindCopy>.ret, 207 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 208 pattern>, 209 MTBUF_SetupAddr<addrKindCopy> { 210 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 211 let mayLoad = 1; 212 let mayStore = 0; 213 let elements = elems; 214} 215 216multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 217 int elems> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 220 MTBUFAddr64Table<0, NAME>; 221 222 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 223 MTBUFAddr64Table<1, NAME>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 int elems, 241 list<dag> pattern=[], 242 // Workaround bug bz30254 243 int addrKindCopy = addrKind, 244 RegisterClass vdataClassCopy = vdataClass> 245 : MTBUF_Pseudo<opName, 246 (outs), 247 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 248 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 249 pattern>, 250 MTBUF_SetupAddr<addrKindCopy> { 251 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 252 let mayLoad = 0; 253 let mayStore = 1; 254 let elements = elems; 255} 256 257multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 258 int elems> { 259 260 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 261 MTBUFAddr64Table<0, NAME>; 262 263 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 264 MTBUFAddr64Table<1, NAME>; 265 266 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 267 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 268 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 269 270 let DisableWQM = 1 in { 271 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 272 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 273 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 274 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 275 } 276} 277 278 279//===----------------------------------------------------------------------===// 280// MUBUF classes 281//===----------------------------------------------------------------------===// 282 283class MUBUFGetBaseOpcode<string Op> { 284 string ret = !subst("DWORDX2", "DWORD", 285 !subst("DWORDX3", "DWORD", 286 !subst("DWORDX4", "DWORD", Op))); 287} 288 289class MUBUF_Pseudo <string opName, dag outs, dag ins, 290 string asmOps, list<dag> pattern=[]> : 291 InstSI<outs, ins, "", pattern>, 292 SIMCInstr<opName, SIEncodingFamily.NONE> { 293 294 let isPseudo = 1; 295 let isCodeGenOnly = 1; 296 let Size = 8; 297 let UseNamedOperandTable = 1; 298 299 string Mnemonic = opName; 300 string AsmOperands = asmOps; 301 302 Instruction Opcode = !cast<Instruction>(NAME); 303 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 304 305 let VM_CNT = 1; 306 let EXP_CNT = 1; 307 let MUBUF = 1; 308 let Uses = [EXEC]; 309 let hasSideEffects = 0; 310 let SchedRW = [WriteVMEM]; 311 312 let AsmMatchConverter = "cvtMubuf"; 313 314 bits<1> offen = 0; 315 bits<1> idxen = 0; 316 bits<1> addr64 = 0; 317 bits<1> lds = 0; 318 bits<1> has_vdata = !not(lds); 319 bits<1> has_vaddr = 1; 320 bits<1> has_glc = 1; 321 bits<1> has_dlc = 1; 322 bits<1> glc_value = 0; // the value for glc if no such operand 323 bits<1> dlc_value = 0; // the value for dlc if no such operand 324 bits<1> has_srsrc = 1; 325 bits<1> has_soffset = 1; 326 bits<1> has_offset = 1; 327 bits<1> has_slc = 1; 328 bits<1> has_tfe = 1; 329 bits<4> elements = 0; 330 bits<1> has_sccb = 1; 331 bits<1> sccb_value = 0; 332 bits<1> IsBufferInv = 0; 333} 334 335class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 336 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 337 338 let isPseudo = 0; 339 let isCodeGenOnly = 0; 340 341 let VM_CNT = 1; 342 let EXP_CNT = 1; 343 let MUBUF = 1; 344 345 // copy relevant pseudo op flags 346 let SubtargetPredicate = ps.SubtargetPredicate; 347 let AsmMatchConverter = ps.AsmMatchConverter; 348 let OtherPredicates = ps.OtherPredicates; 349 let Constraints = ps.Constraints; 350 let DisableEncoding = ps.DisableEncoding; 351 let TSFlags = ps.TSFlags; 352 let UseNamedOperandTable = ps.UseNamedOperandTable; 353 let SchedRW = ps.SchedRW; 354 let mayLoad = ps.mayLoad; 355 let mayStore = ps.mayStore; 356 let IsAtomicRet = ps.IsAtomicRet; 357 let IsAtomicNoRet = ps.IsAtomicNoRet; 358 let VALU = ps.VALU; 359 let LGKM_CNT = ps.LGKM_CNT; 360 361 bits<12> offset; 362 bits<5> cpol; 363 bits<8> vaddr; 364 bits<10> vdata; 365 bits<7> srsrc; 366 bits<1> tfe; 367 bits<8> soffset; 368 369 // GFX90A+ only: instruction uses AccVGPR for data 370 // Bit supersedes tfe. 371 bits<1> acc = !if(ps.has_vdata, vdata{9}, !if(ps.lds, ?, 0)); 372} 373 374 375// For cache invalidation instructions. 376class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 377 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 378 379 let AsmMatchConverter = ""; 380 381 let hasSideEffects = 1; 382 let mayLoad = 0; 383 let mayStore = 0; 384 385 let IsBufferInv = 1; 386 // Set everything else to 0. 387 let offen = 0; 388 let idxen = 0; 389 let addr64 = 0; 390 let has_vdata = 0; 391 let has_vaddr = 0; 392 let has_glc = 0; 393 let has_dlc = 0; 394 let glc_value = 0; 395 let dlc_value = 0; 396 let has_srsrc = 0; 397 let has_soffset = 0; 398 let has_offset = 0; 399 let has_slc = 0; 400 let has_tfe = 0; 401 let has_sccb = 0; 402 let sccb_value = 0; 403} 404 405class getMUBUFInsDA<list<RegisterClass> vdataList, 406 list<RegisterClass> vaddrList=[], 407 bit isLds = 0> { 408 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 409 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 410 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 411 dag InsNoData = !if(!empty(vaddrList), 412 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 413 offset:$offset, CPol_0:$cpol), 414 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 415 offset:$offset, CPol_0:$cpol) 416 ); 417 dag InsData = !if(!empty(vaddrList), 418 (ins vdata_op:$vdata, SReg_128:$srsrc, 419 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 420 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 421 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 422 ); 423 dag ret = !con( 424 !if(!empty(vdataList), InsNoData, InsData), 425 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 426 ); 427} 428 429class getMUBUFElements<ValueType vt> { 430 int ret = 431 !if(!eq(vt, f16), 1, 432 !if(!eq(vt, v2f16), 2, 433 !if(!eq(vt, v3f16), 3, 434 !if(!eq(vt, v4f16), 4, 435 !if(!eq(vt.Size, 32), 1, 436 !if(!eq(vt.Size, 64), 2, 437 !if(!eq(vt.Size, 96), 3, 438 !if(!eq(vt.Size, 128), 4, 0) 439 ) 440 ) 441 ) 442 ) 443 ) 444 ) 445 ); 446} 447 448class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 449 dag ret = 450 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 451 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 452 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 453 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 454 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 455 (ins)))))); 456} 457 458class getMUBUFAsmOps<int addrKind> { 459 string Pfx = 460 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 461 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 462 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 463 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 464 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 465 ""))))); 466 string ret = Pfx # "$offset"; 467} 468 469class MUBUF_SetupAddr<int addrKind> { 470 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 471 !eq(addrKind, BUFAddrKind.BothEn)); 472 473 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 474 !eq(addrKind, BUFAddrKind.BothEn)); 475 476 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 477 478 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 479} 480 481class MUBUF_Load_Pseudo <string opName, 482 int addrKind, 483 ValueType vdata_vt, 484 bit HasTiedDest = 0, 485 bit isLds = 0, 486 bit isLdsOpc = 0, 487 list<dag> pattern=[], 488 // Workaround bug bz30254 489 int addrKindCopy = addrKind, 490 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 491 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 492 : MUBUF_Pseudo<opName, 493 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 494 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 495 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 496 !if(!or(isLds, isLdsOpc), " ", " $vdata, ") # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 497 !if(isLds, " lds", "$tfe") # "$swz", 498 pattern>, 499 MUBUF_SetupAddr<addrKindCopy> { 500 let PseudoInstr = opName # !if(isLds, "_lds", "") # 501 "_" # getAddrName<addrKindCopy>.ret; 502 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 503 504 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 505 let LGKM_CNT = isLds; 506 let has_vdata = !not(isLdsOpc); 507 let mayLoad = 1; 508 let mayStore = isLds; 509 let maybeAtomic = 1; 510 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 511 let has_tfe = !not(isLds); 512 let lds = isLds; 513 let elements = getMUBUFElements<vdata_vt>.ret; 514 let VALU = isLds; 515} 516 517class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 518 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 519 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 520>; 521 522class MUBUF_Addr64_Load_Pat <Instruction inst, 523 ValueType load_vt = i32, 524 SDPatternOperator ld = null_frag> : Pat < 525 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 526 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 527>; 528 529multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 530 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 531 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 532} 533 534 535// FIXME: tfe can't be an operand because it requires a separate 536// opcode because it needs an N+1 register class dest register. 537multiclass MUBUF_Pseudo_Loads<string opName, 538 ValueType load_vt = i32, 539 bit TiedDest = 0, 540 bit isLds = 0> { 541 542 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 543 544 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 545 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 546 547 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 548 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 549 550 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 551 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 552 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 553 554 let DisableWQM = 1 in { 555 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 556 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 557 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 558 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 559 } 560} 561 562multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 563 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 564 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 565} 566 567multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 568 ValueType load_vt = i32, 569 bit TiedDest = 0, 570 bit isLds = 0, 571 bit isLdsOpc = 1> { 572 573 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 574 575 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 576 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 577 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 578 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 579} 580 581class MUBUF_Store_Pseudo <string opName, 582 int addrKind, 583 ValueType store_vt, 584 list<dag> pattern=[], 585 // Workaround bug bz30254 586 int addrKindCopy = addrKind> 587 : MUBUF_Pseudo<opName, 588 (outs), 589 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 590 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 591 pattern>, 592 MUBUF_SetupAddr<addrKindCopy> { 593 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 594 let mayLoad = 0; 595 let mayStore = 1; 596 let maybeAtomic = 1; 597 let elements = getMUBUFElements<store_vt>.ret; 598} 599 600multiclass MUBUF_Pseudo_Stores<string opName, 601 ValueType store_vt = i32, 602 SDPatternOperator st = null_frag> { 603 604 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 605 606 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 607 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 608 i16:$offset))]>, 609 MUBUFAddr64Table<0, NAME>; 610 611 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 612 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 613 i16:$offset))]>, 614 MUBUFAddr64Table<1, NAME>; 615 616 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 617 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 618 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 619 620 let DisableWQM = 1 in { 621 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 622 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 623 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 624 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 625 } 626} 627 628class MUBUF_Pseudo_Store_Lds<string opName> 629 : MUBUF_Pseudo<opName, 630 (outs), 631 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 632 " $srsrc, $soffset$offset lds$cpol$swz"> { 633 let LGKM_CNT = 1; 634 let mayLoad = 1; 635 let mayStore = 1; 636 let maybeAtomic = 1; 637 638 let has_vdata = 0; 639 let has_vaddr = 0; 640 let has_tfe = 0; 641 let lds = 1; 642 let VALU = 1; 643 644 let Uses = [EXEC, M0]; 645 let AsmMatchConverter = "cvtMubufLds"; 646} 647 648class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 649 list<RegisterClass> vaddrList=[]> { 650 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 651 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 652 dag ret = !if(vdata_in, 653 !if(!empty(vaddrList), 654 (ins vdata_op:$vdata_in, 655 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 656 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 657 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 658 ), 659 !if(!empty(vaddrList), 660 (ins vdata_op:$vdata, 661 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 662 (ins vdata_op:$vdata, vaddrClass:$vaddr, 663 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 664 )); 665} 666 667class getMUBUFAtomicIns<int addrKind, 668 RegisterClass vdataClass, 669 bit vdata_in, 670 // Workaround bug bz30254 671 RegisterClass vdataClassCopy=vdataClass> { 672 dag ret = 673 !if(!eq(addrKind, BUFAddrKind.Offset), 674 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 675 !if(!eq(addrKind, BUFAddrKind.OffEn), 676 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 677 !if(!eq(addrKind, BUFAddrKind.IdxEn), 678 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 679 !if(!eq(addrKind, BUFAddrKind.BothEn), 680 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 681 !if(!eq(addrKind, BUFAddrKind.Addr64), 682 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 683 (ins)))))); 684} 685 686class MUBUF_Atomic_Pseudo<string opName, 687 int addrKind, 688 dag outs, 689 dag ins, 690 string asmOps, 691 list<dag> pattern=[], 692 // Workaround bug bz30254 693 int addrKindCopy = addrKind> 694 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 695 MUBUF_SetupAddr<addrKindCopy> { 696 let mayStore = 1; 697 let mayLoad = 1; 698 let hasPostISelHook = 1; 699 let hasSideEffects = 1; 700 let DisableWQM = 1; 701 let has_glc = 0; 702 let has_dlc = 0; 703 let has_tfe = 0; 704 let has_sccb = 1; 705 let maybeAtomic = 1; 706 let AsmMatchConverter = "cvtMubufAtomic"; 707} 708 709class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 710 RegisterClass vdataClass, 711 list<dag> pattern=[], 712 // Workaround bug bz30254 713 int addrKindCopy = addrKind, 714 RegisterClass vdataClassCopy = vdataClass> 715 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 716 (outs), 717 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 718 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 719 pattern>, 720 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 721 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 722 let glc_value = 0; 723 let dlc_value = 0; 724 let sccb_value = 0; 725 let IsAtomicNoRet = 1; 726} 727 728class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 729 RegisterClass vdataClass, 730 list<dag> pattern=[], 731 // Workaround bug bz30254 732 int addrKindCopy = addrKind, 733 RegisterClass vdataClassCopy = vdataClass, 734 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 735 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 736 (outs vdata_op:$vdata), 737 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 738 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 739 pattern>, 740 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 741 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 742 let glc_value = 1; 743 let dlc_value = 0; 744 let sccb_value = 0; 745 let IsAtomicRet = 1; 746 let Constraints = "$vdata = $vdata_in"; 747 let DisableEncoding = "$vdata_in"; 748} 749 750multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 751 RegisterClass vdataClass, 752 ValueType vdataType, 753 bit isFP = isFloatType<vdataType>.ret> { 754 let FPAtomic = isFP in 755 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 756 MUBUFAddr64Table <0, NAME>; 757 758 let FPAtomic = isFP in 759 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 760 MUBUFAddr64Table <1, NAME>; 761 762 let FPAtomic = isFP in 763 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 764 765 let FPAtomic = isFP in 766 767 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 768 769 let FPAtomic = isFP in 770 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 771} 772 773multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 774 RegisterClass vdataClass, 775 ValueType vdataType, 776 SDPatternOperator atomic, 777 bit isFP = isFloatType<vdataType>.ret> { 778 let FPAtomic = isFP in 779 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 780 [(set vdataType:$vdata, 781 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 782 vdataType:$vdata_in))]>, 783 MUBUFAddr64Table <0, NAME # "_RTN">; 784 785 let FPAtomic = isFP in 786 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 787 [(set vdataType:$vdata, 788 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 789 vdataType:$vdata_in))]>, 790 MUBUFAddr64Table <1, NAME # "_RTN">; 791 792 let FPAtomic = isFP in 793 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 794 795 let FPAtomic = isFP in 796 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 797 798 let FPAtomic = isFP in 799 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 800} 801 802multiclass MUBUF_Pseudo_Atomics <string opName, 803 RegisterClass vdataClass, 804 ValueType vdataType, 805 SDPatternOperator atomic = null_frag> : 806 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 807 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 808 809 810//===----------------------------------------------------------------------===// 811// MUBUF Instructions 812//===----------------------------------------------------------------------===// 813 814defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 815 "buffer_load_format_x", f32 816>; 817defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 818 "buffer_load_format_xy", v2f32 819>; 820defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 821 "buffer_load_format_xyz", v3f32 822>; 823defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 824 "buffer_load_format_xyzw", v4f32 825>; 826defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 827 "buffer_store_format_x", f32 828>; 829defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 830 "buffer_store_format_xy", v2f32 831>; 832defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 833 "buffer_store_format_xyz", v3f32 834>; 835defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 836 "buffer_store_format_xyzw", v4f32 837>; 838 839let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 840 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_x", i32 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xy", v2i32 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xyz", v3i32 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyzw", v4i32 851 >; 852 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_x", i32 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xy", v2i32 857 >; 858 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 859 "buffer_store_format_d16_xyz", v3i32 860 >; 861 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 862 "buffer_store_format_d16_xyzw", v4i32 863 >; 864} // End HasUnpackedD16VMem. 865 866let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 867 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 868 "buffer_load_format_d16_x", f16 869 >; 870 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 871 "buffer_load_format_d16_xy", v2f16 872 >; 873 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 874 "buffer_load_format_d16_xyz", v3f16 875 >; 876 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 877 "buffer_load_format_d16_xyzw", v4f16 878 >; 879 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 880 "buffer_store_format_d16_x", f16 881 >; 882 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 883 "buffer_store_format_d16_xy", v2f16 884 >; 885 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 886 "buffer_store_format_d16_xyz", v3f16 887 >; 888 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 889 "buffer_store_format_d16_xyzw", v4f16 890 >; 891} // End HasPackedD16VMem. 892 893defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 894 "buffer_load_ubyte", i32 895>; 896defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 897 "buffer_load_sbyte", i32 898>; 899defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 900 "buffer_load_ushort", i32 901>; 902defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 903 "buffer_load_sshort", i32 904>; 905defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 906 "buffer_load_dword", i32 907>; 908defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 909 "buffer_load_dwordx2", v2i32 910>; 911defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 912 "buffer_load_dwordx3", v3i32 913>; 914defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 915 "buffer_load_dwordx4", v4i32 916>; 917 918defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 919 "buffer_load_lds_b32", i32 920>; 921defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 922 "buffer_load_lds_format_x", f32 923>; 924defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 925 "buffer_load_lds_i8", i32 926>; 927defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 928 "buffer_load_lds_i16", i32 929>; 930defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 931 "buffer_load_lds_u8", i32 932>; 933defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 934 "buffer_load_lds_u16", i32 935>; 936 937defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 938defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 939defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 940defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 941defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 942defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 943defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 944defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 945defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 946defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 947defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 948defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 949defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 950defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 951 952defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 953 "buffer_store_byte", i32, truncstorei8_global 954>; 955defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 956 "buffer_store_short", i32, truncstorei16_global 957>; 958defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 959 "buffer_store_dword", i32, store_global 960>; 961defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 962 "buffer_store_dwordx2", v2i32, store_global 963>; 964defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 965 "buffer_store_dwordx3", v3i32, store_global 966>; 967defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 968 "buffer_store_dwordx4", v4i32, store_global 969>; 970defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_swap", VGPR_32, i32 972>; 973defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_cmpswap", VReg_64, v2i32 975>; 976defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_add", VGPR_32, i32 978>; 979defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_sub", VGPR_32, i32 981>; 982defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_smin", VGPR_32, i32 984>; 985defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_umin", VGPR_32, i32 987>; 988defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_smax", VGPR_32, i32 990>; 991defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_umax", VGPR_32, i32 993>; 994defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_and", VGPR_32, i32 996>; 997defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_or", VGPR_32, i32 999>; 1000defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_xor", VGPR_32, i32 1002>; 1003defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_inc", VGPR_32, i32 1005>; 1006defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_dec", VGPR_32, i32 1008>; 1009defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_swap_x2", VReg_64, i64 1011>; 1012defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1014>; 1015defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_add_x2", VReg_64, i64 1017>; 1018defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_sub_x2", VReg_64, i64 1020>; 1021defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_smin_x2", VReg_64, i64 1023>; 1024defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_umin_x2", VReg_64, i64 1026>; 1027defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_smax_x2", VReg_64, i64 1029>; 1030defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_umax_x2", VReg_64, i64 1032>; 1033defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1034 "buffer_atomic_and_x2", VReg_64, i64 1035>; 1036defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1037 "buffer_atomic_or_x2", VReg_64, i64 1038>; 1039defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1040 "buffer_atomic_xor_x2", VReg_64, i64 1041>; 1042defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1043 "buffer_atomic_inc_x2", VReg_64, i64 1044>; 1045defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1046 "buffer_atomic_dec_x2", VReg_64, i64 1047>; 1048 1049let SubtargetPredicate = HasGFX10_BEncoding in 1050defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1051 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1052>; 1053 1054let SubtargetPredicate = isGFX8GFX9 in { 1055def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1056} 1057 1058let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1059/* 1060defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1061defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1062*/ 1063 1064def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1065 int_amdgcn_buffer_wbinvl1_sc>; 1066} 1067 1068let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1069 1070defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1072>; 1073defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1074 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1075>; 1076defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1077 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1078>; 1079 1080} 1081 1082let SubtargetPredicate = isGFX6GFX7GFX10 in { 1083 1084defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1085 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1086>; 1087defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1088 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1089>; 1090defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1091 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1092>; 1093 1094} 1095 1096let SubtargetPredicate = HasD16LoadStore in { 1097 1098defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1099 "buffer_load_ubyte_d16", i32, 1 1100>; 1101 1102defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1103 "buffer_load_ubyte_d16_hi", i32, 1 1104>; 1105 1106defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1107 "buffer_load_sbyte_d16", i32, 1 1108>; 1109 1110defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1111 "buffer_load_sbyte_d16_hi", i32, 1 1112>; 1113 1114defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1115 "buffer_load_short_d16", i32, 1 1116>; 1117 1118defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1119 "buffer_load_short_d16_hi", i32, 1 1120>; 1121 1122defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1123 "buffer_store_byte_d16_hi", i32 1124>; 1125 1126defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1127 "buffer_store_short_d16_hi", i32 1128>; 1129 1130defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1131 "buffer_load_format_d16_hi_x", i32 1132>; 1133defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1134 "buffer_store_format_d16_hi_x", i32 1135>; 1136 1137} // End HasD16LoadStore 1138 1139def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1140 int_amdgcn_buffer_wbinvl1>; 1141 1142let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1143defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1144 "buffer_atomic_add_f32", VGPR_32, f32 1145>; 1146 1147let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1148defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1149 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1150>; 1151 1152let OtherPredicates = [HasAtomicFaddRtnInsts] in 1153defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1154 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1155>; 1156 1157let OtherPredicates = [isGFX90APlus] in 1158defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1159 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1160>; 1161 1162//===----------------------------------------------------------------------===// 1163// MTBUF Instructions 1164//===----------------------------------------------------------------------===// 1165 1166defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1167defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1168defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1169defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1170defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1171defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1172defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1173defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1174 1175let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1176 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1177 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1178 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1179 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1180 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1181 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1182 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1183 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1184} // End HasUnpackedD16VMem. 1185 1186let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1187 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1188 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1189 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1190 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1191 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1192 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1193 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1194 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1195} // End HasPackedD16VMem. 1196 1197let SubtargetPredicate = isGFX7Plus in { 1198 1199//===----------------------------------------------------------------------===// 1200// Instruction definitions for CI and newer. 1201//===----------------------------------------------------------------------===// 1202 1203def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1204 int_amdgcn_buffer_wbinvl1_vol>; 1205 1206} // End let SubtargetPredicate = isGFX7Plus 1207 1208let SubtargetPredicate = isGFX90APlus in { 1209 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1210 let has_glc = 1; 1211 let has_sccb = 1; 1212 let InOperandList = (ins CPol_0:$cpol); 1213 let AsmOperands = "$cpol"; 1214 } 1215 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1216 let SubtargetPredicate = isGFX90AOnly; 1217 } 1218 1219 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1220 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1221 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1222} // End SubtargetPredicate = isGFX90APlus 1223 1224def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1225 let SubtargetPredicate = isGFX940Plus; 1226 let has_glc = 1; 1227 let has_sccb = 1; 1228 let InOperandList = (ins CPol_0:$cpol); 1229 let AsmOperands = "$cpol"; 1230} 1231 1232let SubtargetPredicate = isGFX10Plus in { 1233 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1234 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1235} // End SubtargetPredicate = isGFX10Plus 1236 1237//===----------------------------------------------------------------------===// 1238// MUBUF Patterns 1239//===----------------------------------------------------------------------===// 1240 1241//===----------------------------------------------------------------------===// 1242// buffer_load/store_format patterns 1243//===----------------------------------------------------------------------===// 1244 1245multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1246 string opcode, ValueType memoryVt = vt> { 1247 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1248 1249 def : GCNPat< 1250 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1251 timm:$auxiliary, 0)), 1252 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1253 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1254 >; 1255 1256 def : GCNPat< 1257 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1258 timm:$auxiliary, 0)), 1259 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1260 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1261 >; 1262 1263 def : GCNPat< 1264 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1265 timm:$auxiliary, timm)), 1266 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1267 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1268 >; 1269 1270 def : GCNPat< 1271 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1272 timm:$auxiliary, timm)), 1273 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1274 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1275 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1276 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1277 >; 1278} 1279 1280defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1281defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1282defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1283defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1288 1289let SubtargetPredicate = HasUnpackedD16VMem in { 1290 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1291 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1292 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1293 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1294 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1295 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1296} // End HasUnpackedD16VMem. 1297 1298let SubtargetPredicate = HasPackedD16VMem in { 1299 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1300 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1301 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1302 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1303 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1304 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1305 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1306 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1307 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1308} // End HasPackedD16VMem. 1309 1310defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1311defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1312defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1313defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1314defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1315defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1316defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1317defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1318defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1319defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1320defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1321defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1322defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1323defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1324defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1325defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1326 1327multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1328 string opcode, ValueType memoryVt = vt> { 1329 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1330 1331 def : GCNPat< 1332 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1333 timm:$auxiliary, 0), 1334 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1335 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1336 >; 1337 1338 def : GCNPat< 1339 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1340 timm:$auxiliary, 0), 1341 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1342 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1343 >; 1344 1345 def : GCNPat< 1346 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1347 timm:$auxiliary, timm), 1348 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1349 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1350 >; 1351 1352 def : GCNPat< 1353 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1354 timm:$auxiliary, timm), 1355 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1356 getVregSrcForVT<vt>.ret:$vdata, 1357 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1358 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1359 0, (extract_swz $auxiliary)) 1360 >; 1361} 1362 1363defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1364defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1365defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1366defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1371 1372let SubtargetPredicate = HasUnpackedD16VMem in { 1373 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1374 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1375 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1376 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1377 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1378 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1379} // End HasUnpackedD16VMem. 1380 1381let SubtargetPredicate = HasPackedD16VMem in { 1382 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1383 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1384 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1385 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1386 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1387 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1388 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1389 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1390 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1391} // End HasPackedD16VMem. 1392 1393defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1394defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1395defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1396defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1397defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1398defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1399defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1400defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1401defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1402defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1403defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1404defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1405defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1406defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1407 1408//===----------------------------------------------------------------------===// 1409// buffer_atomic patterns 1410//===----------------------------------------------------------------------===// 1411 1412multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1413 foreach RtnMode = ["ret", "noret"] in { 1414 1415 defvar Op = !cast<SDPatternOperator>(OpPrefix # "_" # RtnMode 1416 # !if(isIntr, "", "_" # vt.Size)); 1417 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1418 1419 def : GCNPat< 1420 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1421 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1422 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1423 >; 1424 1425 def : GCNPat< 1426 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1427 vt:$vdata_in)), 1428 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1429 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1430 >; 1431 1432 } // end foreach RtnMode 1433} 1434 1435multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1436 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1437} 1438 1439multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1440 foreach RtnMode = ["ret", "noret"] in { 1441 1442 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode 1443 # "_" # vt.Size); 1444 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1445 1446 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1447 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1448 offset:$offset); 1449 def : GCNPat< 1450 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1451 !if(!eq(RtnMode, "ret"), 1452 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1453 !if(!eq(vt, i32), sub0, sub0_sub1)), 1454 OffsetResDag) 1455 >; 1456 1457 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1458 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1459 SCSrc_b32:$soffset, offset:$offset); 1460 def : GCNPat< 1461 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1462 data_vt:$vdata_in)), 1463 !if(!eq(RtnMode, "ret"), 1464 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1465 !if(!eq(vt, i32), sub0, sub0_sub1)), 1466 Addr64ResDag) 1467 >; 1468 1469 } // end foreach RtnMode 1470} 1471 1472foreach Ty = [i32, i64] in { 1473 1474defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1475 1476defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1477defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1478defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1479defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1480defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1481defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1482defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1483defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1484defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1485defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1486defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1487defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1488 1489} // end foreach Ty 1490 1491defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1492defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1493 1494multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1495 list<string> RtnModes = ["ret", "noret"]> { 1496 foreach RtnMode = RtnModes in { 1497 1498 defvar Op = !cast<SDPatternOperator>(!if(!eq(RtnMode, "none"), 1499 OpPrefix, OpPrefix # "_" # RtnMode)); 1500 defvar InstSuffix = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1501 "_RTN", ""); 1502 defvar CachePolicy = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1503 (set_glc $cachepolicy), (timm:$cachepolicy)); 1504 1505 def : GCNPat< 1506 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1507 timm:$offset, timm:$cachepolicy, 0)), 1508 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1509 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1510 (as_i16timm $offset), CachePolicy) 1511 >; 1512 1513 def : GCNPat< 1514 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1515 timm:$offset, timm:$cachepolicy, timm)), 1516 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1517 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1518 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1519 >; 1520 1521 def : GCNPat< 1522 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1523 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1524 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1525 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1526 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1527 >; 1528 1529 def : GCNPat< 1530 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1531 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1532 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1533 getVregSrcForVT<vt>.ret:$vdata_in, 1534 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1535 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1536 >; 1537 1538 } // end foreach RtnMode 1539} 1540 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1545defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1546defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1547defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1548defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1549defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1550defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1551defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1552defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1553defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1554defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["none"]>; 1555defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1556defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1557defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1558defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1559defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1560defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1561defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1562defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1563defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1564defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1565defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1566defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1567 1568let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1569 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1570 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1571} 1572let SubtargetPredicate = isGFX6GFX7GFX10 in { 1573 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1574 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1575} 1576 1577class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1578 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1579 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1580 [{ return SDValue(N, 0).use_empty(); }]> { 1581 1582 let GISelPredicateCode = [{ 1583 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1584 }]; 1585} 1586 1587multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1588 string opcode> { 1589 def : GCNPat< 1590 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1591 0, i32:$soffset, timm:$offset, 1592 timm:$cachepolicy, 0), 1593 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1594 (as_i16timm $offset), $cachepolicy) 1595 >; 1596 1597 def : GCNPat< 1598 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1599 0, i32:$soffset, timm:$offset, 1600 timm:$cachepolicy, timm), 1601 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1602 (as_i16timm $offset), $cachepolicy) 1603 >; 1604 1605 def : GCNPat< 1606 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1607 i32:$voffset, i32:$soffset, timm:$offset, 1608 timm:$cachepolicy, 0), 1609 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1610 (as_i16timm $offset), $cachepolicy) 1611 >; 1612 1613 def : GCNPat< 1614 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1615 i32:$voffset, i32:$soffset, timm:$offset, 1616 timm:$cachepolicy, timm), 1617 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1618 getVregSrcForVT<vt>.ret:$vdata_in, 1619 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1620 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1621 >; 1622} 1623 1624let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1625defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1626 1627let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1628defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1629 1630let SubtargetPredicate = HasAtomicFaddRtnInsts in 1631 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1632 1633let SubtargetPredicate = isGFX90APlus in { 1634 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1635 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1636 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1637 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1638 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1639 1640 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1641 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1642 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1643} // End SubtargetPredicate = isGFX90APlus 1644 1645foreach RtnMode = ["ret", "noret"] in { 1646 1647defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap # "_" # RtnMode); 1648defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1649defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1650 (timm:$cachepolicy)); 1651 1652defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1653 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1654 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1655def : GCNPat< 1656 (Op 1657 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1658 timm:$offset, timm:$cachepolicy, 0), 1659 !if(!eq(RtnMode, "ret"), 1660 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1661 OffsetResDag) 1662>; 1663 1664defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1665 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1666 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1667 CachePolicy); 1668def : GCNPat< 1669 (Op 1670 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1671 0, i32:$soffset, timm:$offset, 1672 timm:$cachepolicy, timm), 1673 !if(!eq(RtnMode, "ret"), 1674 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1675 IdxenResDag) 1676>; 1677 1678defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1679 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1680 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1681 CachePolicy); 1682def : GCNPat< 1683 (Op 1684 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1685 i32:$voffset, i32:$soffset, timm:$offset, 1686 timm:$cachepolicy, 0), 1687 !if(!eq(RtnMode, "ret"), 1688 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1689 OffenResDag) 1690>; 1691 1692defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1693 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1694 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1695 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1696def : GCNPat< 1697 (Op 1698 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1699 i32:$voffset, i32:$soffset, timm:$offset, 1700 timm:$cachepolicy, timm), 1701 !if(!eq(RtnMode, "ret"), 1702 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1703 BothenResDag) 1704>; 1705 1706} // end foreach RtnMode 1707 1708class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1709 PatFrag constant_ld> : GCNPat < 1710 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1711 i16:$offset))), 1712 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1713 >; 1714 1715multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1716 ValueType vt, PatFrag atomic_ld> { 1717 def : GCNPat < 1718 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1719 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1720 >; 1721 1722 def : GCNPat < 1723 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1724 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1725 >; 1726} 1727 1728let SubtargetPredicate = isGFX6GFX7 in { 1729def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1730def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1731def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1732def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1733def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1734def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1735 1736defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1737defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1738} // End SubtargetPredicate = isGFX6GFX7 1739 1740multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1741 PatFrag ld> { 1742 1743 def : GCNPat < 1744 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1745 (Instr_OFFSET $srsrc, $soffset, $offset) 1746 >; 1747} 1748 1749let OtherPredicates = [Has16BitInsts] in { 1750 1751defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1752defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1753defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1754defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1755defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1756defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1757 1758defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1759 1760} // End OtherPredicates = [Has16BitInsts] 1761 1762multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1763 MUBUF_Pseudo InstrOffset, 1764 ValueType vt, PatFrag ld> { 1765 def : GCNPat < 1766 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1767 i32:$soffset, u16imm:$offset))), 1768 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1769 >; 1770 1771 def : GCNPat < 1772 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1773 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1774 >; 1775} 1776 1777// XXX - Is it possible to have a complex pattern in a PatFrag? 1778multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1779 MUBUF_Pseudo InstrOffset, 1780 ValueType vt, PatFrag ld_frag> { 1781 def : GCNPat < 1782 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1783 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1784 >; 1785 1786 def : GCNPat < 1787 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1788 (InstrOffset $srsrc, $soffset, $offset, $in) 1789 >; 1790} 1791 1792let OtherPredicates = [DisableFlatScratch] in { 1793defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1794defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1795defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1796defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1797defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1798defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1799defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1800defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1801defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1802defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1803 1804foreach vt = Reg32Types.types in { 1805defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1806} 1807defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1808defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1809defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1810 1811let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1812defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1813defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1814defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1815defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1816defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1817defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1818 1819defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1820defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1821defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1822defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1823defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1824defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1825} 1826 1827} // End OtherPredicates = [DisableFlatScratch] 1828 1829multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1830 ValueType vt, PatFrag atomic_st> { 1831 // Store follows atomic op convention so address is first 1832 def : GCNPat < 1833 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1834 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1835 >; 1836 1837 def : GCNPat < 1838 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1839 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1840 >; 1841} 1842let SubtargetPredicate = isGFX6GFX7 in { 1843defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1844defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1845defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1846defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1847defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1848defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1849} // End Predicates = isGFX6GFX7 1850 1851 1852multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1853 PatFrag st> { 1854 1855 def : GCNPat < 1856 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1857 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1858 >; 1859} 1860 1861defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1862defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1863 1864multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1865 MUBUF_Pseudo InstrOffset, 1866 ValueType vt, PatFrag st, 1867 RegisterClass rc = VGPR_32> { 1868 def : GCNPat < 1869 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1870 i32:$soffset, u16imm:$offset)), 1871 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1872 >; 1873 1874 def : GCNPat < 1875 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1876 u16imm:$offset)), 1877 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1878 >; 1879} 1880 1881let OtherPredicates = [DisableFlatScratch] in { 1882defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1883defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1884defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1885defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1886 1887foreach vt = Reg32Types.types in { 1888defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1889} 1890 1891defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1892defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1893defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1894 1895 1896let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1897 // Hiding the extract high pattern in the PatFrag seems to not 1898 // automatically increase the complexity. 1899let AddedComplexity = 1 in { 1900defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1901defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1902} 1903} 1904} // End OtherPredicates = [DisableFlatScratch] 1905 1906//===----------------------------------------------------------------------===// 1907// MTBUF Patterns 1908//===----------------------------------------------------------------------===// 1909 1910//===----------------------------------------------------------------------===// 1911// tbuffer_load/store_format patterns 1912//===----------------------------------------------------------------------===// 1913 1914multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1915 string opcode, ValueType memoryVt = vt> { 1916 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1917 1918 def : GCNPat< 1919 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1920 timm:$format, timm:$auxiliary, 0)), 1921 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1922 (as_i8timm $format), 1923 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1924 >; 1925 1926 def : GCNPat< 1927 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1928 timm:$format, timm:$auxiliary, timm)), 1929 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1930 (as_i8timm $format), 1931 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1932 >; 1933 1934 def : GCNPat< 1935 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1936 timm:$format, timm:$auxiliary, 0)), 1937 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1938 (as_i8timm $format), 1939 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1940 >; 1941 1942 def : GCNPat< 1943 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1944 timm:$format, timm:$auxiliary, timm)), 1945 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1946 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1947 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1948 (as_i8timm $format), 1949 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1950 >; 1951} 1952 1953defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1954defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1955defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1956defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1957defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1958defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1959defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1960defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1961 1962let SubtargetPredicate = HasUnpackedD16VMem in { 1963 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1964 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1965 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1966 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1967 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1968} // End HasUnpackedD16VMem. 1969 1970let SubtargetPredicate = HasPackedD16VMem in { 1971 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1972 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1973 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1974 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1975 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1976} // End HasPackedD16VMem. 1977 1978multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1979 string opcode, ValueType memoryVt = vt> { 1980 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1981 1982 def : GCNPat< 1983 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1984 timm:$format, timm:$auxiliary, 0), 1985 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1986 (as_i16timm $offset), (as_i8timm $format), 1987 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1988 >; 1989 1990 def : GCNPat< 1991 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1992 timm:$format, timm:$auxiliary, timm), 1993 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1994 (as_i16timm $offset), (as_i8timm $format), 1995 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1996 >; 1997 1998 def : GCNPat< 1999 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 2000 timm:$format, timm:$auxiliary, 0), 2001 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2002 (as_i16timm $offset), (as_i8timm $format), 2003 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2004 >; 2005 2006 def : GCNPat< 2007 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 2008 timm:$offset, timm:$format, timm:$auxiliary, timm), 2009 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2010 getVregSrcForVT<vt>.ret:$vdata, 2011 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2012 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 2013 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2014 >; 2015} 2016 2017defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2018defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2019defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2020defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2021defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2022defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2023defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2024defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2025 2026let SubtargetPredicate = HasUnpackedD16VMem in { 2027 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2028 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2029 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2030 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2031 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2032} // End HasUnpackedD16VMem. 2033 2034let SubtargetPredicate = HasPackedD16VMem in { 2035 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2036 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2037 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2038 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2039 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2040} // End HasPackedD16VMem. 2041 2042//===----------------------------------------------------------------------===// 2043// Target-specific instruction encodings. 2044//===----------------------------------------------------------------------===// 2045 2046//===----------------------------------------------------------------------===// 2047// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2048//===----------------------------------------------------------------------===// 2049 2050class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2051 string real_name = ps.Mnemonic> : 2052 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2053 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2054 let Inst{31-26} = 0x38; 2055 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2056 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2057 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2058 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2059} 2060 2061class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2062 string real_name = ps.Mnemonic> : 2063 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2064 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2065 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2066 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2067 let Inst{25-18} = op; 2068 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2069 let Inst{54} = ps.offen; 2070 let Inst{55} = ps.idxen; 2071} 2072 2073class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2074 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2075 let Inst{12} = ps.offen; 2076 let Inst{13} = ps.idxen; 2077 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2078 let Inst{16} = ps.lds; 2079 let Inst{24-18} = op; 2080 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2081 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2082} 2083 2084class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2085 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2086 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2087 let Inst{25} = op{7}; 2088} 2089 2090class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2091 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2092 let Inst{15} = ps.addr64; 2093} 2094 2095//===----------------------------------------------------------------------===// 2096// MUBUF - GFX11. 2097//===----------------------------------------------------------------------===// 2098 2099let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2100multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name> { 2101 def _BOTHEN_gfx11 : 2102 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>, 2103 AtomicNoRet<NAME # "_BOTHEN_gfx11", 0>; 2104 def _IDXEN_gfx11 : 2105 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), real_name>, 2106 AtomicNoRet<NAME # "_IDXEN_gfx11", 0>; 2107 def _OFFEN_gfx11 : 2108 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), real_name>, 2109 AtomicNoRet<NAME # "_OFFEN_gfx11", 0>; 2110 def _OFFSET_gfx11 : 2111 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), real_name>, 2112 AtomicNoRet<NAME # "_OFFSET_gfx11", 0>; 2113} 2114 2115multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, MUBUF_Pseudo ps> : 2116 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2117multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op> : 2118 MUBUF_Real_AllAddr_gfx11_Impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2119 2120class Pre_gfx11_MUBUF_Name <MUBUF_Pseudo ps, string real_name> : 2121 MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2122multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> : 2123 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2124 def : Pre_gfx11_MUBUF_Name<!cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2125} 2126 2127let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2128multiclass MUBUF_Real_Atomics_RTN_gfx11_Renamed<bits<8> op, string real_name> { 2129 def _BOTHEN_RTN_gfx11 : 2130 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN"), real_name>, 2131 AtomicNoRet<NAME # "_BOTHEN_gfx11", 1>; 2132 def _IDXEN_RTN_gfx11 : 2133 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN"), real_name>, 2134 AtomicNoRet<NAME # "_IDXEN_gfx11", 1>; 2135 def _OFFEN_RTN_gfx11 : 2136 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN"), real_name>, 2137 AtomicNoRet<NAME # "_OFFEN_gfx11", 1>; 2138 def _OFFSET_RTN_gfx11 : 2139 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN"), real_name>, 2140 AtomicNoRet<NAME # "_OFFSET_gfx11", 1>; 2141} 2142 2143multiclass MUBUF_Real_Atomics_RTN_gfx11_impl<bits<8> op, MUBUF_Pseudo ps> : 2144 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, ps.Mnemonic>; 2145multiclass MUBUF_Real_Atomics_RTN_gfx11<bits<8> op> : 2146 MUBUF_Real_Atomics_RTN_gfx11_impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2147 2148multiclass MUBUF_Real_Atomics_gfx11<bits<8> op> : 2149 MUBUF_Real_AllAddr_gfx11<op>, 2150 MUBUF_Real_Atomics_RTN_gfx11<op>; 2151 2152multiclass MUBUF_Real_Atomics_gfx11_Renamed<bits<8> op, string real_name> : 2153 MUBUF_Real_AllAddr_gfx11_Renamed<op, real_name>, 2154 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, real_name>; 2155 2156let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2157def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2158def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2159} 2160 2161defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">; 2162defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">; 2163defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">; 2164defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">; 2165defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">; 2166defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">; 2167defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">; 2168defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2169defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2170defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">; 2171defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2172defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">; 2173defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">; 2174defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">; 2175defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">; 2176defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x000>; 2177defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x001>; 2178defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x002>; 2179defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x003>; 2180defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">; 2181defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">; 2182defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">; 2183defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">; 2184defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031>; 2185defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032>; 2186defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e>; 2187defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030>; 2188defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d>; 2189defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f>; 2190defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">; 2191defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">; 2192defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">; 2193defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">; 2194defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">; 2195defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">; 2196defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">; 2197defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">; 2198defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2199defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2200defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">; 2201defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">; 2202defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2203defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x004>; 2204defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x005>; 2205defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x006>; 2206defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x007>; 2207defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomics_gfx11<0x056>; 2208defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx11_Renamed<0x035, "buffer_atomic_add_u32">; 2209defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x043, "buffer_atomic_add_u64">; 2210defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">; 2211defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x049, "buffer_atomic_and_b64">; 2212defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2213defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2214defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2215defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx11_Renamed<0x037, "buffer_atomic_csub_u32">; 2216def : MnemonicAlias<"buffer_atomic_csub", "buffer_atomic_csub_u32">, Requires<[isGFX11Plus]>; 2217defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">; 2218defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">; 2219defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">; 2220defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">; 2221defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x052, "buffer_atomic_max_f32">; 2222defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">; 2223defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x047, "buffer_atomic_max_i64">; 2224defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">; 2225defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x048, "buffer_atomic_max_u64">; 2226defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x051, "buffer_atomic_min_f32">; 2227defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x038, "buffer_atomic_min_i32">; 2228defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x045, "buffer_atomic_min_i64">; 2229defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x039, "buffer_atomic_min_u32">; 2230defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x046, "buffer_atomic_min_u64">; 2231defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">; 2232defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">; 2233defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">; 2234defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">; 2235defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">; 2236defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">; 2237defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">; 2238defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">; 2239 2240//===----------------------------------------------------------------------===// 2241// MUBUF - GFX10. 2242//===----------------------------------------------------------------------===// 2243 2244let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2245 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2246 def _BOTHEN_gfx10 : 2247 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2248 def _IDXEN_gfx10 : 2249 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2250 def _OFFEN_gfx10 : 2251 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2252 def _OFFSET_gfx10 : 2253 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2254 } 2255 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2256 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2257 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2258 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2259 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2260 2261 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2262 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2263 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2264 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2265 } 2266 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2267 def _BOTHEN_RTN_gfx10 : 2268 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2269 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2270 def _IDXEN_RTN_gfx10 : 2271 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2272 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2273 def _OFFEN_RTN_gfx10 : 2274 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2275 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2276 def _OFFSET_RTN_gfx10 : 2277 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2278 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2279 } 2280 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2281 MUBUF_Real_Atomics_RTN_gfx10<op> { 2282 def _BOTHEN_gfx10 : 2283 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2284 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2285 def _IDXEN_gfx10 : 2286 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2287 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2288 def _OFFEN_gfx10 : 2289 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2290 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2291 def _OFFSET_gfx10 : 2292 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2293 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2294 } 2295} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2296 2297defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2298defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2299defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2300defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2301defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2302defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2303defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2304defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2305// FIXME-GFX10: Add following instructions: 2306//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2307//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2308defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2309defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2310defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2311defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2312defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2313defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2314defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2315defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2316 2317def BUFFER_GL0_INV_gfx10 : 2318 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2319def BUFFER_GL1_INV_gfx10 : 2320 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2321 2322//===----------------------------------------------------------------------===// 2323// MUBUF - GFX6, GFX7, GFX10. 2324//===----------------------------------------------------------------------===// 2325 2326let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2327 multiclass MUBUF_Real_gfx6<bits<8> op> { 2328 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2329 } 2330} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2331 2332let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2333 multiclass MUBUF_Real_gfx7<bits<8> op> { 2334 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2335 } 2336} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2337 2338let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2339 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2340 def _ADDR64_gfx6_gfx7 : 2341 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2342 def _BOTHEN_gfx6_gfx7 : 2343 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2344 def _IDXEN_gfx6_gfx7 : 2345 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2346 def _OFFEN_gfx6_gfx7 : 2347 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2348 def _OFFSET_gfx6_gfx7 : 2349 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2350 } 2351 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2352 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2353 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2354 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2355 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2356 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2357 2358 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2359 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2360 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2361 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2362 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2363 } 2364 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2365 def _ADDR64_gfx6_gfx7 : 2366 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2367 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2368 def _BOTHEN_gfx6_gfx7 : 2369 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2370 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2371 def _IDXEN_gfx6_gfx7 : 2372 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2373 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2374 def _OFFEN_gfx6_gfx7 : 2375 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2376 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2377 def _OFFSET_gfx6_gfx7 : 2378 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2379 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2380 2381 def _ADDR64_RTN_gfx6_gfx7 : 2382 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2383 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2384 def _BOTHEN_RTN_gfx6_gfx7 : 2385 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2386 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2387 def _IDXEN_RTN_gfx6_gfx7 : 2388 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2389 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2390 def _OFFEN_RTN_gfx6_gfx7 : 2391 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2392 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2393 def _OFFSET_RTN_gfx6_gfx7 : 2394 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2395 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2396 } 2397} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2398 2399multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2400 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2401 2402multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2403 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2404 2405multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2406 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2407 2408// FIXME-GFX6: Following instructions are available only on GFX6. 2409//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2410//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2411 2412defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2413defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2414defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2415defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2416defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2417defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2418defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2419defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2420defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2421defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2422defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2423defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2424defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2425defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2426defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2427defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2428defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2429defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2430defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2431defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2432defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2433defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2434 2435defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2436defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2437defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2438defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2439defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2440defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2441defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2442defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2443defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2444defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2445defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2446defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2447defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2448defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2449defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2450defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2451defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2452defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2453defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2454defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2455defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2456defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2457defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2458defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2459defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2460defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2461defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2462defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2463defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2464// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2465defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2466defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2467defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2468 2469defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2470 2471defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2472defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2473def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2474 2475//===----------------------------------------------------------------------===// 2476// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2477//===----------------------------------------------------------------------===// 2478 2479class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2480 string real_name = ps.Mnemonic> : 2481 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2482 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2483 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2484 let Inst{31-26} = 0x3a; //encoding 2485 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2486 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2487 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2488 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2489} 2490 2491class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2492 string real_name = ps.Mnemonic> : 2493 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2494 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2495 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2496 let Inst{18-15} = op; 2497 let Inst{25-19} = format; 2498 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2499 let Inst{54} = ps.offen; 2500 let Inst{55} = ps.idxen; 2501} 2502 2503class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2504 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2505 let Inst{12} = ps.offen; 2506 let Inst{13} = ps.idxen; 2507 let Inst{18-16} = op; 2508 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2509 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2510} 2511 2512//===----------------------------------------------------------------------===// 2513// MTBUF - GFX11. 2514//===----------------------------------------------------------------------===// 2515 2516let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2517multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> { 2518 def _BOTHEN_gfx11 : 2519 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2520 def _IDXEN_gfx11 : 2521 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2522 def _OFFEN_gfx11 : 2523 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2524 def _OFFSET_gfx11 : 2525 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2526} 2527 2528multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps> 2529 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2530multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op> 2531 : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2532 2533 2534class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2535 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2536multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name> 2537 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2538 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2539} 2540 2541defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">; 2542defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2543defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2544defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2545defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x000>; 2546defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x001>; 2547defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x002>; 2548defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x003>; 2549defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2550defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2551defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2552defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2553defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x004>; 2554defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x005>; 2555defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x006>; 2556defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x007>; 2557 2558//===----------------------------------------------------------------------===// 2559// MTBUF - GFX10. 2560//===----------------------------------------------------------------------===// 2561 2562class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2563 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2564 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2565 let Inst{25-19} = format; 2566 let Inst{53} = op{3}; 2567} 2568 2569let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2570 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2571 def _BOTHEN_gfx10 : 2572 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2573 def _IDXEN_gfx10 : 2574 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2575 def _OFFEN_gfx10 : 2576 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2577 def _OFFSET_gfx10 : 2578 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2579 } 2580} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2581 2582defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2583defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2584defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2585defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2586defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2587defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2588defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2589defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2590 2591//===----------------------------------------------------------------------===// 2592// MTBUF - GFX6, GFX7, GFX10. 2593//===----------------------------------------------------------------------===// 2594 2595class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2596 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2597 let Inst{15} = ps.addr64; 2598 let Inst{22-19} = dfmt; 2599 let Inst{25-23} = nfmt; 2600} 2601 2602let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2603 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2604 def _ADDR64_gfx6_gfx7 : 2605 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2606 def _BOTHEN_gfx6_gfx7 : 2607 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2608 def _IDXEN_gfx6_gfx7 : 2609 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2610 def _OFFEN_gfx6_gfx7 : 2611 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2612 def _OFFSET_gfx6_gfx7 : 2613 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2614 } 2615} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2616 2617multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2618 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2619 2620defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2621defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2622defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2623defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2624defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2625defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2626defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2627defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2628 2629//===----------------------------------------------------------------------===// 2630// GFX8, GFX9 (VI). 2631//===----------------------------------------------------------------------===// 2632 2633class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2634 bit has_sccb = ps.has_sccb> : 2635 MUBUF_Real<ps>, 2636 Enc64, 2637 SIMCInstr<ps.PseudoInstr, Enc>, 2638 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2639 !if(ps.IsAtomicRet, 1, ?))> { 2640 2641 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2642 let Inst{12} = ps.offen; 2643 let Inst{13} = ps.idxen; 2644 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2645 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2646 let Inst{16} = ps.lds; 2647 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2648 let Inst{24-18} = op; 2649 let Inst{31-26} = 0x38; //encoding 2650 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2651 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2652 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2653 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2654} 2655 2656class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2657 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2658 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2659 let DecoderNamespace = "GFX8"; 2660 2661 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2662} 2663 2664class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2665 bit has_sccb = ps.has_sccb> : 2666 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2667 let AssemblerPredicate = isGFX90APlus; 2668 let DecoderNamespace = "GFX90A"; 2669 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2670 !subst("$tfe", "", ps.AsmOperands)); 2671 2672 let Inst{55} = acc; 2673} 2674 2675class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2676 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2677 let AssemblerPredicate = isGFX940Plus; 2678 let DecoderNamespace = "GFX9"; 2679 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2680 2681 let Inst{55} = acc; 2682} 2683 2684multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2685 def _vi : MUBUF_Real_vi<op, ps>; 2686 2687 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2688 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2689 2690 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2691 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2692 let SubtargetPredicate = isGFX90AOnly; 2693 let AssemblerPredicate = isGFX90AOnly; 2694 } 2695 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2696 } 2697} 2698 2699multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2700 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2701 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2702 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2703 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2704} 2705 2706multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2707 2708 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2709 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2710 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2711 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2712 2713 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2714 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2715 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2716 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2717 2718 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2719 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2720 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2721 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2722 2723 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2724 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2725 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2726 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2727} 2728 2729class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2730 MUBUF_Real<ps>, 2731 Enc64, 2732 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2733 let AssemblerPredicate=HasUnpackedD16VMem; 2734 let DecoderNamespace="GFX80_UNPACKED"; 2735 2736 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2737 let Inst{12} = ps.offen; 2738 let Inst{13} = ps.idxen; 2739 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2740 let Inst{16} = ps.lds; 2741 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2742 let Inst{24-18} = op; 2743 let Inst{31-26} = 0x38; //encoding 2744 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2745 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2746 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2747 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2748 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2749} 2750 2751multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2752 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2753 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2754 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2755 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2756} 2757 2758multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2759 MUBUF_Real_AllAddr_vi<op> { 2760 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2761 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2762 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2763 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2764} 2765 2766defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2767defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2768defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2769defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2770defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2771defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2772defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2773defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2774let SubtargetPredicate = HasUnpackedD16VMem in { 2775 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2776 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2777 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2778 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2779 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2780 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2781 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2782 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2783} // End HasUnpackedD16VMem. 2784let SubtargetPredicate = HasPackedD16VMem in { 2785 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2786 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2787 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2788 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2789 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2790 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2791 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2792 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2793} // End HasPackedD16VMem. 2794defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2795defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2796defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2797defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2798defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2799defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 2800defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 2801defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 2802defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2803defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2804defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2805defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2806defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2807defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2808defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2809defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2810 2811defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2812defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2813defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2814defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2815defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2816defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2817 2818defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2819defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2820 2821defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2822defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2823defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2824defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2825defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2826defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2827defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2828defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2829defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2830defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2831defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2832defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2833defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2834 2835defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2836defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2837defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2838defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2839defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2840defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2841defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2842defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2843defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2844defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2845defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2846defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2847defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2848 2849defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2850 2851let AssemblerPredicate = isGFX8GFX9 in { 2852def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2853def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2854} // End AssemblerPredicate = isGFX8GFX9 2855 2856let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2857 2858defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2859defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2860 2861} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2862 2863let SubtargetPredicate = isGFX90APlus in { 2864 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2865 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2866 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2867} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2868 2869def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2870 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2871 let AssemblerPredicate = isGFX90AOnly; 2872 let SubtargetPredicate = isGFX90AOnly; 2873} 2874def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2875 2876let SubtargetPredicate = isGFX940Plus in { 2877def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2878def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2879} 2880 2881class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2882 MTBUF_Real<ps>, 2883 Enc64, 2884 SIMCInstr<ps.PseudoInstr, Enc> { 2885 2886 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2887 let Inst{12} = ps.offen; 2888 let Inst{13} = ps.idxen; 2889 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2890 let Inst{18-15} = op; 2891 let Inst{22-19} = dfmt; 2892 let Inst{25-23} = nfmt; 2893 let Inst{31-26} = 0x3a; //encoding 2894 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2895 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2896 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2897 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2898 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2899 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2900 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2901} 2902 2903class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2904 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2905 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2906 let DecoderNamespace = "GFX8"; 2907 2908 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2909} 2910 2911class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2912 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2913 let AssemblerPredicate = isGFX90APlus; 2914 let DecoderNamespace = "GFX90A"; 2915 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2916 2917 let Inst{55} = acc; 2918} 2919 2920multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2921 def _vi : MTBUF_Real_vi<op, ps>; 2922 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2923} 2924 2925multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2926 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2927 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2928 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2929 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2930} 2931 2932class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2933 MTBUF_Real<ps>, 2934 Enc64, 2935 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2936 let AssemblerPredicate=HasUnpackedD16VMem; 2937 let DecoderNamespace="GFX80_UNPACKED"; 2938 2939 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2940 let Inst{12} = ps.offen; 2941 let Inst{13} = ps.idxen; 2942 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2943 let Inst{18-15} = op; 2944 let Inst{22-19} = dfmt; 2945 let Inst{25-23} = nfmt; 2946 let Inst{31-26} = 0x3a; //encoding 2947 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2948 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2949 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2950 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2951 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2952 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2953} 2954 2955multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2956 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2957 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2958 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2959 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2960} 2961 2962defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2963defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2964defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2965defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2966defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2967defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2968defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2969defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2970let SubtargetPredicate = HasUnpackedD16VMem in { 2971 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2972 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2973 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2974 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2975 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2976 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2977 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2978 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2979} // End HasUnpackedD16VMem. 2980let SubtargetPredicate = HasPackedD16VMem in { 2981 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2982 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2983 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2984 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2985 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2986 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2987 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2988 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2989} // End HasUnpackedD16VMem. 2990 2991def MUBUFInfoTable : GenericTable { 2992 let FilterClass = "MUBUF_Pseudo"; 2993 let CppTypeName = "MUBUFInfo"; 2994 let Fields = [ 2995 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2996 "IsBufferInv" 2997 ]; 2998 2999 let PrimaryKey = ["Opcode"]; 3000 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3001} 3002 3003def getMUBUFInfoFromOpcode : SearchIndex { 3004 let Table = MUBUFInfoTable; 3005 let Key = ["Opcode"]; 3006} 3007 3008def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3009 let Table = MUBUFInfoTable; 3010 let Key = ["BaseOpcode", "elements"]; 3011} 3012 3013def MTBUFInfoTable : GenericTable { 3014 let FilterClass = "MTBUF_Pseudo"; 3015 let CppTypeName = "MTBUFInfo"; 3016 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3017 3018 let PrimaryKey = ["Opcode"]; 3019 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3020} 3021 3022def getMTBUFInfoFromOpcode : SearchIndex { 3023 let Table = MTBUFInfoTable; 3024 let Key = ["Opcode"]; 3025} 3026 3027def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3028 let Table = MTBUFInfoTable; 3029 let Key = ["BaseOpcode", "elements"]; 3030} 3031