1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MTBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43//===----------------------------------------------------------------------===// 44// MTBUF classes 45//===----------------------------------------------------------------------===// 46 47class MTBUFGetBaseOpcode<string Op> { 48 string ret = !subst("FORMAT_XY", "FORMAT_X", 49 !subst("FORMAT_XYZ", "FORMAT_X", 50 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 51} 52 53 54class MTBUF_Pseudo <string opName, dag outs, dag ins, 55 string asmOps, list<dag> pattern=[]> : 56 InstSI<outs, ins, "", pattern>, 57 SIMCInstr<opName, SIEncodingFamily.NONE> { 58 59 let isPseudo = 1; 60 let isCodeGenOnly = 1; 61 let Size = 8; 62 let UseNamedOperandTable = 1; 63 64 string Mnemonic = opName; 65 string AsmOperands = asmOps; 66 67 Instruction Opcode = !cast<Instruction>(NAME); 68 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 69 70 let VM_CNT = 1; 71 let EXP_CNT = 1; 72 let MTBUF = 1; 73 let Uses = [EXEC]; 74 let hasSideEffects = 0; 75 let SchedRW = [WriteVMEM]; 76 77 let AsmMatchConverter = "cvtMtbuf"; 78 79 bits<1> offen = 0; 80 bits<1> idxen = 0; 81 bits<1> addr64 = 0; 82 bits<1> has_vdata = 1; 83 bits<1> has_vaddr = 1; 84 bits<1> has_glc = 1; 85 bits<1> has_dlc = 1; 86 bits<1> glc_value = 0; // the value for glc if no such operand 87 bits<1> dlc_value = 0; // the value for dlc if no such operand 88 bits<1> has_srsrc = 1; 89 bits<1> has_soffset = 1; 90 bits<1> has_offset = 1; 91 bits<1> has_slc = 1; 92 bits<1> has_tfe = 1; 93 bits<4> elements = 0; 94 bits<1> has_sccb = 1; 95 bits<1> sccb_value = 0; 96} 97 98class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 99 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 100 101 let isPseudo = 0; 102 let isCodeGenOnly = 0; 103 104 let VM_CNT = 1; 105 let EXP_CNT = 1; 106 let MTBUF = 1; 107 108 // copy relevant pseudo op flags 109 let UseNamedOperandTable = ps.UseNamedOperandTable; 110 let SubtargetPredicate = ps.SubtargetPredicate; 111 let AsmMatchConverter = ps.AsmMatchConverter; 112 let Constraints = ps.Constraints; 113 let DisableEncoding = ps.DisableEncoding; 114 let TSFlags = ps.TSFlags; 115 let SchedRW = ps.SchedRW; 116 let mayLoad = ps.mayLoad; 117 let mayStore = ps.mayStore; 118 let IsAtomicRet = ps.IsAtomicRet; 119 let IsAtomicNoRet = ps.IsAtomicNoRet; 120 121 bits<12> offset; 122 bits<5> cpol; 123 bits<7> format; 124 bits<8> vaddr; 125 bits<10> vdata; 126 bits<7> srsrc; 127 bits<1> tfe; 128 bits<8> soffset; 129 130 bits<4> dfmt = format{3-0}; 131 bits<3> nfmt = format{6-4}; 132 133 // GFX90A+ only: instruction uses AccVGPR for data 134 // Bit supersedes tfe. 135 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdata_op:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 152 TFE:$tfe, SWZ:$swz), 153 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 155 TFE:$tfe, SWZ:$swz) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc,$format $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc,$format $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc,$format $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc,$format $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 187 !eq(addrKind, BUFAddrKind.BothEn)); 188 189 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 190 !eq(addrKind, BUFAddrKind.BothEn)); 191 192 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 193 194 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 int elems, 201 list<dag> pattern=[], 202 // Workaround bug bz30254 203 int addrKindCopy = addrKind> 204 : MTBUF_Pseudo<opName, 205 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 206 getMTBUFIns<addrKindCopy>.ret, 207 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 208 pattern>, 209 MTBUF_SetupAddr<addrKindCopy> { 210 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 211 let mayLoad = 1; 212 let mayStore = 0; 213 let elements = elems; 214} 215 216multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 217 int elems> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 220 MTBUFAddr64Table<0, NAME>; 221 222 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 223 MTBUFAddr64Table<1, NAME>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 int elems, 241 list<dag> pattern=[], 242 // Workaround bug bz30254 243 int addrKindCopy = addrKind, 244 RegisterClass vdataClassCopy = vdataClass> 245 : MTBUF_Pseudo<opName, 246 (outs), 247 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 248 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 249 pattern>, 250 MTBUF_SetupAddr<addrKindCopy> { 251 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 252 let mayLoad = 0; 253 let mayStore = 1; 254 let elements = elems; 255} 256 257multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 258 int elems> { 259 260 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 261 MTBUFAddr64Table<0, NAME>; 262 263 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 264 MTBUFAddr64Table<1, NAME>; 265 266 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 267 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 268 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 269 270 let DisableWQM = 1 in { 271 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 272 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 273 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 274 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 275 } 276} 277 278 279//===----------------------------------------------------------------------===// 280// MUBUF classes 281//===----------------------------------------------------------------------===// 282 283class MUBUFGetBaseOpcode<string Op> { 284 string ret = !subst("DWORDX2", "DWORD", 285 !subst("DWORDX3", "DWORD", 286 !subst("DWORDX4", "DWORD", Op))); 287} 288 289class MUBUF_Pseudo <string opName, dag outs, dag ins, 290 string asmOps, list<dag> pattern=[]> : 291 InstSI<outs, ins, "", pattern>, 292 SIMCInstr<opName, SIEncodingFamily.NONE> { 293 294 let isPseudo = 1; 295 let isCodeGenOnly = 1; 296 let Size = 8; 297 let UseNamedOperandTable = 1; 298 299 string Mnemonic = opName; 300 string AsmOperands = asmOps; 301 302 Instruction Opcode = !cast<Instruction>(NAME); 303 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 304 305 let VM_CNT = 1; 306 let EXP_CNT = 1; 307 let MUBUF = 1; 308 let Uses = [EXEC]; 309 let hasSideEffects = 0; 310 let SchedRW = [WriteVMEM]; 311 312 let AsmMatchConverter = "cvtMubuf"; 313 314 bits<1> offen = 0; 315 bits<1> idxen = 0; 316 bits<1> addr64 = 0; 317 bits<1> lds = 0; 318 bits<1> has_vdata = !not(lds); 319 bits<1> has_vaddr = 1; 320 bits<1> has_glc = 1; 321 bits<1> has_dlc = 1; 322 bits<1> glc_value = 0; // the value for glc if no such operand 323 bits<1> dlc_value = 0; // the value for dlc if no such operand 324 bits<1> has_srsrc = 1; 325 bits<1> has_soffset = 1; 326 bits<1> has_offset = 1; 327 bits<1> has_slc = 1; 328 bits<1> has_tfe = 1; 329 bits<4> elements = 0; 330 bits<1> has_sccb = 1; 331 bits<1> sccb_value = 0; 332 bits<1> IsBufferInv = 0; 333} 334 335class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 336 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 337 338 let isPseudo = 0; 339 let isCodeGenOnly = 0; 340 341 let VM_CNT = 1; 342 let EXP_CNT = 1; 343 let MUBUF = 1; 344 345 // copy relevant pseudo op flags 346 let SubtargetPredicate = ps.SubtargetPredicate; 347 let AsmMatchConverter = ps.AsmMatchConverter; 348 let OtherPredicates = ps.OtherPredicates; 349 let Constraints = ps.Constraints; 350 let DisableEncoding = ps.DisableEncoding; 351 let TSFlags = ps.TSFlags; 352 let UseNamedOperandTable = ps.UseNamedOperandTable; 353 let SchedRW = ps.SchedRW; 354 let mayLoad = ps.mayLoad; 355 let mayStore = ps.mayStore; 356 let IsAtomicRet = ps.IsAtomicRet; 357 let IsAtomicNoRet = ps.IsAtomicNoRet; 358 let VALU = ps.VALU; 359 let LGKM_CNT = ps.LGKM_CNT; 360 361 bits<12> offset; 362 bits<5> cpol; 363 bits<8> vaddr; 364 bits<10> vdata; 365 bits<7> srsrc; 366 bits<1> tfe; 367 bits<8> soffset; 368 369 // GFX90A+ only: instruction uses AccVGPR for data 370 // Bit supersedes tfe. 371 bits<1> acc = !if(ps.has_vdata, vdata{9}, !if(ps.lds, ?, 0)); 372} 373 374 375// For cache invalidation instructions. 376class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 377 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 378 379 let AsmMatchConverter = ""; 380 381 let hasSideEffects = 1; 382 let mayLoad = 0; 383 let mayStore = 0; 384 385 let IsBufferInv = 1; 386 // Set everything else to 0. 387 let offen = 0; 388 let idxen = 0; 389 let addr64 = 0; 390 let has_vdata = 0; 391 let has_vaddr = 0; 392 let has_glc = 0; 393 let has_dlc = 0; 394 let glc_value = 0; 395 let dlc_value = 0; 396 let has_srsrc = 0; 397 let has_soffset = 0; 398 let has_offset = 0; 399 let has_slc = 0; 400 let has_tfe = 0; 401 let has_sccb = 0; 402 let sccb_value = 0; 403} 404 405class getMUBUFInsDA<list<RegisterClass> vdataList, 406 list<RegisterClass> vaddrList=[], 407 bit isLds = 0> { 408 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 409 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 410 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 411 dag InsNoData = !if(!empty(vaddrList), 412 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 413 offset:$offset, CPol_0:$cpol), 414 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 415 offset:$offset, CPol_0:$cpol) 416 ); 417 dag InsData = !if(!empty(vaddrList), 418 (ins vdata_op:$vdata, SReg_128:$srsrc, 419 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 420 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 421 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 422 ); 423 dag ret = !con( 424 !if(!empty(vdataList), InsNoData, InsData), 425 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 426 ); 427} 428 429class getMUBUFElements<ValueType vt> { 430 int ret = 431 !if(!eq(vt, f16), 1, 432 !if(!eq(vt, v2f16), 2, 433 !if(!eq(vt, v3f16), 3, 434 !if(!eq(vt, v4f16), 4, 435 !if(!eq(vt.Size, 32), 1, 436 !if(!eq(vt.Size, 64), 2, 437 !if(!eq(vt.Size, 96), 3, 438 !if(!eq(vt.Size, 128), 4, 0) 439 ) 440 ) 441 ) 442 ) 443 ) 444 ) 445 ); 446} 447 448class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 449 dag ret = 450 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 451 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 452 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 453 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 454 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 455 (ins)))))); 456} 457 458class getMUBUFAsmOps<int addrKind> { 459 string Pfx = 460 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 461 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 462 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 463 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 464 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 465 ""))))); 466 string ret = Pfx # "$offset"; 467} 468 469class MUBUF_SetupAddr<int addrKind> { 470 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 471 !eq(addrKind, BUFAddrKind.BothEn)); 472 473 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 474 !eq(addrKind, BUFAddrKind.BothEn)); 475 476 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 477 478 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 479} 480 481class MUBUF_Load_Pseudo <string opName, 482 int addrKind, 483 ValueType vdata_vt, 484 bit HasTiedDest = 0, 485 bit isLds = 0, 486 bit isLdsOpc = 0, 487 list<dag> pattern=[], 488 // Workaround bug bz30254 489 int addrKindCopy = addrKind, 490 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 491 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 492 : MUBUF_Pseudo<opName, 493 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 494 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 495 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 496 !if(!or(isLds, isLdsOpc), " ", " $vdata, ") # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 497 !if(isLds, " lds", "$tfe") # "$swz", 498 pattern>, 499 MUBUF_SetupAddr<addrKindCopy> { 500 let PseudoInstr = opName # !if(isLds, "_lds", "") # 501 "_" # getAddrName<addrKindCopy>.ret; 502 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 503 504 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 505 let LGKM_CNT = isLds; 506 let has_vdata = !not(isLdsOpc); 507 let mayLoad = 1; 508 let mayStore = isLds; 509 let maybeAtomic = 1; 510 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 511 let has_tfe = !not(isLds); 512 let lds = isLds; 513 let elements = getMUBUFElements<vdata_vt>.ret; 514 let VALU = isLds; 515} 516 517class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 518 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 519 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 520>; 521 522class MUBUF_Addr64_Load_Pat <Instruction inst, 523 ValueType load_vt = i32, 524 SDPatternOperator ld = null_frag> : Pat < 525 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 526 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 527>; 528 529multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 530 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 531 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 532} 533 534 535// FIXME: tfe can't be an operand because it requires a separate 536// opcode because it needs an N+1 register class dest register. 537multiclass MUBUF_Pseudo_Loads<string opName, 538 ValueType load_vt = i32, 539 bit TiedDest = 0, 540 bit isLds = 0> { 541 542 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 543 544 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 545 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 546 547 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 548 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 549 550 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 551 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 552 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 553 554 let DisableWQM = 1 in { 555 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 556 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 557 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 558 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 559 } 560} 561 562multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 563 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 564 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 565} 566 567multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 568 ValueType load_vt = i32, 569 bit TiedDest = 0, 570 bit isLds = 0, 571 bit isLdsOpc = 1> { 572 573 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 574 575 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 576 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 577 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 578 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 579} 580 581class MUBUF_Store_Pseudo <string opName, 582 int addrKind, 583 ValueType store_vt, 584 list<dag> pattern=[], 585 // Workaround bug bz30254 586 int addrKindCopy = addrKind> 587 : MUBUF_Pseudo<opName, 588 (outs), 589 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 590 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 591 pattern>, 592 MUBUF_SetupAddr<addrKindCopy> { 593 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 594 let mayLoad = 0; 595 let mayStore = 1; 596 let maybeAtomic = 1; 597 let elements = getMUBUFElements<store_vt>.ret; 598} 599 600multiclass MUBUF_Pseudo_Stores<string opName, 601 ValueType store_vt = i32, 602 SDPatternOperator st = null_frag> { 603 604 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 605 606 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 607 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 608 i16:$offset))]>, 609 MUBUFAddr64Table<0, NAME>; 610 611 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 612 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 613 i16:$offset))]>, 614 MUBUFAddr64Table<1, NAME>; 615 616 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 617 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 618 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 619 620 let DisableWQM = 1 in { 621 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 622 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 623 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 624 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 625 } 626} 627 628class MUBUF_Pseudo_Store_Lds<string opName> 629 : MUBUF_Pseudo<opName, 630 (outs), 631 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 632 " $srsrc, $soffset$offset lds$cpol$swz"> { 633 let LGKM_CNT = 1; 634 let mayLoad = 1; 635 let mayStore = 1; 636 let maybeAtomic = 1; 637 638 let has_vdata = 0; 639 let has_vaddr = 0; 640 let has_tfe = 0; 641 let lds = 1; 642 let VALU = 1; 643 644 let Uses = [EXEC, M0]; 645 let AsmMatchConverter = "cvtMubufLds"; 646} 647 648class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 649 list<RegisterClass> vaddrList=[]> { 650 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 651 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 652 dag ret = !if(vdata_in, 653 !if(!empty(vaddrList), 654 (ins vdata_op:$vdata_in, 655 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 656 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 657 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 658 ), 659 !if(!empty(vaddrList), 660 (ins vdata_op:$vdata, 661 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 662 (ins vdata_op:$vdata, vaddrClass:$vaddr, 663 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 664 )); 665} 666 667class getMUBUFAtomicIns<int addrKind, 668 RegisterClass vdataClass, 669 bit vdata_in, 670 // Workaround bug bz30254 671 RegisterClass vdataClassCopy=vdataClass> { 672 dag ret = 673 !if(!eq(addrKind, BUFAddrKind.Offset), 674 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 675 !if(!eq(addrKind, BUFAddrKind.OffEn), 676 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 677 !if(!eq(addrKind, BUFAddrKind.IdxEn), 678 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 679 !if(!eq(addrKind, BUFAddrKind.BothEn), 680 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 681 !if(!eq(addrKind, BUFAddrKind.Addr64), 682 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 683 (ins)))))); 684} 685 686class MUBUF_Atomic_Pseudo<string opName, 687 int addrKind, 688 dag outs, 689 dag ins, 690 string asmOps, 691 list<dag> pattern=[], 692 // Workaround bug bz30254 693 int addrKindCopy = addrKind> 694 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 695 MUBUF_SetupAddr<addrKindCopy> { 696 let mayStore = 1; 697 let mayLoad = 1; 698 let hasPostISelHook = 1; 699 let hasSideEffects = 1; 700 let DisableWQM = 1; 701 let has_glc = 0; 702 let has_dlc = 0; 703 let has_tfe = 0; 704 let has_sccb = 1; 705 let maybeAtomic = 1; 706 let AsmMatchConverter = "cvtMubufAtomic"; 707} 708 709class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 710 RegisterClass vdataClass, 711 list<dag> pattern=[], 712 // Workaround bug bz30254 713 int addrKindCopy = addrKind, 714 RegisterClass vdataClassCopy = vdataClass> 715 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 716 (outs), 717 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 718 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 719 pattern>, 720 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 721 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 722 let glc_value = 0; 723 let dlc_value = 0; 724 let sccb_value = 0; 725 let IsAtomicNoRet = 1; 726} 727 728class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 729 RegisterClass vdataClass, 730 list<dag> pattern=[], 731 // Workaround bug bz30254 732 int addrKindCopy = addrKind, 733 RegisterClass vdataClassCopy = vdataClass, 734 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 735 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 736 (outs vdata_op:$vdata), 737 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 738 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 739 pattern>, 740 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 741 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 742 let glc_value = 1; 743 let dlc_value = 0; 744 let sccb_value = 0; 745 let IsAtomicRet = 1; 746 let Constraints = "$vdata = $vdata_in"; 747 let DisableEncoding = "$vdata_in"; 748} 749 750multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 751 RegisterClass vdataClass, 752 ValueType vdataType, 753 bit isFP = isFloatType<vdataType>.ret> { 754 let FPAtomic = isFP in 755 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 756 MUBUFAddr64Table <0, NAME>; 757 758 let FPAtomic = isFP in 759 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 760 MUBUFAddr64Table <1, NAME>; 761 762 let FPAtomic = isFP in 763 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 764 765 let FPAtomic = isFP in 766 767 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 768 769 let FPAtomic = isFP in 770 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 771} 772 773multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 774 RegisterClass vdataClass, 775 ValueType vdataType, 776 SDPatternOperator atomic, 777 bit isFP = isFloatType<vdataType>.ret> { 778 let FPAtomic = isFP in 779 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 780 [(set vdataType:$vdata, 781 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 782 vdataType:$vdata_in))]>, 783 MUBUFAddr64Table <0, NAME # "_RTN">; 784 785 let FPAtomic = isFP in 786 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 787 [(set vdataType:$vdata, 788 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 789 vdataType:$vdata_in))]>, 790 MUBUFAddr64Table <1, NAME # "_RTN">; 791 792 let FPAtomic = isFP in 793 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 794 795 let FPAtomic = isFP in 796 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 797 798 let FPAtomic = isFP in 799 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 800} 801 802multiclass MUBUF_Pseudo_Atomics <string opName, 803 RegisterClass vdataClass, 804 ValueType vdataType, 805 SDPatternOperator atomic = null_frag> : 806 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 807 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 808 809 810//===----------------------------------------------------------------------===// 811// MUBUF Instructions 812//===----------------------------------------------------------------------===// 813 814defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 815 "buffer_load_format_x", f32 816>; 817defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 818 "buffer_load_format_xy", v2f32 819>; 820defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 821 "buffer_load_format_xyz", v3f32 822>; 823defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 824 "buffer_load_format_xyzw", v4f32 825>; 826defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 827 "buffer_store_format_x", f32 828>; 829defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 830 "buffer_store_format_xy", v2f32 831>; 832defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 833 "buffer_store_format_xyz", v3f32 834>; 835defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 836 "buffer_store_format_xyzw", v4f32 837>; 838 839let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 840 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_x", i32 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xy", v2i32 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xyz", v3i32 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyzw", v4i32 851 >; 852 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_x", i32 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xy", v2i32 857 >; 858 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 859 "buffer_store_format_d16_xyz", v3i32 860 >; 861 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 862 "buffer_store_format_d16_xyzw", v4i32 863 >; 864} // End HasUnpackedD16VMem. 865 866let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 867 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 868 "buffer_load_format_d16_x", f16 869 >; 870 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 871 "buffer_load_format_d16_xy", v2f16 872 >; 873 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 874 "buffer_load_format_d16_xyz", v3f16 875 >; 876 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 877 "buffer_load_format_d16_xyzw", v4f16 878 >; 879 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 880 "buffer_store_format_d16_x", f16 881 >; 882 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 883 "buffer_store_format_d16_xy", v2f16 884 >; 885 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 886 "buffer_store_format_d16_xyz", v3f16 887 >; 888 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 889 "buffer_store_format_d16_xyzw", v4f16 890 >; 891} // End HasPackedD16VMem. 892 893defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 894 "buffer_load_ubyte", i32 895>; 896defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 897 "buffer_load_sbyte", i32 898>; 899defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 900 "buffer_load_ushort", i32 901>; 902defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 903 "buffer_load_sshort", i32 904>; 905defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 906 "buffer_load_dword", i32 907>; 908defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 909 "buffer_load_dwordx2", v2i32 910>; 911defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 912 "buffer_load_dwordx3", v3i32 913>; 914defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 915 "buffer_load_dwordx4", v4i32 916>; 917 918defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 919 "buffer_load_lds_b32", i32 920>; 921defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 922 "buffer_load_lds_format_x", f32 923>; 924defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 925 "buffer_load_lds_i8", i32 926>; 927defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 928 "buffer_load_lds_i16", i32 929>; 930defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 931 "buffer_load_lds_u8", i32 932>; 933defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 934 "buffer_load_lds_u16", i32 935>; 936 937defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 938defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 939defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 940defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 941defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 942defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 943defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 944defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 945defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 946defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 947defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 948defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 949defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 950defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 951 952defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 953 "buffer_store_byte", i32, truncstorei8_global 954>; 955defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 956 "buffer_store_short", i32, truncstorei16_global 957>; 958defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 959 "buffer_store_dword", i32, store_global 960>; 961defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 962 "buffer_store_dwordx2", v2i32, store_global 963>; 964defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 965 "buffer_store_dwordx3", v3i32, store_global 966>; 967defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 968 "buffer_store_dwordx4", v4i32, store_global 969>; 970defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_swap", VGPR_32, i32 972>; 973defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_cmpswap", VReg_64, v2i32 975>; 976defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_add", VGPR_32, i32 978>; 979defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_sub", VGPR_32, i32 981>; 982defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_smin", VGPR_32, i32 984>; 985defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_umin", VGPR_32, i32 987>; 988defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_smax", VGPR_32, i32 990>; 991defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_umax", VGPR_32, i32 993>; 994defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_and", VGPR_32, i32 996>; 997defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_or", VGPR_32, i32 999>; 1000defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_xor", VGPR_32, i32 1002>; 1003defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_inc", VGPR_32, i32 1005>; 1006defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_dec", VGPR_32, i32 1008>; 1009defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_swap_x2", VReg_64, i64 1011>; 1012defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1014>; 1015defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_add_x2", VReg_64, i64 1017>; 1018defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_sub_x2", VReg_64, i64 1020>; 1021defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_smin_x2", VReg_64, i64 1023>; 1024defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_umin_x2", VReg_64, i64 1026>; 1027defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_smax_x2", VReg_64, i64 1029>; 1030defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_umax_x2", VReg_64, i64 1032>; 1033defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1034 "buffer_atomic_and_x2", VReg_64, i64 1035>; 1036defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1037 "buffer_atomic_or_x2", VReg_64, i64 1038>; 1039defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1040 "buffer_atomic_xor_x2", VReg_64, i64 1041>; 1042defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1043 "buffer_atomic_inc_x2", VReg_64, i64 1044>; 1045defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1046 "buffer_atomic_dec_x2", VReg_64, i64 1047>; 1048 1049let SubtargetPredicate = HasGFX10_BEncoding in 1050defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1051 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1052>; 1053 1054let SubtargetPredicate = isGFX8GFX9 in { 1055def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1056} 1057 1058let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1059/* 1060defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1061defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1062*/ 1063 1064def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1065 int_amdgcn_buffer_wbinvl1_sc>; 1066} 1067 1068let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1069 1070defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1072>; 1073defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1074 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1075>; 1076defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1077 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1078>; 1079 1080} 1081 1082let SubtargetPredicate = isGFX6GFX7GFX10 in { 1083 1084defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1085 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1086>; 1087defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1088 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1089>; 1090defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1091 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1092>; 1093 1094} 1095 1096let SubtargetPredicate = HasD16LoadStore in { 1097 1098defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1099 "buffer_load_ubyte_d16", i32, 1 1100>; 1101 1102defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1103 "buffer_load_ubyte_d16_hi", i32, 1 1104>; 1105 1106defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1107 "buffer_load_sbyte_d16", i32, 1 1108>; 1109 1110defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1111 "buffer_load_sbyte_d16_hi", i32, 1 1112>; 1113 1114defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1115 "buffer_load_short_d16", i32, 1 1116>; 1117 1118defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1119 "buffer_load_short_d16_hi", i32, 1 1120>; 1121 1122defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1123 "buffer_store_byte_d16_hi", i32 1124>; 1125 1126defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1127 "buffer_store_short_d16_hi", i32 1128>; 1129 1130defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1131 "buffer_load_format_d16_hi_x", i32 1132>; 1133defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1134 "buffer_store_format_d16_hi_x", i32 1135>; 1136 1137} // End HasD16LoadStore 1138 1139def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1140 int_amdgcn_buffer_wbinvl1>; 1141 1142let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1143defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1144 "buffer_atomic_add_f32", VGPR_32, f32 1145>; 1146 1147let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1148defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1149 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1150>; 1151 1152let OtherPredicates = [HasAtomicFaddRtnInsts] in 1153defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1154 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1155>; 1156 1157let OtherPredicates = [isGFX90APlus] in 1158defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1159 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1160>; 1161 1162//===----------------------------------------------------------------------===// 1163// MTBUF Instructions 1164//===----------------------------------------------------------------------===// 1165 1166defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1167defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1168defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1169defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1170defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1171defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1172defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1173defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1174 1175let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1176 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1177 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1178 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1179 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1180 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1181 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1182 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1183 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1184} // End HasUnpackedD16VMem. 1185 1186let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1187 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1188 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1189 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1190 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1191 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1192 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1193 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1194 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1195} // End HasPackedD16VMem. 1196 1197let SubtargetPredicate = isGFX7Plus in { 1198 1199//===----------------------------------------------------------------------===// 1200// Instruction definitions for CI and newer. 1201//===----------------------------------------------------------------------===// 1202 1203def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1204 int_amdgcn_buffer_wbinvl1_vol>; 1205 1206} // End let SubtargetPredicate = isGFX7Plus 1207 1208let SubtargetPredicate = isGFX90APlus in { 1209 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1210 let has_glc = 1; 1211 let has_sccb = 1; 1212 let InOperandList = (ins CPol_0:$cpol); 1213 let AsmOperands = "$cpol"; 1214 } 1215 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1216 let SubtargetPredicate = isGFX90AOnly; 1217 } 1218 1219 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1220 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1221 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1222} // End SubtargetPredicate = isGFX90APlus 1223 1224def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1225 let SubtargetPredicate = isGFX940Plus; 1226 let has_glc = 1; 1227 let has_sccb = 1; 1228 let InOperandList = (ins CPol_0:$cpol); 1229 let AsmOperands = "$cpol"; 1230} 1231 1232let SubtargetPredicate = isGFX10Plus in { 1233 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1234 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1235} // End SubtargetPredicate = isGFX10Plus 1236 1237//===----------------------------------------------------------------------===// 1238// MUBUF Patterns 1239//===----------------------------------------------------------------------===// 1240 1241//===----------------------------------------------------------------------===// 1242// buffer_load/store_format patterns 1243//===----------------------------------------------------------------------===// 1244 1245multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1246 string opcode, ValueType memoryVt = vt> { 1247 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1248 1249 def : GCNPat< 1250 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1251 timm:$auxiliary, 0)), 1252 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1253 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1254 >; 1255 1256 def : GCNPat< 1257 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1258 timm:$auxiliary, 0)), 1259 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1260 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1261 >; 1262 1263 def : GCNPat< 1264 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1265 timm:$auxiliary, timm)), 1266 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1267 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1268 >; 1269 1270 def : GCNPat< 1271 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1272 timm:$auxiliary, timm)), 1273 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1274 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1275 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1276 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1277 >; 1278} 1279 1280defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1281defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1282defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1283defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1288 1289let SubtargetPredicate = HasUnpackedD16VMem in { 1290 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1291 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1292 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1293 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1294 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1295 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1296} // End HasUnpackedD16VMem. 1297 1298let SubtargetPredicate = HasPackedD16VMem in { 1299 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1300 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1301 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1302 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1303 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1304 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1305 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1306 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1307 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1308} // End HasPackedD16VMem. 1309 1310defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1311defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1312defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1313defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1314defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1315defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1316defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1317defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1318defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1319defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1320defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1321defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1322defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1323defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1324defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1325defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1326 1327multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1328 string opcode, ValueType memoryVt = vt> { 1329 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1330 1331 def : GCNPat< 1332 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1333 timm:$auxiliary, 0), 1334 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1335 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1336 >; 1337 1338 def : GCNPat< 1339 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1340 timm:$auxiliary, 0), 1341 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1342 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1343 >; 1344 1345 def : GCNPat< 1346 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1347 timm:$auxiliary, timm), 1348 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1349 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1350 >; 1351 1352 def : GCNPat< 1353 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1354 timm:$auxiliary, timm), 1355 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1356 getVregSrcForVT<vt>.ret:$vdata, 1357 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1358 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1359 0, (extract_swz $auxiliary)) 1360 >; 1361} 1362 1363defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1364defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1365defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1366defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1371 1372let SubtargetPredicate = HasUnpackedD16VMem in { 1373 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1374 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1375 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1376 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1377 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1378 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1379} // End HasUnpackedD16VMem. 1380 1381let SubtargetPredicate = HasPackedD16VMem in { 1382 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1383 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1384 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1385 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1386 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1387 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1388 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1389 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1390 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1391} // End HasPackedD16VMem. 1392 1393defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1394defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1395defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1396defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1397defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1398defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1399defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1400defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1401defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1402defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1403defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1404defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1405defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1406defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1407 1408//===----------------------------------------------------------------------===// 1409// buffer_atomic patterns 1410//===----------------------------------------------------------------------===// 1411 1412multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1413 foreach RtnMode = ["ret", "noret"] in { 1414 1415 defvar Op = !cast<SDPatternOperator>(OpPrefix # "_" # RtnMode 1416 # !if(isIntr, "", "_" # vt.Size)); 1417 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1418 1419 def : GCNPat< 1420 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1421 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1422 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1423 >; 1424 1425 def : GCNPat< 1426 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1427 vt:$vdata_in)), 1428 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1429 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1430 >; 1431 1432 } // end foreach RtnMode 1433} 1434 1435multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1436 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1437} 1438 1439multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1440 foreach RtnMode = ["ret", "noret"] in { 1441 1442 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode 1443 # "_" # vt.Size); 1444 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1445 1446 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1447 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1448 offset:$offset); 1449 def : GCNPat< 1450 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1451 !if(!eq(RtnMode, "ret"), 1452 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1453 !if(!eq(vt, i32), sub0, sub0_sub1)), 1454 OffsetResDag) 1455 >; 1456 1457 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1458 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1459 SCSrc_b32:$soffset, offset:$offset); 1460 def : GCNPat< 1461 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1462 data_vt:$vdata_in)), 1463 !if(!eq(RtnMode, "ret"), 1464 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1465 !if(!eq(vt, i32), sub0, sub0_sub1)), 1466 Addr64ResDag) 1467 >; 1468 1469 } // end foreach RtnMode 1470} 1471 1472foreach Ty = [i32, i64] in { 1473 1474defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1475 1476defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1477defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1478defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1479defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1480defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1481defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1482defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1483defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1484defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1485defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1486defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1487defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1488 1489} // end foreach Ty 1490 1491defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1492defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1493 1494multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1495 list<string> RtnModes = ["ret", "noret"]> { 1496 foreach RtnMode = RtnModes in { 1497 1498 defvar Op = !cast<SDPatternOperator>(!if(!eq(RtnMode, "none"), 1499 OpPrefix, OpPrefix # "_" # RtnMode)); 1500 defvar InstSuffix = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1501 "_RTN", ""); 1502 defvar CachePolicy = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1503 (set_glc $cachepolicy), (timm:$cachepolicy)); 1504 1505 def : GCNPat< 1506 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1507 timm:$offset, timm:$cachepolicy, 0)), 1508 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1509 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1510 (as_i16timm $offset), CachePolicy) 1511 >; 1512 1513 def : GCNPat< 1514 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1515 timm:$offset, timm:$cachepolicy, timm)), 1516 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1517 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1518 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1519 >; 1520 1521 def : GCNPat< 1522 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1523 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1524 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1525 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1526 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1527 >; 1528 1529 def : GCNPat< 1530 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1531 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1532 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1533 getVregSrcForVT<vt>.ret:$vdata_in, 1534 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1535 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1536 >; 1537 1538 } // end foreach RtnMode 1539} 1540 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1545defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1546defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1547defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1548defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1549defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1550defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1551defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1552defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1553defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1554defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["none"]>; 1555defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1556defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1557defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1558defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1559defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1560defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1561defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1562defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1563defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1564defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1565defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1566defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1567 1568let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1569 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1570 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1571} 1572let SubtargetPredicate = isGFX6GFX7GFX10 in { 1573 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1574 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1575} 1576 1577class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1578 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1579 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1580 [{ return SDValue(N, 0).use_empty(); }]> { 1581 1582 let GISelPredicateCode = [{ 1583 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1584 }]; 1585} 1586 1587multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1588 string opcode> { 1589 def : GCNPat< 1590 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1591 0, i32:$soffset, timm:$offset, 1592 timm:$cachepolicy, 0), 1593 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1594 (as_i16timm $offset), $cachepolicy) 1595 >; 1596 1597 def : GCNPat< 1598 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1599 0, i32:$soffset, timm:$offset, 1600 timm:$cachepolicy, timm), 1601 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1602 (as_i16timm $offset), $cachepolicy) 1603 >; 1604 1605 def : GCNPat< 1606 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1607 i32:$voffset, i32:$soffset, timm:$offset, 1608 timm:$cachepolicy, 0), 1609 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1610 (as_i16timm $offset), $cachepolicy) 1611 >; 1612 1613 def : GCNPat< 1614 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1615 i32:$voffset, i32:$soffset, timm:$offset, 1616 timm:$cachepolicy, timm), 1617 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1618 getVregSrcForVT<vt>.ret:$vdata_in, 1619 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1620 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1621 >; 1622} 1623 1624let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1625defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1626 1627let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1628defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1629 1630let SubtargetPredicate = HasAtomicFaddRtnInsts in 1631 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1632 1633let SubtargetPredicate = isGFX90APlus in { 1634 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1635 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1636 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1637 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1638 1639 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1640 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1641 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1642} // End SubtargetPredicate = isGFX90APlus 1643 1644foreach RtnMode = ["ret", "noret"] in { 1645 1646defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap # "_" # RtnMode); 1647defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1648defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1649 (timm:$cachepolicy)); 1650 1651defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1652 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1653 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1654def : GCNPat< 1655 (Op 1656 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1657 timm:$offset, timm:$cachepolicy, 0), 1658 !if(!eq(RtnMode, "ret"), 1659 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1660 OffsetResDag) 1661>; 1662 1663defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1664 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1665 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1666 CachePolicy); 1667def : GCNPat< 1668 (Op 1669 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1670 0, i32:$soffset, timm:$offset, 1671 timm:$cachepolicy, timm), 1672 !if(!eq(RtnMode, "ret"), 1673 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1674 IdxenResDag) 1675>; 1676 1677defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1678 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1679 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1680 CachePolicy); 1681def : GCNPat< 1682 (Op 1683 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1684 i32:$voffset, i32:$soffset, timm:$offset, 1685 timm:$cachepolicy, 0), 1686 !if(!eq(RtnMode, "ret"), 1687 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1688 OffenResDag) 1689>; 1690 1691defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1692 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1693 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1694 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1695def : GCNPat< 1696 (Op 1697 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1698 i32:$voffset, i32:$soffset, timm:$offset, 1699 timm:$cachepolicy, timm), 1700 !if(!eq(RtnMode, "ret"), 1701 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1702 BothenResDag) 1703>; 1704 1705} // end foreach RtnMode 1706 1707class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1708 PatFrag constant_ld> : GCNPat < 1709 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1710 i16:$offset))), 1711 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1712 >; 1713 1714multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1715 ValueType vt, PatFrag atomic_ld> { 1716 def : GCNPat < 1717 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1718 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1719 >; 1720 1721 def : GCNPat < 1722 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1723 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1724 >; 1725} 1726 1727let SubtargetPredicate = isGFX6GFX7 in { 1728def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1729def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1730def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1731def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1732def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1733def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1734 1735defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1736defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1737} // End SubtargetPredicate = isGFX6GFX7 1738 1739multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1740 PatFrag ld> { 1741 1742 def : GCNPat < 1743 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1744 (Instr_OFFSET $srsrc, $soffset, $offset) 1745 >; 1746} 1747 1748let OtherPredicates = [Has16BitInsts] in { 1749 1750defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1751defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1752defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1753defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1754defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1755defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1756 1757defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1758 1759} // End OtherPredicates = [Has16BitInsts] 1760 1761multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1762 MUBUF_Pseudo InstrOffset, 1763 ValueType vt, PatFrag ld> { 1764 def : GCNPat < 1765 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1766 i32:$soffset, u16imm:$offset))), 1767 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1768 >; 1769 1770 def : GCNPat < 1771 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1772 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1773 >; 1774} 1775 1776// XXX - Is it possible to have a complex pattern in a PatFrag? 1777multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1778 MUBUF_Pseudo InstrOffset, 1779 ValueType vt, PatFrag ld_frag> { 1780 def : GCNPat < 1781 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1782 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1783 >; 1784 1785 def : GCNPat < 1786 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1787 (InstrOffset $srsrc, $soffset, $offset, $in) 1788 >; 1789} 1790 1791let OtherPredicates = [DisableFlatScratch] in { 1792defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1793defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1794defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1795defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1796defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1797defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1798defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1799defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1800defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1801defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1802 1803foreach vt = Reg32Types.types in { 1804defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1805} 1806defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1807defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1808defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1809 1810let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1811defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1812defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1813defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1814defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1815defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1816defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1817 1818defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1819defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1820defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1821defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1822defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1823defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1824} 1825 1826} // End OtherPredicates = [DisableFlatScratch] 1827 1828multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1829 ValueType vt, PatFrag atomic_st> { 1830 // Store follows atomic op convention so address is first 1831 def : GCNPat < 1832 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1833 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1834 >; 1835 1836 def : GCNPat < 1837 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1838 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1839 >; 1840} 1841let SubtargetPredicate = isGFX6GFX7 in { 1842defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1843defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1844defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1845defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1846defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1847defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1848} // End Predicates = isGFX6GFX7 1849 1850 1851multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1852 PatFrag st> { 1853 1854 def : GCNPat < 1855 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1856 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1857 >; 1858} 1859 1860defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1861defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1862 1863multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1864 MUBUF_Pseudo InstrOffset, 1865 ValueType vt, PatFrag st, 1866 RegisterClass rc = VGPR_32> { 1867 def : GCNPat < 1868 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1869 i32:$soffset, u16imm:$offset)), 1870 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1871 >; 1872 1873 def : GCNPat < 1874 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1875 u16imm:$offset)), 1876 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1877 >; 1878} 1879 1880let OtherPredicates = [DisableFlatScratch] in { 1881defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1882defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1883defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1884defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1885 1886foreach vt = Reg32Types.types in { 1887defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1888} 1889 1890defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1891defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1892defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1893 1894 1895let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1896 // Hiding the extract high pattern in the PatFrag seems to not 1897 // automatically increase the complexity. 1898let AddedComplexity = 1 in { 1899defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1900defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1901} 1902} 1903} // End OtherPredicates = [DisableFlatScratch] 1904 1905//===----------------------------------------------------------------------===// 1906// MTBUF Patterns 1907//===----------------------------------------------------------------------===// 1908 1909//===----------------------------------------------------------------------===// 1910// tbuffer_load/store_format patterns 1911//===----------------------------------------------------------------------===// 1912 1913multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1914 string opcode, ValueType memoryVt = vt> { 1915 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1916 1917 def : GCNPat< 1918 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1919 timm:$format, timm:$auxiliary, 0)), 1920 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1921 (as_i8timm $format), 1922 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1923 >; 1924 1925 def : GCNPat< 1926 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1927 timm:$format, timm:$auxiliary, timm)), 1928 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1929 (as_i8timm $format), 1930 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1931 >; 1932 1933 def : GCNPat< 1934 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1935 timm:$format, timm:$auxiliary, 0)), 1936 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1937 (as_i8timm $format), 1938 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1939 >; 1940 1941 def : GCNPat< 1942 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1943 timm:$format, timm:$auxiliary, timm)), 1944 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1945 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1946 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1947 (as_i8timm $format), 1948 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1949 >; 1950} 1951 1952defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1953defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1954defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1955defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1956defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1957defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1958defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1959defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1960 1961let SubtargetPredicate = HasUnpackedD16VMem in { 1962 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1963 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1964 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1965 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1966 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1967} // End HasUnpackedD16VMem. 1968 1969let SubtargetPredicate = HasPackedD16VMem in { 1970 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1971 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1972 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1973 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1974 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1975} // End HasPackedD16VMem. 1976 1977multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1978 string opcode, ValueType memoryVt = vt> { 1979 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1980 1981 def : GCNPat< 1982 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1983 timm:$format, timm:$auxiliary, 0), 1984 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1985 (as_i16timm $offset), (as_i8timm $format), 1986 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1987 >; 1988 1989 def : GCNPat< 1990 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1991 timm:$format, timm:$auxiliary, timm), 1992 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1993 (as_i16timm $offset), (as_i8timm $format), 1994 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1995 >; 1996 1997 def : GCNPat< 1998 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1999 timm:$format, timm:$auxiliary, 0), 2000 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2001 (as_i16timm $offset), (as_i8timm $format), 2002 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2003 >; 2004 2005 def : GCNPat< 2006 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 2007 timm:$offset, timm:$format, timm:$auxiliary, timm), 2008 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2009 getVregSrcForVT<vt>.ret:$vdata, 2010 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2011 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 2012 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2013 >; 2014} 2015 2016defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2017defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2018defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2019defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2020defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2021defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2022defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2023defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2024 2025let SubtargetPredicate = HasUnpackedD16VMem in { 2026 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2027 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2028 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2029 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2030 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2031} // End HasUnpackedD16VMem. 2032 2033let SubtargetPredicate = HasPackedD16VMem in { 2034 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2035 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2036 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2037 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2038 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2039} // End HasPackedD16VMem. 2040 2041//===----------------------------------------------------------------------===// 2042// Target-specific instruction encodings. 2043//===----------------------------------------------------------------------===// 2044 2045//===----------------------------------------------------------------------===// 2046// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2047//===----------------------------------------------------------------------===// 2048 2049class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2050 string real_name = ps.Mnemonic> : 2051 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2052 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2053 let Inst{31-26} = 0x38; 2054 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2055 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2056 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2057 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2058} 2059 2060class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2061 string real_name = ps.Mnemonic> : 2062 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2063 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2064 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2065 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2066 let Inst{25-18} = op; 2067 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2068 let Inst{54} = ps.offen; 2069 let Inst{55} = ps.idxen; 2070} 2071 2072class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2073 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2074 let Inst{12} = ps.offen; 2075 let Inst{13} = ps.idxen; 2076 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2077 let Inst{16} = ps.lds; 2078 let Inst{24-18} = op; 2079 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2080 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2081} 2082 2083class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2084 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2085 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2086 let Inst{25} = op{7}; 2087} 2088 2089class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2090 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2091 let Inst{15} = ps.addr64; 2092} 2093 2094//===----------------------------------------------------------------------===// 2095// MUBUF - GFX11. 2096//===----------------------------------------------------------------------===// 2097 2098let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2099multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name> { 2100 def _BOTHEN_gfx11 : 2101 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>, 2102 AtomicNoRet<NAME # "_BOTHEN_gfx11", 0>; 2103 def _IDXEN_gfx11 : 2104 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), real_name>, 2105 AtomicNoRet<NAME # "_IDXEN_gfx11", 0>; 2106 def _OFFEN_gfx11 : 2107 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), real_name>, 2108 AtomicNoRet<NAME # "_OFFEN_gfx11", 0>; 2109 def _OFFSET_gfx11 : 2110 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), real_name>, 2111 AtomicNoRet<NAME # "_OFFSET_gfx11", 0>; 2112} 2113 2114multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, MUBUF_Pseudo ps> : 2115 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2116multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op> : 2117 MUBUF_Real_AllAddr_gfx11_Impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2118 2119class Pre_gfx11_MUBUF_Name <MUBUF_Pseudo ps, string real_name> : 2120 MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2121multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> : 2122 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2123 def : Pre_gfx11_MUBUF_Name<!cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2124} 2125 2126let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2127multiclass MUBUF_Real_Atomics_RTN_gfx11_Renamed<bits<8> op, string real_name> { 2128 def _BOTHEN_RTN_gfx11 : 2129 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN"), real_name>, 2130 AtomicNoRet<NAME # "_BOTHEN_gfx11", 1>; 2131 def _IDXEN_RTN_gfx11 : 2132 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN"), real_name>, 2133 AtomicNoRet<NAME # "_IDXEN_gfx11", 1>; 2134 def _OFFEN_RTN_gfx11 : 2135 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN"), real_name>, 2136 AtomicNoRet<NAME # "_OFFEN_gfx11", 1>; 2137 def _OFFSET_RTN_gfx11 : 2138 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN"), real_name>, 2139 AtomicNoRet<NAME # "_OFFSET_gfx11", 1>; 2140} 2141 2142multiclass MUBUF_Real_Atomics_RTN_gfx11_impl<bits<8> op, MUBUF_Pseudo ps> : 2143 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, ps.Mnemonic>; 2144multiclass MUBUF_Real_Atomics_RTN_gfx11<bits<8> op> : 2145 MUBUF_Real_Atomics_RTN_gfx11_impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2146 2147multiclass MUBUF_Real_Atomics_gfx11<bits<8> op> : 2148 MUBUF_Real_AllAddr_gfx11<op>, 2149 MUBUF_Real_Atomics_RTN_gfx11<op>; 2150 2151multiclass MUBUF_Real_Atomics_gfx11_Renamed<bits<8> op, string real_name> : 2152 MUBUF_Real_AllAddr_gfx11_Renamed<op, real_name>, 2153 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, real_name>; 2154 2155let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2156def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2157def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2158} 2159 2160defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">; 2161defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">; 2162defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">; 2163defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">; 2164defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">; 2165defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">; 2166defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">; 2167defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2168defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2169defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">; 2170defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2171defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">; 2172defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">; 2173defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">; 2174defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">; 2175defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x000>; 2176defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x001>; 2177defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x002>; 2178defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x003>; 2179defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">; 2180defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">; 2181defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">; 2182defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">; 2183defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031>; 2184defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032>; 2185defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e>; 2186defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030>; 2187defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d>; 2188defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f>; 2189defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">; 2190defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">; 2191defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">; 2192defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">; 2193defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">; 2194defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">; 2195defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">; 2196defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">; 2197defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2198defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2199defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">; 2200defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">; 2201defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2202defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x004>; 2203defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x005>; 2204defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x006>; 2205defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x007>; 2206defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomics_gfx11<0x056>; 2207defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx11_Renamed<0x035, "buffer_atomic_add_u32">; 2208defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x043, "buffer_atomic_add_u64">; 2209defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">; 2210defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x049, "buffer_atomic_and_b64">; 2211defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2212defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2213defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2214defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx11_Renamed<0x037, "buffer_atomic_csub_u32">; 2215def : MnemonicAlias<"buffer_atomic_csub", "buffer_atomic_csub_u32">, Requires<[isGFX11Plus]>; 2216defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">; 2217defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">; 2218defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">; 2219defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">; 2220defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x052, "buffer_atomic_max_f32">; 2221defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">; 2222defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x047, "buffer_atomic_max_i64">; 2223defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">; 2224defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x048, "buffer_atomic_max_u64">; 2225defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x051, "buffer_atomic_min_f32">; 2226defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x038, "buffer_atomic_min_i32">; 2227defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x045, "buffer_atomic_min_i64">; 2228defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x039, "buffer_atomic_min_u32">; 2229defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x046, "buffer_atomic_min_u64">; 2230defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">; 2231defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">; 2232defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">; 2233defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">; 2234defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">; 2235defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">; 2236defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">; 2237defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">; 2238 2239//===----------------------------------------------------------------------===// 2240// MUBUF - GFX10. 2241//===----------------------------------------------------------------------===// 2242 2243let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2244 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2245 def _BOTHEN_gfx10 : 2246 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2247 def _IDXEN_gfx10 : 2248 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2249 def _OFFEN_gfx10 : 2250 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2251 def _OFFSET_gfx10 : 2252 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2253 } 2254 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2255 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2256 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2257 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2258 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2259 2260 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2261 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2262 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2263 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2264 } 2265 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2266 def _BOTHEN_RTN_gfx10 : 2267 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2268 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2269 def _IDXEN_RTN_gfx10 : 2270 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2271 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2272 def _OFFEN_RTN_gfx10 : 2273 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2274 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2275 def _OFFSET_RTN_gfx10 : 2276 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2277 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2278 } 2279 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2280 MUBUF_Real_Atomics_RTN_gfx10<op> { 2281 def _BOTHEN_gfx10 : 2282 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2283 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2284 def _IDXEN_gfx10 : 2285 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2286 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2287 def _OFFEN_gfx10 : 2288 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2289 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2290 def _OFFSET_gfx10 : 2291 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2292 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2293 } 2294} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2295 2296defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2297defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2298defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2299defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2300defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2301defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2302defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2303defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2304// FIXME-GFX10: Add following instructions: 2305//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2306//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2307defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2308defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2309defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2310defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2311defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2312defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2313defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2314defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2315 2316def BUFFER_GL0_INV_gfx10 : 2317 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2318def BUFFER_GL1_INV_gfx10 : 2319 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2320 2321//===----------------------------------------------------------------------===// 2322// MUBUF - GFX6, GFX7, GFX10. 2323//===----------------------------------------------------------------------===// 2324 2325let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2326 multiclass MUBUF_Real_gfx6<bits<8> op> { 2327 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2328 } 2329} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2330 2331let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2332 multiclass MUBUF_Real_gfx7<bits<8> op> { 2333 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2334 } 2335} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2336 2337let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2338 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2339 def _ADDR64_gfx6_gfx7 : 2340 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2341 def _BOTHEN_gfx6_gfx7 : 2342 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2343 def _IDXEN_gfx6_gfx7 : 2344 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2345 def _OFFEN_gfx6_gfx7 : 2346 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2347 def _OFFSET_gfx6_gfx7 : 2348 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2349 } 2350 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2351 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2352 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2353 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2354 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2355 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2356 2357 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2358 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2359 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2360 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2361 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2362 } 2363 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2364 def _ADDR64_gfx6_gfx7 : 2365 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2366 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2367 def _BOTHEN_gfx6_gfx7 : 2368 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2369 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2370 def _IDXEN_gfx6_gfx7 : 2371 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2372 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2373 def _OFFEN_gfx6_gfx7 : 2374 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2375 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2376 def _OFFSET_gfx6_gfx7 : 2377 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2378 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2379 2380 def _ADDR64_RTN_gfx6_gfx7 : 2381 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2382 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2383 def _BOTHEN_RTN_gfx6_gfx7 : 2384 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2385 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2386 def _IDXEN_RTN_gfx6_gfx7 : 2387 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2388 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2389 def _OFFEN_RTN_gfx6_gfx7 : 2390 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2391 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2392 def _OFFSET_RTN_gfx6_gfx7 : 2393 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2394 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2395 } 2396} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2397 2398multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2399 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2400 2401multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2402 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2403 2404multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2405 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2406 2407// FIXME-GFX6: Following instructions are available only on GFX6. 2408//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2409//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2410 2411defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2412defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2413defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2414defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2415defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2416defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2417defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2418defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2419defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2420defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2421defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2422defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2423defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2424defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2425defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2426defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2427defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2428defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2429defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2430defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2431defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2432defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2433 2434defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2435defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2436defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2437defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2438defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2439defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2440defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2441defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2442defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2443defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2444defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2445defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2446defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2447defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2448defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2449defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2450defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2451defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2452defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2453defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2454defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2455defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2456defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2457defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2458defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2459defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2460defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2461defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2462defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2463// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2464defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2465defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2466defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2467 2468defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2469 2470defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2471defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2472def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2473 2474//===----------------------------------------------------------------------===// 2475// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2476//===----------------------------------------------------------------------===// 2477 2478class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2479 string real_name = ps.Mnemonic> : 2480 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2481 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2482 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2483 let Inst{31-26} = 0x3a; //encoding 2484 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2485 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2486 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2487 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2488} 2489 2490class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2491 string real_name = ps.Mnemonic> : 2492 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2493 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2494 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2495 let Inst{18-15} = op; 2496 let Inst{25-19} = format; 2497 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2498 let Inst{54} = ps.offen; 2499 let Inst{55} = ps.idxen; 2500} 2501 2502class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2503 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2504 let Inst{12} = ps.offen; 2505 let Inst{13} = ps.idxen; 2506 let Inst{18-16} = op; 2507 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2508 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2509} 2510 2511//===----------------------------------------------------------------------===// 2512// MTBUF - GFX11. 2513//===----------------------------------------------------------------------===// 2514 2515let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2516multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> { 2517 def _BOTHEN_gfx11 : 2518 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2519 def _IDXEN_gfx11 : 2520 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2521 def _OFFEN_gfx11 : 2522 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2523 def _OFFSET_gfx11 : 2524 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2525} 2526 2527multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps> 2528 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2529multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op> 2530 : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2531 2532 2533class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2534 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2535multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name> 2536 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2537 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2538} 2539 2540defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">; 2541defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2542defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2543defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2544defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x000>; 2545defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x001>; 2546defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x002>; 2547defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x003>; 2548defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2549defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2550defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2551defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2552defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x004>; 2553defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x005>; 2554defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x006>; 2555defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x007>; 2556 2557//===----------------------------------------------------------------------===// 2558// MTBUF - GFX10. 2559//===----------------------------------------------------------------------===// 2560 2561class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2562 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2563 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2564 let Inst{25-19} = format; 2565 let Inst{53} = op{3}; 2566} 2567 2568let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2569 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2570 def _BOTHEN_gfx10 : 2571 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2572 def _IDXEN_gfx10 : 2573 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2574 def _OFFEN_gfx10 : 2575 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2576 def _OFFSET_gfx10 : 2577 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2578 } 2579} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2580 2581defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2582defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2583defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2584defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2585defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2586defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2587defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2588defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2589 2590//===----------------------------------------------------------------------===// 2591// MTBUF - GFX6, GFX7, GFX10. 2592//===----------------------------------------------------------------------===// 2593 2594class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2595 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2596 let Inst{15} = ps.addr64; 2597 let Inst{22-19} = dfmt; 2598 let Inst{25-23} = nfmt; 2599} 2600 2601let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2602 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2603 def _ADDR64_gfx6_gfx7 : 2604 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2605 def _BOTHEN_gfx6_gfx7 : 2606 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2607 def _IDXEN_gfx6_gfx7 : 2608 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2609 def _OFFEN_gfx6_gfx7 : 2610 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2611 def _OFFSET_gfx6_gfx7 : 2612 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2613 } 2614} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2615 2616multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2617 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2618 2619defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2620defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2621defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2622defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2623defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2624defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2625defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2626defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2627 2628//===----------------------------------------------------------------------===// 2629// GFX8, GFX9 (VI). 2630//===----------------------------------------------------------------------===// 2631 2632class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2633 bit has_sccb = ps.has_sccb> : 2634 MUBUF_Real<ps>, 2635 Enc64, 2636 SIMCInstr<ps.PseudoInstr, Enc>, 2637 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2638 !if(ps.IsAtomicRet, 1, ?))> { 2639 2640 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2641 let Inst{12} = ps.offen; 2642 let Inst{13} = ps.idxen; 2643 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2644 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2645 let Inst{16} = ps.lds; 2646 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2647 let Inst{24-18} = op; 2648 let Inst{31-26} = 0x38; //encoding 2649 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2650 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2651 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2652 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2653} 2654 2655class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2656 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2657 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2658 let DecoderNamespace = "GFX8"; 2659 2660 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2661} 2662 2663class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2664 bit has_sccb = ps.has_sccb> : 2665 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2666 let AssemblerPredicate = isGFX90APlus; 2667 let DecoderNamespace = "GFX90A"; 2668 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2669 !subst("$tfe", "", ps.AsmOperands)); 2670 2671 let Inst{55} = acc; 2672} 2673 2674class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2675 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2676 let AssemblerPredicate = isGFX940Plus; 2677 let DecoderNamespace = "GFX9"; 2678 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2679 2680 let Inst{55} = acc; 2681} 2682 2683multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2684 def _vi : MUBUF_Real_vi<op, ps>; 2685 2686 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2687 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2688 2689 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2690 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2691 let SubtargetPredicate = isGFX90AOnly; 2692 let AssemblerPredicate = isGFX90AOnly; 2693 } 2694 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2695 } 2696} 2697 2698multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2699 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2700 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2701 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2702 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2703} 2704 2705multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2706 2707 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2708 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2709 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2710 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2711 2712 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2713 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2714 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2715 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2716 2717 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2718 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2719 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2720 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2721 2722 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2723 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2724 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2725 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2726} 2727 2728class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2729 MUBUF_Real<ps>, 2730 Enc64, 2731 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2732 let AssemblerPredicate=HasUnpackedD16VMem; 2733 let DecoderNamespace="GFX80_UNPACKED"; 2734 2735 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2736 let Inst{12} = ps.offen; 2737 let Inst{13} = ps.idxen; 2738 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2739 let Inst{16} = ps.lds; 2740 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2741 let Inst{24-18} = op; 2742 let Inst{31-26} = 0x38; //encoding 2743 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2744 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2745 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2746 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2747 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2748} 2749 2750multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2751 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2752 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2753 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2754 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2755} 2756 2757multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2758 MUBUF_Real_AllAddr_vi<op> { 2759 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2760 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2761 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2762 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2763} 2764 2765defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2766defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2767defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2768defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2769defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2770defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2771defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2772defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2773let SubtargetPredicate = HasUnpackedD16VMem in { 2774 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2775 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2776 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2777 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2778 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2779 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2780 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2781 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2782} // End HasUnpackedD16VMem. 2783let SubtargetPredicate = HasPackedD16VMem in { 2784 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2785 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2786 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2787 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2788 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2789 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2790 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2791 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2792} // End HasPackedD16VMem. 2793defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2794defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2795defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2796defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2797defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2798defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 2799defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 2800defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 2801defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2802defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2803defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2804defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2805defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2806defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2807defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2808defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2809 2810defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2811defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2812defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2813defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2814defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2815defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2816 2817defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2818defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2819 2820defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2821defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2822defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2823defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2824defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2825defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2826defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2827defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2828defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2829defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2830defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2831defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2832defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2833 2834defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2835defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2836defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2837defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2838defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2839defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2840defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2841defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2842defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2843defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2844defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2845defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2846defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2847 2848defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2849 2850let AssemblerPredicate = isGFX8GFX9 in { 2851def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2852def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2853} // End AssemblerPredicate = isGFX8GFX9 2854 2855let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2856 2857defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2858defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2859 2860} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2861 2862let SubtargetPredicate = isGFX90APlus in { 2863 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2864 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2865 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2866} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2867 2868def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2869 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2870 let AssemblerPredicate = isGFX90AOnly; 2871 let SubtargetPredicate = isGFX90AOnly; 2872} 2873def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2874 2875let SubtargetPredicate = isGFX940Plus in { 2876def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2877def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2878} 2879 2880class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2881 MTBUF_Real<ps>, 2882 Enc64, 2883 SIMCInstr<ps.PseudoInstr, Enc> { 2884 2885 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2886 let Inst{12} = ps.offen; 2887 let Inst{13} = ps.idxen; 2888 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2889 let Inst{18-15} = op; 2890 let Inst{22-19} = dfmt; 2891 let Inst{25-23} = nfmt; 2892 let Inst{31-26} = 0x3a; //encoding 2893 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2894 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2895 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2896 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2897 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2898 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2899 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2900} 2901 2902class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2903 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2904 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2905 let DecoderNamespace = "GFX8"; 2906 2907 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2908} 2909 2910class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2911 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2912 let AssemblerPredicate = isGFX90APlus; 2913 let DecoderNamespace = "GFX90A"; 2914 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2915 2916 let Inst{55} = acc; 2917} 2918 2919multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2920 def _vi : MTBUF_Real_vi<op, ps>; 2921 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2922} 2923 2924multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2925 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2926 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2927 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2928 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2929} 2930 2931class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2932 MTBUF_Real<ps>, 2933 Enc64, 2934 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2935 let AssemblerPredicate=HasUnpackedD16VMem; 2936 let DecoderNamespace="GFX80_UNPACKED"; 2937 2938 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2939 let Inst{12} = ps.offen; 2940 let Inst{13} = ps.idxen; 2941 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2942 let Inst{18-15} = op; 2943 let Inst{22-19} = dfmt; 2944 let Inst{25-23} = nfmt; 2945 let Inst{31-26} = 0x3a; //encoding 2946 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2947 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2948 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2949 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2950 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2951 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2952} 2953 2954multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2955 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2956 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2957 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2958 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2959} 2960 2961defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2962defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2963defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2964defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2965defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2966defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2967defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2968defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2969let SubtargetPredicate = HasUnpackedD16VMem in { 2970 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2971 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2972 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2973 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2974 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2975 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2976 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2977 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2978} // End HasUnpackedD16VMem. 2979let SubtargetPredicate = HasPackedD16VMem in { 2980 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2981 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2982 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2983 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2984 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2985 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2986 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2987 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2988} // End HasUnpackedD16VMem. 2989 2990def MUBUFInfoTable : GenericTable { 2991 let FilterClass = "MUBUF_Pseudo"; 2992 let CppTypeName = "MUBUFInfo"; 2993 let Fields = [ 2994 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2995 "IsBufferInv" 2996 ]; 2997 2998 let PrimaryKey = ["Opcode"]; 2999 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3000} 3001 3002def getMUBUFInfoFromOpcode : SearchIndex { 3003 let Table = MUBUFInfoTable; 3004 let Key = ["Opcode"]; 3005} 3006 3007def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3008 let Table = MUBUFInfoTable; 3009 let Key = ["BaseOpcode", "elements"]; 3010} 3011 3012def MTBUFInfoTable : GenericTable { 3013 let FilterClass = "MTBUF_Pseudo"; 3014 let CppTypeName = "MTBUFInfo"; 3015 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3016 3017 let PrimaryKey = ["Opcode"]; 3018 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3019} 3020 3021def getMTBUFInfoFromOpcode : SearchIndex { 3022 let Table = MTBUFInfoTable; 3023 let Key = ["Opcode"]; 3024} 3025 3026def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3027 let Table = MTBUFInfoTable; 3028 let Key = ["BaseOpcode", "elements"]; 3029} 3030