1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MUBUFLdsTable <bit is_lds, string Name> { 39 bit IsLds = is_lds; 40 string OpName = Name; 41} 42 43class MTBUFAddr64Table <bit is_addr64, string Name> { 44 bit IsAddr64 = is_addr64; 45 string OpName = Name; 46} 47 48//===----------------------------------------------------------------------===// 49// MTBUF classes 50//===----------------------------------------------------------------------===// 51 52class MTBUFGetBaseOpcode<string Op> { 53 string ret = !subst("FORMAT_XY", "FORMAT_X", 54 !subst("FORMAT_XYZ", "FORMAT_X", 55 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 56} 57 58 59class MTBUF_Pseudo <string opName, dag outs, dag ins, 60 string asmOps, list<dag> pattern=[]> : 61 InstSI<outs, ins, "", pattern>, 62 SIMCInstr<opName, SIEncodingFamily.NONE> { 63 64 let isPseudo = 1; 65 let isCodeGenOnly = 1; 66 let Size = 8; 67 let UseNamedOperandTable = 1; 68 69 string Mnemonic = opName; 70 string AsmOperands = asmOps; 71 72 Instruction Opcode = !cast<Instruction>(NAME); 73 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 74 75 let VM_CNT = 1; 76 let EXP_CNT = 1; 77 let MTBUF = 1; 78 let Uses = [EXEC]; 79 let hasSideEffects = 0; 80 let SchedRW = [WriteVMEM]; 81 82 let AsmMatchConverter = "cvtMtbuf"; 83 84 bits<1> offen = 0; 85 bits<1> idxen = 0; 86 bits<1> addr64 = 0; 87 bits<1> has_vdata = 1; 88 bits<1> has_vaddr = 1; 89 bits<1> has_glc = 1; 90 bits<1> has_dlc = 1; 91 bits<1> glc_value = 0; // the value for glc if no such operand 92 bits<1> dlc_value = 0; // the value for dlc if no such operand 93 bits<1> has_srsrc = 1; 94 bits<1> has_soffset = 1; 95 bits<1> has_offset = 1; 96 bits<1> has_slc = 1; 97 bits<1> has_tfe = 1; 98 bits<4> elements = 0; 99 bits<1> has_sccb = 1; 100 bits<1> sccb_value = 0; 101} 102 103class MTBUF_Real <MTBUF_Pseudo ps> : 104 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 105 106 let isPseudo = 0; 107 let isCodeGenOnly = 0; 108 109 let VM_CNT = 1; 110 let EXP_CNT = 1; 111 let MTBUF = 1; 112 113 // copy relevant pseudo op flags 114 let UseNamedOperandTable = ps.UseNamedOperandTable; 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 let SchedRW = ps.SchedRW; 121 let mayLoad = ps.mayLoad; 122 let mayStore = ps.mayStore; 123 let IsAtomicRet = ps.IsAtomicRet; 124 let IsAtomicNoRet = ps.IsAtomicNoRet; 125 126 bits<12> offset; 127 bits<5> cpol; 128 bits<7> format; 129 bits<8> vaddr; 130 bits<10> vdata; 131 bits<7> srsrc; 132 bits<1> tfe; 133 bits<8> soffset; 134 135 bits<4> dfmt = format{3-0}; 136 bits<3> nfmt = format{6-4}; 137 138 // GFX90A+ only: instruction uses AccVGPR for data 139 // Bit supersedes tfe. 140 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 141} 142 143class getMTBUFInsDA<list<RegisterClass> vdataList, 144 list<RegisterClass> vaddrList=[]> { 145 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 146 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 147 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 148 dag InsNoData = !if(!empty(vaddrList), 149 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 150 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 151 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 152 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 153 ); 154 dag InsData = !if(!empty(vaddrList), 155 (ins vdata_op:$vdata, SReg_128:$srsrc, 156 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 157 TFE:$tfe, SWZ:$swz), 158 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 159 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 160 TFE:$tfe, SWZ:$swz) 161 ); 162 dag ret = !if(!empty(vdataList), InsNoData, InsData); 163} 164 165class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 166 dag ret = 167 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 168 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 169 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 170 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 171 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 172 (ins)))))); 173} 174 175class getMTBUFAsmOps<int addrKind> { 176 string Pfx = 177 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 178 !if(!eq(addrKind, BUFAddrKind.OffEn), 179 "$vaddr, $srsrc,$format $soffset offen", 180 !if(!eq(addrKind, BUFAddrKind.IdxEn), 181 "$vaddr, $srsrc,$format $soffset idxen", 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 183 "$vaddr, $srsrc,$format $soffset idxen offen", 184 !if(!eq(addrKind, BUFAddrKind.Addr64), 185 "$vaddr, $srsrc,$format $soffset addr64", 186 ""))))); 187 string ret = Pfx # "$offset"; 188} 189 190class MTBUF_SetupAddr<int addrKind> { 191 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 192 !eq(addrKind, BUFAddrKind.BothEn)); 193 194 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 195 !eq(addrKind, BUFAddrKind.BothEn)); 196 197 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 198 199 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 200} 201 202class MTBUF_Load_Pseudo <string opName, 203 int addrKind, 204 RegisterClass vdataClass, 205 int elems, 206 list<dag> pattern=[], 207 // Workaround bug bz30254 208 int addrKindCopy = addrKind> 209 : MTBUF_Pseudo<opName, 210 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 211 getMTBUFIns<addrKindCopy>.ret, 212 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 213 pattern>, 214 MTBUF_SetupAddr<addrKindCopy> { 215 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 216 let mayLoad = 1; 217 let mayStore = 0; 218 let elements = elems; 219} 220 221multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 222 int elems> { 223 224 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 225 MTBUFAddr64Table<0, NAME>; 226 227 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 228 MTBUFAddr64Table<1, NAME>; 229 230 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 231 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 232 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 233 234 let DisableWQM = 1 in { 235 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 236 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 237 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 238 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 239 } 240} 241 242class MTBUF_Store_Pseudo <string opName, 243 int addrKind, 244 RegisterClass vdataClass, 245 int elems, 246 list<dag> pattern=[], 247 // Workaround bug bz30254 248 int addrKindCopy = addrKind, 249 RegisterClass vdataClassCopy = vdataClass> 250 : MTBUF_Pseudo<opName, 251 (outs), 252 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 253 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 254 pattern>, 255 MTBUF_SetupAddr<addrKindCopy> { 256 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 257 let mayLoad = 0; 258 let mayStore = 1; 259 let elements = elems; 260} 261 262multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 263 int elems> { 264 265 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 266 MTBUFAddr64Table<0, NAME>; 267 268 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 269 MTBUFAddr64Table<1, NAME>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUFGetBaseOpcode<string Op> { 289 string ret = !subst("DWORDX2", "DWORD", 290 !subst("DWORDX3", "DWORD", 291 !subst("DWORDX4", "DWORD", Op))); 292} 293 294class MUBUF_Pseudo <string opName, dag outs, dag ins, 295 string asmOps, list<dag> pattern=[]> : 296 InstSI<outs, ins, "", pattern>, 297 SIMCInstr<opName, SIEncodingFamily.NONE> { 298 299 let isPseudo = 1; 300 let isCodeGenOnly = 1; 301 let Size = 8; 302 let UseNamedOperandTable = 1; 303 304 string Mnemonic = opName; 305 string AsmOperands = asmOps; 306 307 Instruction Opcode = !cast<Instruction>(NAME); 308 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 309 310 let VM_CNT = 1; 311 let EXP_CNT = 1; 312 let MUBUF = 1; 313 let Uses = [EXEC]; 314 let hasSideEffects = 0; 315 let SchedRW = [WriteVMEM]; 316 317 let AsmMatchConverter = "cvtMubuf"; 318 319 bits<1> offen = 0; 320 bits<1> idxen = 0; 321 bits<1> addr64 = 0; 322 bits<1> lds = 0; 323 bits<1> has_vdata = !not(lds); 324 bits<1> has_vaddr = 1; 325 bits<1> has_glc = 1; 326 bits<1> has_dlc = 1; 327 bits<1> glc_value = 0; // the value for glc if no such operand 328 bits<1> dlc_value = 0; // the value for dlc if no such operand 329 bits<1> has_srsrc = 1; 330 bits<1> has_soffset = 1; 331 bits<1> has_offset = 1; 332 bits<1> has_slc = 1; 333 bits<1> has_tfe = 1; 334 bits<4> elements = 0; 335 bits<1> has_sccb = 1; 336 bits<1> sccb_value = 0; 337 bits<1> IsBufferInv = 0; 338} 339 340class MUBUF_Real <MUBUF_Pseudo ps> : 341 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 342 343 let isPseudo = 0; 344 let isCodeGenOnly = 0; 345 346 let VM_CNT = 1; 347 let EXP_CNT = 1; 348 let MUBUF = 1; 349 350 // copy relevant pseudo op flags 351 let SubtargetPredicate = ps.SubtargetPredicate; 352 let AsmMatchConverter = ps.AsmMatchConverter; 353 let OtherPredicates = ps.OtherPredicates; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 let UseNamedOperandTable = ps.UseNamedOperandTable; 358 let SchedRW = ps.SchedRW; 359 let mayLoad = ps.mayLoad; 360 let mayStore = ps.mayStore; 361 let IsAtomicRet = ps.IsAtomicRet; 362 let IsAtomicNoRet = ps.IsAtomicNoRet; 363 let VALU = ps.VALU; 364 let LGKM_CNT = ps.LGKM_CNT; 365 366 bits<12> offset; 367 bits<5> cpol; 368 bits<8> vaddr; 369 bits<10> vdata; 370 bits<7> srsrc; 371 bits<1> tfe; 372 bits<8> soffset; 373 374 // GFX90A+ only: instruction uses AccVGPR for data 375 // Bit supersedes tfe. 376 bits<1> acc = !if(ps.has_vdata, vdata{9}, !if(ps.lds, ?, 0)); 377} 378 379 380// For cache invalidation instructions. 381class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 382 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 383 384 let AsmMatchConverter = ""; 385 386 let hasSideEffects = 1; 387 let mayLoad = 0; 388 let mayStore = 0; 389 390 let IsBufferInv = 1; 391 // Set everything else to 0. 392 let offen = 0; 393 let idxen = 0; 394 let addr64 = 0; 395 let has_vdata = 0; 396 let has_vaddr = 0; 397 let has_glc = 0; 398 let has_dlc = 0; 399 let glc_value = 0; 400 let dlc_value = 0; 401 let has_srsrc = 0; 402 let has_soffset = 0; 403 let has_offset = 0; 404 let has_slc = 0; 405 let has_tfe = 0; 406 let has_sccb = 0; 407 let sccb_value = 0; 408} 409 410class getMUBUFInsDA<list<RegisterClass> vdataList, 411 list<RegisterClass> vaddrList=[], 412 bit isLds = 0> { 413 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 414 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 415 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 416 dag InsNoData = !if(!empty(vaddrList), 417 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 418 offset:$offset, CPol_0:$cpol), 419 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 420 offset:$offset, CPol_0:$cpol) 421 ); 422 dag InsData = !if(!empty(vaddrList), 423 (ins vdata_op:$vdata, SReg_128:$srsrc, 424 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 425 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 426 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 427 ); 428 dag ret = !con( 429 !if(!empty(vdataList), InsNoData, InsData), 430 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 431 ); 432} 433 434class getMUBUFElements<ValueType vt> { 435 int ret = 436 !if(!eq(vt, f16), 1, 437 !if(!eq(vt, v2f16), 2, 438 !if(!eq(vt, v3f16), 3, 439 !if(!eq(vt, v4f16), 4, 440 !if(!eq(vt.Size, 32), 1, 441 !if(!eq(vt.Size, 64), 2, 442 !if(!eq(vt.Size, 96), 3, 443 !if(!eq(vt.Size, 128), 4, 0) 444 ) 445 ) 446 ) 447 ) 448 ) 449 ) 450 ); 451} 452 453class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 454 dag ret = 455 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 456 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 457 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 458 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 459 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 460 (ins)))))); 461} 462 463class getMUBUFAsmOps<int addrKind> { 464 string Pfx = 465 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 466 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 467 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 468 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 469 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 470 ""))))); 471 string ret = Pfx # "$offset"; 472} 473 474class MUBUF_SetupAddr<int addrKind> { 475 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 476 !eq(addrKind, BUFAddrKind.BothEn)); 477 478 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 479 !eq(addrKind, BUFAddrKind.BothEn)); 480 481 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 482 483 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 484} 485 486class MUBUF_Load_Pseudo <string opName, 487 int addrKind, 488 ValueType vdata_vt, 489 bit HasTiedDest = 0, 490 bit isLds = 0, 491 list<dag> pattern=[], 492 // Workaround bug bz30254 493 int addrKindCopy = addrKind, 494 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 495 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 496 : MUBUF_Pseudo<opName, 497 !if(isLds, (outs), (outs vdata_op:$vdata)), 498 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 499 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 500 !if(isLds, " ", " $vdata, ") # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 501 !if(isLds, " lds", "$tfe") # "$swz", 502 pattern>, 503 MUBUF_SetupAddr<addrKindCopy> { 504 let PseudoInstr = opName # !if(isLds, "_lds", "") # 505 "_" # getAddrName<addrKindCopy>.ret; 506 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 507 508 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 509 let LGKM_CNT = isLds; 510 let mayLoad = 1; 511 let mayStore = isLds; 512 let maybeAtomic = 1; 513 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 514 let has_tfe = !not(isLds); 515 let lds = isLds; 516 let elements = getMUBUFElements<vdata_vt>.ret; 517 let VALU = isLds; 518} 519 520class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 521 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 522 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 523>; 524 525class MUBUF_Addr64_Load_Pat <Instruction inst, 526 ValueType load_vt = i32, 527 SDPatternOperator ld = null_frag> : Pat < 528 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 529 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 530>; 531 532multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 533 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 534 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 535} 536 537 538// FIXME: tfe can't be an operand because it requires a separate 539// opcode because it needs an N+1 register class dest register. 540multiclass MUBUF_Pseudo_Loads<string opName, 541 ValueType load_vt = i32, 542 bit TiedDest = 0, 543 bit isLds = 0> { 544 545 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 546 547 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 548 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 549 550 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 551 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 552 553 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 554 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 555 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 556 557 let DisableWQM = 1 in { 558 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 559 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 560 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 561 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 562 } 563} 564 565multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 566 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 567 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 568} 569 570class MUBUF_Store_Pseudo <string opName, 571 int addrKind, 572 ValueType store_vt, 573 list<dag> pattern=[], 574 // Workaround bug bz30254 575 int addrKindCopy = addrKind> 576 : MUBUF_Pseudo<opName, 577 (outs), 578 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 579 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 580 pattern>, 581 MUBUF_SetupAddr<addrKindCopy> { 582 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 583 let mayLoad = 0; 584 let mayStore = 1; 585 let maybeAtomic = 1; 586 let elements = getMUBUFElements<store_vt>.ret; 587} 588 589multiclass MUBUF_Pseudo_Stores<string opName, 590 ValueType store_vt = i32, 591 SDPatternOperator st = null_frag> { 592 593 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 594 595 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 596 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 597 i16:$offset))]>, 598 MUBUFAddr64Table<0, NAME>; 599 600 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 601 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 602 i16:$offset))]>, 603 MUBUFAddr64Table<1, NAME>; 604 605 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 606 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 607 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 608 609 let DisableWQM = 1 in { 610 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 611 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 612 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 613 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 614 } 615} 616 617class MUBUF_Pseudo_Store_Lds<string opName> 618 : MUBUF_Pseudo<opName, 619 (outs), 620 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 621 " $srsrc, $soffset$offset lds$cpol$swz"> { 622 let LGKM_CNT = 1; 623 let mayLoad = 1; 624 let mayStore = 1; 625 let maybeAtomic = 1; 626 627 let has_vdata = 0; 628 let has_vaddr = 0; 629 let has_tfe = 0; 630 let lds = 1; 631 let VALU = 1; 632 633 let Uses = [EXEC, M0]; 634 let AsmMatchConverter = "cvtMubufLds"; 635} 636 637class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 638 list<RegisterClass> vaddrList=[]> { 639 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 640 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 641 dag ret = !if(vdata_in, 642 !if(!empty(vaddrList), 643 (ins vdata_op:$vdata_in, 644 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 645 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 646 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 647 ), 648 !if(!empty(vaddrList), 649 (ins vdata_op:$vdata, 650 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 651 (ins vdata_op:$vdata, vaddrClass:$vaddr, 652 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 653 )); 654} 655 656class getMUBUFAtomicIns<int addrKind, 657 RegisterClass vdataClass, 658 bit vdata_in, 659 // Workaround bug bz30254 660 RegisterClass vdataClassCopy=vdataClass> { 661 dag ret = 662 !if(!eq(addrKind, BUFAddrKind.Offset), 663 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 664 !if(!eq(addrKind, BUFAddrKind.OffEn), 665 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 666 !if(!eq(addrKind, BUFAddrKind.IdxEn), 667 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 668 !if(!eq(addrKind, BUFAddrKind.BothEn), 669 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 670 !if(!eq(addrKind, BUFAddrKind.Addr64), 671 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 672 (ins)))))); 673} 674 675class MUBUF_Atomic_Pseudo<string opName, 676 int addrKind, 677 dag outs, 678 dag ins, 679 string asmOps, 680 list<dag> pattern=[], 681 // Workaround bug bz30254 682 int addrKindCopy = addrKind> 683 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 684 MUBUF_SetupAddr<addrKindCopy> { 685 let mayStore = 1; 686 let mayLoad = 1; 687 let hasPostISelHook = 1; 688 let hasSideEffects = 1; 689 let DisableWQM = 1; 690 let has_glc = 0; 691 let has_dlc = 0; 692 let has_tfe = 0; 693 let has_sccb = 1; 694 let maybeAtomic = 1; 695 let AsmMatchConverter = "cvtMubufAtomic"; 696} 697 698class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 699 RegisterClass vdataClass, 700 list<dag> pattern=[], 701 // Workaround bug bz30254 702 int addrKindCopy = addrKind, 703 RegisterClass vdataClassCopy = vdataClass> 704 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 705 (outs), 706 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 707 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 708 pattern>, 709 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 710 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 711 let glc_value = 0; 712 let dlc_value = 0; 713 let sccb_value = 0; 714 let IsAtomicNoRet = 1; 715} 716 717class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 718 RegisterClass vdataClass, 719 list<dag> pattern=[], 720 // Workaround bug bz30254 721 int addrKindCopy = addrKind, 722 RegisterClass vdataClassCopy = vdataClass, 723 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 724 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 725 (outs vdata_op:$vdata), 726 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 727 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 728 pattern>, 729 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 730 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 731 let glc_value = 1; 732 let dlc_value = 0; 733 let sccb_value = 0; 734 let IsAtomicRet = 1; 735 let Constraints = "$vdata = $vdata_in"; 736 let DisableEncoding = "$vdata_in"; 737} 738 739multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 740 RegisterClass vdataClass, 741 ValueType vdataType, 742 bit isFP = isFloatType<vdataType>.ret> { 743 let FPAtomic = isFP in 744 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 745 MUBUFAddr64Table <0, NAME>; 746 747 let FPAtomic = isFP in 748 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 749 MUBUFAddr64Table <1, NAME>; 750 751 let FPAtomic = isFP in 752 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 753 754 let FPAtomic = isFP in 755 756 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 757 758 let FPAtomic = isFP in 759 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 760} 761 762multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 763 RegisterClass vdataClass, 764 ValueType vdataType, 765 SDPatternOperator atomic, 766 bit isFP = isFloatType<vdataType>.ret> { 767 let FPAtomic = isFP in 768 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 769 [(set vdataType:$vdata, 770 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 771 vdataType:$vdata_in))]>, 772 MUBUFAddr64Table <0, NAME # "_RTN">; 773 774 let FPAtomic = isFP in 775 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 776 [(set vdataType:$vdata, 777 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 778 vdataType:$vdata_in))]>, 779 MUBUFAddr64Table <1, NAME # "_RTN">; 780 781 let FPAtomic = isFP in 782 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 783 784 let FPAtomic = isFP in 785 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 786 787 let FPAtomic = isFP in 788 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 789} 790 791multiclass MUBUF_Pseudo_Atomics <string opName, 792 RegisterClass vdataClass, 793 ValueType vdataType, 794 SDPatternOperator atomic = null_frag> : 795 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 796 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 797 798 799//===----------------------------------------------------------------------===// 800// MUBUF Instructions 801//===----------------------------------------------------------------------===// 802 803defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 804 "buffer_load_format_x", f32 805>; 806defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 807 "buffer_load_format_xy", v2f32 808>; 809defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 810 "buffer_load_format_xyz", v3f32 811>; 812defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 813 "buffer_load_format_xyzw", v4f32 814>; 815defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 816 "buffer_store_format_x", f32 817>; 818defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 819 "buffer_store_format_xy", v2f32 820>; 821defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 822 "buffer_store_format_xyz", v3f32 823>; 824defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 825 "buffer_store_format_xyzw", v4f32 826>; 827 828let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 829 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 830 "buffer_load_format_d16_x", i32 831 >; 832 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 833 "buffer_load_format_d16_xy", v2i32 834 >; 835 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 836 "buffer_load_format_d16_xyz", v3i32 837 >; 838 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 839 "buffer_load_format_d16_xyzw", v4i32 840 >; 841 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 842 "buffer_store_format_d16_x", i32 843 >; 844 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_xy", v2i32 846 >; 847 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 848 "buffer_store_format_d16_xyz", v3i32 849 >; 850 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 851 "buffer_store_format_d16_xyzw", v4i32 852 >; 853} // End HasUnpackedD16VMem. 854 855let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 856 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 857 "buffer_load_format_d16_x", f16 858 >; 859 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 860 "buffer_load_format_d16_xy", v2f16 861 >; 862 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 863 "buffer_load_format_d16_xyz", v3f16 864 >; 865 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 866 "buffer_load_format_d16_xyzw", v4f16 867 >; 868 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 869 "buffer_store_format_d16_x", f16 870 >; 871 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 872 "buffer_store_format_d16_xy", v2f16 873 >; 874 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 875 "buffer_store_format_d16_xyz", v3f16 876 >; 877 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 878 "buffer_store_format_d16_xyzw", v4f16 879 >; 880} // End HasPackedD16VMem. 881 882defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 883 "buffer_load_ubyte", i32 884>; 885defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 886 "buffer_load_sbyte", i32 887>; 888defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 889 "buffer_load_ushort", i32 890>; 891defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 892 "buffer_load_sshort", i32 893>; 894defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 895 "buffer_load_dword", i32 896>; 897defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 898 "buffer_load_dwordx2", v2i32 899>; 900defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 901 "buffer_load_dwordx3", v3i32 902>; 903defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 904 "buffer_load_dwordx4", v4i32 905>; 906 907defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 908defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 909defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 910defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 911defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 912defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 913defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 914defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 915defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 916defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 917defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 918defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 919defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 920defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 921 922// This is not described in AMD documentation, 923// but 'lds' versions of these opcodes are available 924// in at least GFX8+ chips. See Bug 37653. 925let SubtargetPredicate = isGFX8GFX9 in { 926defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 927 "buffer_load_dwordx2", v2i32, 0, 1 928>; 929defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 930 "buffer_load_dwordx3", v3i32, 0, 1 931>; 932defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 933 "buffer_load_dwordx4", v4i32, 0, 1 934>; 935} 936 937defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 938 "buffer_store_byte", i32, truncstorei8_global 939>; 940defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 941 "buffer_store_short", i32, truncstorei16_global 942>; 943defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 944 "buffer_store_dword", i32, store_global 945>; 946defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 947 "buffer_store_dwordx2", v2i32, store_global 948>; 949defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 950 "buffer_store_dwordx3", v3i32, store_global 951>; 952defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 953 "buffer_store_dwordx4", v4i32, store_global 954>; 955defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_swap", VGPR_32, i32 957>; 958defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_cmpswap", VReg_64, v2i32 960>; 961defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_add", VGPR_32, i32 963>; 964defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_sub", VGPR_32, i32 966>; 967defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_smin", VGPR_32, i32 969>; 970defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_umin", VGPR_32, i32 972>; 973defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_smax", VGPR_32, i32 975>; 976defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_umax", VGPR_32, i32 978>; 979defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_and", VGPR_32, i32 981>; 982defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_or", VGPR_32, i32 984>; 985defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_xor", VGPR_32, i32 987>; 988defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_inc", VGPR_32, i32 990>; 991defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_dec", VGPR_32, i32 993>; 994defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_swap_x2", VReg_64, i64 996>; 997defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 999>; 1000defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_add_x2", VReg_64, i64 1002>; 1003defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_sub_x2", VReg_64, i64 1005>; 1006defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_smin_x2", VReg_64, i64 1008>; 1009defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_umin_x2", VReg_64, i64 1011>; 1012defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_smax_x2", VReg_64, i64 1014>; 1015defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_umax_x2", VReg_64, i64 1017>; 1018defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_and_x2", VReg_64, i64 1020>; 1021defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_or_x2", VReg_64, i64 1023>; 1024defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_xor_x2", VReg_64, i64 1026>; 1027defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_inc_x2", VReg_64, i64 1029>; 1030defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_dec_x2", VReg_64, i64 1032>; 1033 1034let SubtargetPredicate = HasGFX10_BEncoding in 1035defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1036 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1037>; 1038 1039let SubtargetPredicate = isGFX8GFX9 in { 1040def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1041} 1042 1043let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1044/* 1045defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1046defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1047*/ 1048 1049def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1050 int_amdgcn_buffer_wbinvl1_sc>; 1051} 1052 1053let SubtargetPredicate = isGFX6GFX7GFX10 in { 1054 1055defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1056 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1057>; 1058defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1059 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1060>; 1061defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1062 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1063>; 1064defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1066>; 1067defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1069>; 1070defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1072>; 1073 1074} 1075 1076let SubtargetPredicate = HasD16LoadStore in { 1077 1078defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1079 "buffer_load_ubyte_d16", i32, 1 1080>; 1081 1082defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1083 "buffer_load_ubyte_d16_hi", i32, 1 1084>; 1085 1086defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1087 "buffer_load_sbyte_d16", i32, 1 1088>; 1089 1090defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1091 "buffer_load_sbyte_d16_hi", i32, 1 1092>; 1093 1094defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1095 "buffer_load_short_d16", i32, 1 1096>; 1097 1098defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1099 "buffer_load_short_d16_hi", i32, 1 1100>; 1101 1102defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1103 "buffer_store_byte_d16_hi", i32 1104>; 1105 1106defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1107 "buffer_store_short_d16_hi", i32 1108>; 1109 1110defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1111 "buffer_load_format_d16_hi_x", i32 1112>; 1113defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1114 "buffer_store_format_d16_hi_x", i32 1115>; 1116 1117} // End HasD16LoadStore 1118 1119def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1120 int_amdgcn_buffer_wbinvl1>; 1121 1122let SubtargetPredicate = HasAtomicFaddInsts in { 1123defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1124 "buffer_atomic_add_f32", VGPR_32, f32 1125>; 1126defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1127 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1128>; 1129 1130let OtherPredicates = [isGFX90APlus] in { 1131defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN < 1132 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1133>; 1134defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1135 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1136>; 1137} 1138} // End SubtargetPredicate = HasAtomicFaddInsts 1139 1140//===----------------------------------------------------------------------===// 1141// MTBUF Instructions 1142//===----------------------------------------------------------------------===// 1143 1144defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1145defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1146defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1147defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1148defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1149defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1150defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1151defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1152 1153let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1154 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1155 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1156 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1157 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1158 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1159 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1160 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1161 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1162} // End HasUnpackedD16VMem. 1163 1164let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1165 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1166 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1167 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1168 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1169 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1170 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1171 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1172 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1173} // End HasPackedD16VMem. 1174 1175let SubtargetPredicate = isGFX7Plus in { 1176 1177//===----------------------------------------------------------------------===// 1178// Instruction definitions for CI and newer. 1179//===----------------------------------------------------------------------===// 1180 1181def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1182 int_amdgcn_buffer_wbinvl1_vol>; 1183 1184} // End let SubtargetPredicate = isGFX7Plus 1185 1186let SubtargetPredicate = isGFX90APlus in { 1187 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1188 let has_glc = 1; 1189 let has_sccb = 1; 1190 let InOperandList = (ins CPol_0:$cpol); 1191 let AsmOperands = "$cpol"; 1192 } 1193 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1194 let SubtargetPredicate = isGFX90AOnly; 1195 } 1196 1197 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1198 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1199 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1200} // End SubtargetPredicate = isGFX90APlus 1201 1202def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1203 let SubtargetPredicate = isGFX940Plus; 1204 let has_glc = 1; 1205 let has_sccb = 1; 1206 let InOperandList = (ins CPol_0:$cpol); 1207 let AsmOperands = "$cpol"; 1208} 1209 1210let SubtargetPredicate = isGFX10Plus in { 1211 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1212 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1213} // End SubtargetPredicate = isGFX10Plus 1214 1215//===----------------------------------------------------------------------===// 1216// MUBUF Patterns 1217//===----------------------------------------------------------------------===// 1218 1219//===----------------------------------------------------------------------===// 1220// buffer_load/store_format patterns 1221//===----------------------------------------------------------------------===// 1222 1223multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1224 string opcode, ValueType memoryVt = vt> { 1225 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1226 1227 def : GCNPat< 1228 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1229 timm:$auxiliary, 0)), 1230 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1231 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1232 >; 1233 1234 def : GCNPat< 1235 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1236 timm:$auxiliary, 0)), 1237 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1238 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1239 >; 1240 1241 def : GCNPat< 1242 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1243 timm:$auxiliary, timm)), 1244 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1245 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1246 >; 1247 1248 def : GCNPat< 1249 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1250 timm:$auxiliary, timm)), 1251 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1252 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1253 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1254 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1255 >; 1256} 1257 1258defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1259defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1260defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1261defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1262defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1263defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1264defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1265defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1266 1267let SubtargetPredicate = HasUnpackedD16VMem in { 1268 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1269 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1270 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1271 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1272 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1273 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1274} // End HasUnpackedD16VMem. 1275 1276let SubtargetPredicate = HasPackedD16VMem in { 1277 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1278 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1282 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1283 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1284 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1285 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1286} // End HasPackedD16VMem. 1287 1288defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1289defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1291defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1292defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1293defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1294defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1295defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1296defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1297defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1298defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1299defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1300defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1301defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1302defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1303defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1304 1305multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1306 string opcode, ValueType memoryVt = vt> { 1307 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1308 1309 def : GCNPat< 1310 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1311 timm:$auxiliary, 0), 1312 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1313 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1314 >; 1315 1316 def : GCNPat< 1317 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1318 timm:$auxiliary, 0), 1319 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1320 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1321 >; 1322 1323 def : GCNPat< 1324 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1325 timm:$auxiliary, timm), 1326 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1327 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1328 >; 1329 1330 def : GCNPat< 1331 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1332 timm:$auxiliary, timm), 1333 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1334 getVregSrcForVT<vt>.ret:$vdata, 1335 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1336 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1337 0, (extract_swz $auxiliary)) 1338 >; 1339} 1340 1341defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1342defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1343defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1344defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1345defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1346defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1347defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1348defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1349 1350let SubtargetPredicate = HasUnpackedD16VMem in { 1351 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1352 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1353 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1354 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1355 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1356 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1357} // End HasUnpackedD16VMem. 1358 1359let SubtargetPredicate = HasPackedD16VMem in { 1360 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1361 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1362 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1363 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1365 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1366 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1367 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1368 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1369} // End HasPackedD16VMem. 1370 1371defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1372defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1373defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1374defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1375defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1376defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1377defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1378defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1379defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1380defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1381defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1382defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1383defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1384defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1385 1386//===----------------------------------------------------------------------===// 1387// buffer_atomic patterns 1388//===----------------------------------------------------------------------===// 1389 1390multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1391 foreach RtnMode = ["ret", "noret"] in { 1392 1393 defvar Op = !cast<SDPatternOperator>(OpPrefix # "_" # RtnMode 1394 # !if(isIntr, "", "_" # vt.Size)); 1395 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1396 1397 def : GCNPat< 1398 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1399 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1400 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1401 >; 1402 1403 def : GCNPat< 1404 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1405 vt:$vdata_in)), 1406 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1407 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1408 >; 1409 1410 } // end foreach RtnMode 1411} 1412 1413multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1414 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1415} 1416 1417multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1418 foreach RtnMode = ["ret", "noret"] in { 1419 1420 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode 1421 # "_" # vt.Size); 1422 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1423 1424 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1425 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1426 offset:$offset); 1427 def : GCNPat< 1428 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1429 !if(!eq(RtnMode, "ret"), 1430 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1431 !if(!eq(vt, i32), sub0, sub0_sub1)), 1432 OffsetResDag) 1433 >; 1434 1435 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1436 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1437 SCSrc_b32:$soffset, offset:$offset); 1438 def : GCNPat< 1439 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1440 data_vt:$vdata_in)), 1441 !if(!eq(RtnMode, "ret"), 1442 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1443 !if(!eq(vt, i32), sub0, sub0_sub1)), 1444 Addr64ResDag) 1445 >; 1446 1447 } // end foreach RtnMode 1448} 1449 1450foreach Ty = [i32, i64] in { 1451 1452defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1453 1454defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1455defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1456defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1457defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1458defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1459defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1460defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1461defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1462defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1463defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1464defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1465defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1466 1467} // end foreach Ty 1468 1469defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1470defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1471 1472multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1473 list<string> RtnModes = ["ret", "noret"]> { 1474 foreach RtnMode = RtnModes in { 1475 1476 defvar Op = !cast<SDPatternOperator>(!if(!eq(RtnMode, "none"), 1477 OpPrefix, OpPrefix # "_" # RtnMode)); 1478 defvar InstSuffix = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1479 "_RTN", ""); 1480 defvar CachePolicy = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1481 (set_glc $cachepolicy), (timm:$cachepolicy)); 1482 1483 def : GCNPat< 1484 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1485 timm:$offset, timm:$cachepolicy, 0)), 1486 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1487 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1488 (as_i16timm $offset), CachePolicy) 1489 >; 1490 1491 def : GCNPat< 1492 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1493 timm:$offset, timm:$cachepolicy, timm)), 1494 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1495 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1496 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1497 >; 1498 1499 def : GCNPat< 1500 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1501 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1502 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1503 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1504 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1505 >; 1506 1507 def : GCNPat< 1508 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1509 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1510 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1511 getVregSrcForVT<vt>.ret:$vdata_in, 1512 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1513 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1514 >; 1515 1516 } // end foreach RtnMode 1517} 1518 1519defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1520defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1521defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1522defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1523defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1524defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1525defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1526defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1527defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1528defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1529defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1530defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1531defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1532defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["none"]>; 1533defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1534defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1535defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1536defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1537defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1538defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1539defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1540defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1545 1546let SubtargetPredicate = isGFX6GFX7GFX10 in { 1547 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1548 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1549 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1550 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1551} 1552 1553class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1554 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1555 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1556 [{ return SDValue(N, 0).use_empty(); }]> { 1557 1558 let GISelPredicateCode = [{ 1559 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1560 }]; 1561} 1562 1563multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1564 string opcode> { 1565 def : GCNPat< 1566 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1567 0, i32:$soffset, timm:$offset, 1568 timm:$cachepolicy, 0), 1569 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1570 (as_i16timm $offset), $cachepolicy) 1571 >; 1572 1573 def : GCNPat< 1574 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1575 0, i32:$soffset, timm:$offset, 1576 timm:$cachepolicy, timm), 1577 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1578 (as_i16timm $offset), $cachepolicy) 1579 >; 1580 1581 def : GCNPat< 1582 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1583 i32:$voffset, i32:$soffset, timm:$offset, 1584 timm:$cachepolicy, 0), 1585 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1586 (as_i16timm $offset), $cachepolicy) 1587 >; 1588 1589 def : GCNPat< 1590 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1591 i32:$voffset, i32:$soffset, timm:$offset, 1592 timm:$cachepolicy, timm), 1593 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1594 getVregSrcForVT<vt>.ret:$vdata_in, 1595 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1596 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1597 >; 1598} 1599 1600let SubtargetPredicate = HasAtomicFaddInsts in { 1601defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1602defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1603} 1604 1605let SubtargetPredicate = isGFX90APlus in { 1606 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1607 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1608 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1609 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1610 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1611 1612 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1613 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1614 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1615} // End SubtargetPredicate = isGFX90APlus 1616 1617foreach RtnMode = ["ret", "noret"] in { 1618 1619defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap # "_" # RtnMode); 1620defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1621defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1622 (timm:$cachepolicy)); 1623 1624defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1625 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1626 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1627def : GCNPat< 1628 (Op 1629 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1630 timm:$offset, timm:$cachepolicy, 0), 1631 !if(!eq(RtnMode, "ret"), 1632 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1633 OffsetResDag) 1634>; 1635 1636defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1637 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1638 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1639 CachePolicy); 1640def : GCNPat< 1641 (Op 1642 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1643 0, i32:$soffset, timm:$offset, 1644 timm:$cachepolicy, timm), 1645 !if(!eq(RtnMode, "ret"), 1646 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1647 IdxenResDag) 1648>; 1649 1650defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1651 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1652 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1653 CachePolicy); 1654def : GCNPat< 1655 (Op 1656 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1657 i32:$voffset, i32:$soffset, timm:$offset, 1658 timm:$cachepolicy, 0), 1659 !if(!eq(RtnMode, "ret"), 1660 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1661 OffenResDag) 1662>; 1663 1664defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1665 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1666 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1667 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1668def : GCNPat< 1669 (Op 1670 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1671 i32:$voffset, i32:$soffset, timm:$offset, 1672 timm:$cachepolicy, timm), 1673 !if(!eq(RtnMode, "ret"), 1674 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1675 BothenResDag) 1676>; 1677 1678} // end foreach RtnMode 1679 1680class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1681 PatFrag constant_ld> : GCNPat < 1682 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1683 i16:$offset))), 1684 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1685 >; 1686 1687multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1688 ValueType vt, PatFrag atomic_ld> { 1689 def : GCNPat < 1690 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1691 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1692 >; 1693 1694 def : GCNPat < 1695 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1696 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1697 >; 1698} 1699 1700let SubtargetPredicate = isGFX6GFX7 in { 1701def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1702def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1703def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1704def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1705def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1706def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1707 1708defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1709defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1710} // End SubtargetPredicate = isGFX6GFX7 1711 1712multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1713 PatFrag ld> { 1714 1715 def : GCNPat < 1716 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1717 (Instr_OFFSET $srsrc, $soffset, $offset) 1718 >; 1719} 1720 1721let OtherPredicates = [Has16BitInsts] in { 1722 1723defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1724defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1725defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1726defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1727defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1728defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1729 1730defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1731 1732} // End OtherPredicates = [Has16BitInsts] 1733 1734multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1735 MUBUF_Pseudo InstrOffset, 1736 ValueType vt, PatFrag ld> { 1737 def : GCNPat < 1738 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1739 i32:$soffset, u16imm:$offset))), 1740 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1741 >; 1742 1743 def : GCNPat < 1744 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1745 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1746 >; 1747} 1748 1749// XXX - Is it possible to have a complex pattern in a PatFrag? 1750multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1751 MUBUF_Pseudo InstrOffset, 1752 ValueType vt, PatFrag ld_frag> { 1753 def : GCNPat < 1754 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1755 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1756 >; 1757 1758 def : GCNPat < 1759 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1760 (InstrOffset $srsrc, $soffset, $offset, $in) 1761 >; 1762} 1763 1764let OtherPredicates = [DisableFlatScratch] in { 1765defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1766defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1767defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1768defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1769defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1770defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1771defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1772defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1773defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1774defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1775 1776foreach vt = Reg32Types.types in { 1777defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1778} 1779defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1780defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1781defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1782 1783let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1784defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1785defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1786defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1787defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1788defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1789defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1790 1791defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1792defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1793defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1794defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1795defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1796defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1797} 1798 1799} // End OtherPredicates = [DisableFlatScratch] 1800 1801multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1802 ValueType vt, PatFrag atomic_st> { 1803 // Store follows atomic op convention so address is first 1804 def : GCNPat < 1805 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1806 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1807 >; 1808 1809 def : GCNPat < 1810 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1811 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1812 >; 1813} 1814let SubtargetPredicate = isGFX6GFX7 in { 1815defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1816defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1817defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1818defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1819defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1820defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1821} // End Predicates = isGFX6GFX7 1822 1823 1824multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1825 PatFrag st> { 1826 1827 def : GCNPat < 1828 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1829 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1830 >; 1831} 1832 1833defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1834defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1835 1836multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1837 MUBUF_Pseudo InstrOffset, 1838 ValueType vt, PatFrag st, 1839 RegisterClass rc = VGPR_32> { 1840 def : GCNPat < 1841 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1842 i32:$soffset, u16imm:$offset)), 1843 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1844 >; 1845 1846 def : GCNPat < 1847 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1848 u16imm:$offset)), 1849 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1850 >; 1851} 1852 1853let OtherPredicates = [DisableFlatScratch] in { 1854defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1855defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1856defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1857defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1858 1859foreach vt = Reg32Types.types in { 1860defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1861} 1862 1863defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1864defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1865defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1866 1867 1868let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1869 // Hiding the extract high pattern in the PatFrag seems to not 1870 // automatically increase the complexity. 1871let AddedComplexity = 1 in { 1872defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1873defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1874} 1875} 1876} // End OtherPredicates = [DisableFlatScratch] 1877 1878//===----------------------------------------------------------------------===// 1879// MTBUF Patterns 1880//===----------------------------------------------------------------------===// 1881 1882//===----------------------------------------------------------------------===// 1883// tbuffer_load/store_format patterns 1884//===----------------------------------------------------------------------===// 1885 1886multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1887 string opcode, ValueType memoryVt = vt> { 1888 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1889 1890 def : GCNPat< 1891 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1892 timm:$format, timm:$auxiliary, 0)), 1893 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1894 (as_i8timm $format), 1895 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1896 >; 1897 1898 def : GCNPat< 1899 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1900 timm:$format, timm:$auxiliary, timm)), 1901 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1902 (as_i8timm $format), 1903 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1904 >; 1905 1906 def : GCNPat< 1907 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1908 timm:$format, timm:$auxiliary, 0)), 1909 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1910 (as_i8timm $format), 1911 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1912 >; 1913 1914 def : GCNPat< 1915 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1916 timm:$format, timm:$auxiliary, timm)), 1917 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1918 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1919 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1920 (as_i8timm $format), 1921 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1922 >; 1923} 1924 1925defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1926defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1927defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1928defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1929defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1930defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1931defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1932defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1933 1934let SubtargetPredicate = HasUnpackedD16VMem in { 1935 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1936 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1937 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1938 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1939 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1940} // End HasUnpackedD16VMem. 1941 1942let SubtargetPredicate = HasPackedD16VMem in { 1943 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1944 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1945 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1946 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1947 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1948} // End HasPackedD16VMem. 1949 1950multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1951 string opcode, ValueType memoryVt = vt> { 1952 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1953 1954 def : GCNPat< 1955 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1956 timm:$format, timm:$auxiliary, 0), 1957 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1958 (as_i16timm $offset), (as_i8timm $format), 1959 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1960 >; 1961 1962 def : GCNPat< 1963 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1964 timm:$format, timm:$auxiliary, timm), 1965 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1966 (as_i16timm $offset), (as_i8timm $format), 1967 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1968 >; 1969 1970 def : GCNPat< 1971 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1972 timm:$format, timm:$auxiliary, 0), 1973 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1974 (as_i16timm $offset), (as_i8timm $format), 1975 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1976 >; 1977 1978 def : GCNPat< 1979 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1980 timm:$offset, timm:$format, timm:$auxiliary, timm), 1981 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1982 getVregSrcForVT<vt>.ret:$vdata, 1983 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1984 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1985 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1986 >; 1987} 1988 1989defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1990defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1991defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1992defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1993defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1994defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1995defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1996defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1997 1998let SubtargetPredicate = HasUnpackedD16VMem in { 1999 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2000 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2001 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2002 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2003 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2004} // End HasUnpackedD16VMem. 2005 2006let SubtargetPredicate = HasPackedD16VMem in { 2007 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2008 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2009 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2010 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2011 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2012} // End HasPackedD16VMem. 2013 2014//===----------------------------------------------------------------------===// 2015// Target-specific instruction encodings. 2016//===----------------------------------------------------------------------===// 2017 2018//===----------------------------------------------------------------------===// 2019// Base ENC_MUBUF for GFX6, GFX7, GFX10. 2020//===----------------------------------------------------------------------===// 2021 2022class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2023 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2024 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2025 let Inst{12} = ps.offen; 2026 let Inst{13} = ps.idxen; 2027 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2028 let Inst{16} = ps.lds; 2029 let Inst{24-18} = op; 2030 let Inst{31-26} = 0x38; 2031 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2032 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2033 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2034 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2035 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2036 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2037} 2038 2039class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2040 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2041 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2042 let Inst{25} = op{7}; 2043} 2044 2045class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2046 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2047 let Inst{15} = ps.addr64; 2048} 2049 2050//===----------------------------------------------------------------------===// 2051// MUBUF - GFX10. 2052//===----------------------------------------------------------------------===// 2053 2054let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2055 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2056 def _BOTHEN_gfx10 : 2057 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2058 def _IDXEN_gfx10 : 2059 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2060 def _OFFEN_gfx10 : 2061 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2062 def _OFFSET_gfx10 : 2063 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2064 } 2065 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2066 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2067 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 2068 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2069 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 2070 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2071 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 2072 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2073 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 2074 2075 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2076 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 2077 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2078 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 2079 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2080 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 2081 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2082 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 2083 } 2084 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2085 def _BOTHEN_RTN_gfx10 : 2086 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2087 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2088 def _IDXEN_RTN_gfx10 : 2089 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2090 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2091 def _OFFEN_RTN_gfx10 : 2092 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2093 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2094 def _OFFSET_RTN_gfx10 : 2095 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2096 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2097 } 2098 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2099 MUBUF_Real_Atomics_RTN_gfx10<op> { 2100 def _BOTHEN_gfx10 : 2101 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2102 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2103 def _IDXEN_gfx10 : 2104 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2105 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2106 def _OFFEN_gfx10 : 2107 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2108 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2109 def _OFFSET_gfx10 : 2110 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2111 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2112 } 2113} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2114 2115defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2116defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2117defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2118defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2119defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2120defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2121defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2122defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2123// FIXME-GFX10: Add following instructions: 2124//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2125//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2126defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2127defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2128defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2129defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2130defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2131defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2132defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2133defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2134 2135def BUFFER_GL0_INV_gfx10 : 2136 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2137def BUFFER_GL1_INV_gfx10 : 2138 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2139 2140//===----------------------------------------------------------------------===// 2141// MUBUF - GFX6, GFX7, GFX10. 2142//===----------------------------------------------------------------------===// 2143 2144let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2145 multiclass MUBUF_Real_gfx6<bits<8> op> { 2146 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2147 } 2148} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2149 2150let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2151 multiclass MUBUF_Real_gfx7<bits<8> op> { 2152 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2153 } 2154} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2155 2156let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2157 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2158 def _ADDR64_gfx6_gfx7 : 2159 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2160 def _BOTHEN_gfx6_gfx7 : 2161 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2162 def _IDXEN_gfx6_gfx7 : 2163 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2164 def _OFFEN_gfx6_gfx7 : 2165 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2166 def _OFFSET_gfx6_gfx7 : 2167 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2168 } 2169 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2170 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2171 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2172 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2173 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2174 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2175 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2176 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2177 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2178 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2179 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2180 2181 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2182 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2183 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2184 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2185 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2186 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2187 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2188 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2189 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2190 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2191 } 2192 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2193 def _ADDR64_gfx6_gfx7 : 2194 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2195 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2196 def _BOTHEN_gfx6_gfx7 : 2197 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2198 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2199 def _IDXEN_gfx6_gfx7 : 2200 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2201 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2202 def _OFFEN_gfx6_gfx7 : 2203 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2204 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2205 def _OFFSET_gfx6_gfx7 : 2206 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2207 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2208 2209 def _ADDR64_RTN_gfx6_gfx7 : 2210 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2211 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2212 def _BOTHEN_RTN_gfx6_gfx7 : 2213 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2214 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2215 def _IDXEN_RTN_gfx6_gfx7 : 2216 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2217 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2218 def _OFFEN_RTN_gfx6_gfx7 : 2219 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2220 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2221 def _OFFSET_RTN_gfx6_gfx7 : 2222 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2223 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2224 } 2225} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2226 2227multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2228 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2229 2230multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2231 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2232 2233multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2234 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2235 2236// FIXME-GFX6: Following instructions are available only on GFX6. 2237//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2238//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2239 2240defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2241defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2242defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2243defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2244defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2245defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2246defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2247defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2248defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2249defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2250defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2251defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2252defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2253defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2254defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2255defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2256defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2257defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2258defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2259defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2260defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2261defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2262 2263defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2264defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2265defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2266defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2267defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2268defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2269defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2270defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2271defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2272defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2273defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2274defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2275defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2276defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2277defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2278defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2279defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2280defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2281defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2282defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2283defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2284defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2285defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2286defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2287defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2288defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2289defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2290defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2291defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2292// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2293defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2294defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2295defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2296 2297defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2298 2299defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2300defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2301def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2302 2303//===----------------------------------------------------------------------===// 2304// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2305//===----------------------------------------------------------------------===// 2306 2307class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2308 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2309 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2310 let Inst{12} = ps.offen; 2311 let Inst{13} = ps.idxen; 2312 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2313 let Inst{18-16} = op; 2314 let Inst{31-26} = 0x3a; //encoding 2315 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2316 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2317 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2318 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2319 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2320 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2321} 2322 2323//===----------------------------------------------------------------------===// 2324// MTBUF - GFX10. 2325//===----------------------------------------------------------------------===// 2326 2327class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2328 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2329 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2330 let Inst{25-19} = format; 2331 let Inst{53} = op{3}; 2332} 2333 2334let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2335 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2336 def _BOTHEN_gfx10 : 2337 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2338 def _IDXEN_gfx10 : 2339 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2340 def _OFFEN_gfx10 : 2341 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2342 def _OFFSET_gfx10 : 2343 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2344 } 2345} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2346 2347defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2348defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2349defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2350defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2351defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2352defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2353defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2354defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2355 2356//===----------------------------------------------------------------------===// 2357// MTBUF - GFX6, GFX7, GFX10. 2358//===----------------------------------------------------------------------===// 2359 2360class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2361 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2362 let Inst{15} = ps.addr64; 2363 let Inst{22-19} = dfmt; 2364 let Inst{25-23} = nfmt; 2365} 2366 2367let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2368 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2369 def _ADDR64_gfx6_gfx7 : 2370 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2371 def _BOTHEN_gfx6_gfx7 : 2372 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2373 def _IDXEN_gfx6_gfx7 : 2374 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2375 def _OFFEN_gfx6_gfx7 : 2376 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2377 def _OFFSET_gfx6_gfx7 : 2378 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2379 } 2380} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2381 2382multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2383 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2384 2385defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2386defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2387defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2388defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2389defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2390defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2391defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2392defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2393 2394//===----------------------------------------------------------------------===// 2395// GFX8, GFX9 (VI). 2396//===----------------------------------------------------------------------===// 2397 2398class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2399 bit has_sccb = ps.has_sccb> : 2400 MUBUF_Real<ps>, 2401 Enc64, 2402 SIMCInstr<ps.PseudoInstr, Enc>, 2403 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2404 !if(ps.IsAtomicRet, 1, ?))> { 2405 2406 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2407 let Inst{12} = ps.offen; 2408 let Inst{13} = ps.idxen; 2409 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2410 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2411 let Inst{16} = ps.lds; 2412 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2413 let Inst{24-18} = op; 2414 let Inst{31-26} = 0x38; //encoding 2415 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2416 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2417 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2418 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2419} 2420 2421class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2422 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2423 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2424 let DecoderNamespace = "GFX8"; 2425 2426 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2427} 2428 2429class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2430 bit has_sccb = ps.has_sccb> : 2431 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2432 let AssemblerPredicate = isGFX90APlus; 2433 let DecoderNamespace = "GFX90A"; 2434 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2435 !subst("$tfe", "", ps.AsmOperands)); 2436 2437 let Inst{55} = acc; 2438} 2439 2440class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2441 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2442 let AssemblerPredicate = isGFX940Plus; 2443 let DecoderNamespace = "GFX9"; 2444 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2445 2446 let Inst{55} = acc; 2447} 2448 2449multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2450 def _vi : MUBUF_Real_vi<op, ps>; 2451 2452 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2453 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2454 2455 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2456 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2457 let SubtargetPredicate = isGFX90AOnly; 2458 let AssemblerPredicate = isGFX90AOnly; 2459 } 2460 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2461 } 2462} 2463 2464multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2465 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2466 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2467 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2468 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2469} 2470 2471multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2472 2473 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2474 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2475 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2476 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2477 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2478 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2479 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2480 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2481 2482 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2483 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2484 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2485 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2486 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2487 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2488 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2489 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2490 2491 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2492 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2493 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2494 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2495 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2496 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2497 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2498 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2499 2500 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2501 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2502 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2503 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2504 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2505 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2506 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2507 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2508} 2509 2510class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2511 MUBUF_Real<ps>, 2512 Enc64, 2513 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2514 let AssemblerPredicate=HasUnpackedD16VMem; 2515 let DecoderNamespace="GFX80_UNPACKED"; 2516 2517 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2518 let Inst{12} = ps.offen; 2519 let Inst{13} = ps.idxen; 2520 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2521 let Inst{16} = ps.lds; 2522 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2523 let Inst{24-18} = op; 2524 let Inst{31-26} = 0x38; //encoding 2525 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2526 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2527 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2528 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2529 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2530} 2531 2532multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2533 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2534 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2535 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2536 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2537} 2538 2539multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2540 MUBUF_Real_AllAddr_vi<op> { 2541 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2542 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2543 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2544 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2545} 2546 2547defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2548defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2549defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2550defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2551defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2552defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2553defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2554defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2555let SubtargetPredicate = HasUnpackedD16VMem in { 2556 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2557 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2558 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2559 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2560 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2561 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2562 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2563 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2564} // End HasUnpackedD16VMem. 2565let SubtargetPredicate = HasPackedD16VMem in { 2566 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2567 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2568 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2569 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2570 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2571 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2572 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2573 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2574} // End HasPackedD16VMem. 2575defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2576defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2577defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2578defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2579defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2580defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2581defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2582defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2583defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2584defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2585defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2586defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2587defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2588defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2589defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2590defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2591 2592defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2593defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2594defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2595defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2596defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2597defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2598 2599defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2600defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2601 2602defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2603defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2604defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2605defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2606defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2607defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2608defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2609defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2610defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2611defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2612defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2613defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2614defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2615 2616defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2617defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2618defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2619defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2620defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2621defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2622defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2623defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2624defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2625defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2626defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2627defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2628defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2629 2630defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2631 2632let AssemblerPredicate = isGFX8GFX9 in { 2633def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2634def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2635} // End AssemblerPredicate = isGFX8GFX9 2636 2637let SubtargetPredicate = HasAtomicFaddInsts in { 2638 2639defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2640defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2641 2642} // End SubtargetPredicate = HasAtomicFaddInsts 2643 2644let SubtargetPredicate = isGFX90APlus in { 2645 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2646 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2647 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2648} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2649 2650def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2651 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2652 let AssemblerPredicate = isGFX90AOnly; 2653 let SubtargetPredicate = isGFX90AOnly; 2654} 2655def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2656 2657let SubtargetPredicate = isGFX940Plus in { 2658def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2659def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2660} 2661 2662class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2663 MTBUF_Real<ps>, 2664 Enc64, 2665 SIMCInstr<ps.PseudoInstr, Enc> { 2666 2667 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2668 let Inst{12} = ps.offen; 2669 let Inst{13} = ps.idxen; 2670 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2671 let Inst{18-15} = op; 2672 let Inst{22-19} = dfmt; 2673 let Inst{25-23} = nfmt; 2674 let Inst{31-26} = 0x3a; //encoding 2675 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2676 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2677 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2678 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2679 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2680 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2681 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2682} 2683 2684class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2685 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2686 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2687 let DecoderNamespace = "GFX8"; 2688 2689 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2690} 2691 2692class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2693 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2694 let AssemblerPredicate = isGFX90APlus; 2695 let DecoderNamespace = "GFX90A"; 2696 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2697 2698 let Inst{55} = acc; 2699} 2700 2701multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2702 def _vi : MTBUF_Real_vi<op, ps>; 2703 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2704} 2705 2706multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2707 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2708 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2709 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2710 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2711} 2712 2713class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2714 MTBUF_Real<ps>, 2715 Enc64, 2716 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2717 let AssemblerPredicate=HasUnpackedD16VMem; 2718 let DecoderNamespace="GFX80_UNPACKED"; 2719 2720 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2721 let Inst{12} = ps.offen; 2722 let Inst{13} = ps.idxen; 2723 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2724 let Inst{18-15} = op; 2725 let Inst{22-19} = dfmt; 2726 let Inst{25-23} = nfmt; 2727 let Inst{31-26} = 0x3a; //encoding 2728 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2729 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2730 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2731 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2732 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2733 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2734} 2735 2736multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2737 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2738 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2739 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2740 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2741} 2742 2743defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2744defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2745defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2746defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2747defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2748defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2749defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2750defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2751let SubtargetPredicate = HasUnpackedD16VMem in { 2752 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2753 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2754 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2755 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2756 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2757 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2758 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2759 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2760} // End HasUnpackedD16VMem. 2761let SubtargetPredicate = HasPackedD16VMem in { 2762 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2763 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2764 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2765 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2766 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2767 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2768 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2769 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2770} // End HasUnpackedD16VMem. 2771 2772def MUBUFInfoTable : GenericTable { 2773 let FilterClass = "MUBUF_Pseudo"; 2774 let CppTypeName = "MUBUFInfo"; 2775 let Fields = [ 2776 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2777 "IsBufferInv" 2778 ]; 2779 2780 let PrimaryKey = ["Opcode"]; 2781 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2782} 2783 2784def getMUBUFInfoFromOpcode : SearchIndex { 2785 let Table = MUBUFInfoTable; 2786 let Key = ["Opcode"]; 2787} 2788 2789def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2790 let Table = MUBUFInfoTable; 2791 let Key = ["BaseOpcode", "elements"]; 2792} 2793 2794def MTBUFInfoTable : GenericTable { 2795 let FilterClass = "MTBUF_Pseudo"; 2796 let CppTypeName = "MTBUFInfo"; 2797 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2798 2799 let PrimaryKey = ["Opcode"]; 2800 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2801} 2802 2803def getMTBUFInfoFromOpcode : SearchIndex { 2804 let Table = MTBUFInfoTable; 2805 let Key = ["Opcode"]; 2806} 2807 2808def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2809 let Table = MTBUFInfoTable; 2810 let Key = ["BaseOpcode", "elements"]; 2811} 2812