1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MUBUFLdsTable <bit is_lds, string Name> { 39 bit IsLds = is_lds; 40 string OpName = Name; 41} 42 43class MTBUFAddr64Table <bit is_addr64, string Name> { 44 bit IsAddr64 = is_addr64; 45 string OpName = Name; 46} 47 48//===----------------------------------------------------------------------===// 49// MTBUF classes 50//===----------------------------------------------------------------------===// 51 52class MTBUFGetBaseOpcode<string Op> { 53 string ret = !subst("FORMAT_XY", "FORMAT_X", 54 !subst("FORMAT_XYZ", "FORMAT_X", 55 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 56} 57 58 59class MTBUF_Pseudo <string opName, dag outs, dag ins, 60 string asmOps, list<dag> pattern=[]> : 61 InstSI<outs, ins, "", pattern>, 62 SIMCInstr<opName, SIEncodingFamily.NONE> { 63 64 let isPseudo = 1; 65 let isCodeGenOnly = 1; 66 let Size = 8; 67 let UseNamedOperandTable = 1; 68 69 string Mnemonic = opName; 70 string AsmOperands = asmOps; 71 72 Instruction Opcode = !cast<Instruction>(NAME); 73 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 74 75 let VM_CNT = 1; 76 let EXP_CNT = 1; 77 let MTBUF = 1; 78 let Uses = [EXEC]; 79 let hasSideEffects = 0; 80 let SchedRW = [WriteVMEM]; 81 82 let AsmMatchConverter = "cvtMtbuf"; 83 84 bits<1> offen = 0; 85 bits<1> idxen = 0; 86 bits<1> addr64 = 0; 87 bits<1> has_vdata = 1; 88 bits<1> has_vaddr = 1; 89 bits<1> has_glc = 1; 90 bits<1> has_dlc = 1; 91 bits<1> glc_value = 0; // the value for glc if no such operand 92 bits<1> dlc_value = 0; // the value for dlc if no such operand 93 bits<1> has_srsrc = 1; 94 bits<1> has_soffset = 1; 95 bits<1> has_offset = 1; 96 bits<1> has_slc = 1; 97 bits<1> has_tfe = 1; 98 bits<4> elements = 0; 99 bits<1> has_sccb = 1; 100 bits<1> sccb_value = 0; 101} 102 103class MTBUF_Real <MTBUF_Pseudo ps> : 104 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 105 106 let isPseudo = 0; 107 let isCodeGenOnly = 0; 108 109 let VM_CNT = 1; 110 let EXP_CNT = 1; 111 let MTBUF = 1; 112 113 // copy relevant pseudo op flags 114 let UseNamedOperandTable = ps.UseNamedOperandTable; 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 let SchedRW = ps.SchedRW; 121 let mayLoad = ps.mayLoad; 122 let mayStore = ps.mayStore; 123 let IsAtomicRet = ps.IsAtomicRet; 124 let IsAtomicNoRet = ps.IsAtomicNoRet; 125 126 bits<12> offset; 127 bits<5> cpol; 128 bits<7> format; 129 bits<8> vaddr; 130 bits<10> vdata; 131 bits<7> srsrc; 132 bits<1> tfe; 133 bits<8> soffset; 134 135 bits<4> dfmt = format{3-0}; 136 bits<3> nfmt = format{6-4}; 137 138 // GFX90A+ only: instruction uses AccVGPR for data 139 // Bit supersedes tfe. 140 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 141} 142 143class getMTBUFInsDA<list<RegisterClass> vdataList, 144 list<RegisterClass> vaddrList=[]> { 145 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 146 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 147 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 148 dag InsNoData = !if(!empty(vaddrList), 149 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 150 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 151 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 152 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 153 ); 154 dag InsData = !if(!empty(vaddrList), 155 (ins vdata_op:$vdata, SReg_128:$srsrc, 156 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 157 TFE:$tfe, SWZ:$swz), 158 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 159 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 160 TFE:$tfe, SWZ:$swz) 161 ); 162 dag ret = !if(!empty(vdataList), InsNoData, InsData); 163} 164 165class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 166 dag ret = 167 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 168 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 169 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 170 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 171 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 172 (ins)))))); 173} 174 175class getMTBUFAsmOps<int addrKind> { 176 string Pfx = 177 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 178 !if(!eq(addrKind, BUFAddrKind.OffEn), 179 "$vaddr, $srsrc,$format $soffset offen", 180 !if(!eq(addrKind, BUFAddrKind.IdxEn), 181 "$vaddr, $srsrc,$format $soffset idxen", 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 183 "$vaddr, $srsrc,$format $soffset idxen offen", 184 !if(!eq(addrKind, BUFAddrKind.Addr64), 185 "$vaddr, $srsrc,$format $soffset addr64", 186 ""))))); 187 string ret = Pfx # "$offset"; 188} 189 190class MTBUF_SetupAddr<int addrKind> { 191 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 192 !eq(addrKind, BUFAddrKind.BothEn)); 193 194 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 195 !eq(addrKind, BUFAddrKind.BothEn)); 196 197 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 198 199 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 200} 201 202class MTBUF_Load_Pseudo <string opName, 203 int addrKind, 204 RegisterClass vdataClass, 205 int elems, 206 list<dag> pattern=[], 207 // Workaround bug bz30254 208 int addrKindCopy = addrKind> 209 : MTBUF_Pseudo<opName, 210 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 211 getMTBUFIns<addrKindCopy>.ret, 212 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 213 pattern>, 214 MTBUF_SetupAddr<addrKindCopy> { 215 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 216 let mayLoad = 1; 217 let mayStore = 0; 218 let elements = elems; 219} 220 221multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 222 int elems> { 223 224 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 225 MTBUFAddr64Table<0, NAME>; 226 227 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 228 MTBUFAddr64Table<1, NAME>; 229 230 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 231 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 232 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 233 234 let DisableWQM = 1 in { 235 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 236 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 237 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 238 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 239 } 240} 241 242class MTBUF_Store_Pseudo <string opName, 243 int addrKind, 244 RegisterClass vdataClass, 245 int elems, 246 list<dag> pattern=[], 247 // Workaround bug bz30254 248 int addrKindCopy = addrKind, 249 RegisterClass vdataClassCopy = vdataClass> 250 : MTBUF_Pseudo<opName, 251 (outs), 252 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 253 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 254 pattern>, 255 MTBUF_SetupAddr<addrKindCopy> { 256 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 257 let mayLoad = 0; 258 let mayStore = 1; 259 let elements = elems; 260} 261 262multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 263 int elems> { 264 265 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 266 MTBUFAddr64Table<0, NAME>; 267 268 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 269 MTBUFAddr64Table<1, NAME>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUFGetBaseOpcode<string Op> { 289 string ret = !subst("DWORDX2", "DWORD", 290 !subst("DWORDX3", "DWORD", 291 !subst("DWORDX4", "DWORD", Op))); 292} 293 294class MUBUF_Pseudo <string opName, dag outs, dag ins, 295 string asmOps, list<dag> pattern=[]> : 296 InstSI<outs, ins, "", pattern>, 297 SIMCInstr<opName, SIEncodingFamily.NONE> { 298 299 let isPseudo = 1; 300 let isCodeGenOnly = 1; 301 let Size = 8; 302 let UseNamedOperandTable = 1; 303 304 string Mnemonic = opName; 305 string AsmOperands = asmOps; 306 307 Instruction Opcode = !cast<Instruction>(NAME); 308 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 309 310 let VM_CNT = 1; 311 let EXP_CNT = 1; 312 let MUBUF = 1; 313 let Uses = [EXEC]; 314 let hasSideEffects = 0; 315 let SchedRW = [WriteVMEM]; 316 317 let AsmMatchConverter = "cvtMubuf"; 318 319 bits<1> offen = 0; 320 bits<1> idxen = 0; 321 bits<1> addr64 = 0; 322 bits<1> lds = 0; 323 bits<1> has_vdata = 1; 324 bits<1> has_vaddr = 1; 325 bits<1> has_glc = 1; 326 bits<1> has_dlc = 1; 327 bits<1> glc_value = 0; // the value for glc if no such operand 328 bits<1> dlc_value = 0; // the value for dlc if no such operand 329 bits<1> has_srsrc = 1; 330 bits<1> has_soffset = 1; 331 bits<1> has_offset = 1; 332 bits<1> has_slc = 1; 333 bits<1> has_tfe = 1; 334 bits<4> elements = 0; 335 bits<1> has_sccb = 1; 336 bits<1> sccb_value = 0; 337 bits<1> IsBufferInv = 0; 338} 339 340class MUBUF_Real <MUBUF_Pseudo ps> : 341 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 342 343 let isPseudo = 0; 344 let isCodeGenOnly = 0; 345 346 let VM_CNT = 1; 347 let EXP_CNT = 1; 348 let MUBUF = 1; 349 350 // copy relevant pseudo op flags 351 let SubtargetPredicate = ps.SubtargetPredicate; 352 let AsmMatchConverter = ps.AsmMatchConverter; 353 let OtherPredicates = ps.OtherPredicates; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 let UseNamedOperandTable = ps.UseNamedOperandTable; 358 let SchedRW = ps.SchedRW; 359 let mayLoad = ps.mayLoad; 360 let mayStore = ps.mayStore; 361 let IsAtomicRet = ps.IsAtomicRet; 362 let IsAtomicNoRet = ps.IsAtomicNoRet; 363 364 bits<12> offset; 365 bits<5> cpol; 366 bits<8> vaddr; 367 bits<10> vdata; 368 bits<7> srsrc; 369 bits<1> tfe; 370 bits<8> soffset; 371 372 // GFX90A+ only: instruction uses AccVGPR for data 373 // Bit supersedes tfe. 374 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 375} 376 377 378// For cache invalidation instructions. 379class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 380 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 381 382 let AsmMatchConverter = ""; 383 384 let hasSideEffects = 1; 385 let mayLoad = 0; 386 let mayStore = 0; 387 388 let IsBufferInv = 1; 389 // Set everything else to 0. 390 let offen = 0; 391 let idxen = 0; 392 let addr64 = 0; 393 let has_vdata = 0; 394 let has_vaddr = 0; 395 let has_glc = 0; 396 let has_dlc = 0; 397 let glc_value = 0; 398 let dlc_value = 0; 399 let has_srsrc = 0; 400 let has_soffset = 0; 401 let has_offset = 0; 402 let has_slc = 0; 403 let has_tfe = 0; 404 let has_sccb = 0; 405 let sccb_value = 0; 406} 407 408class getMUBUFInsDA<list<RegisterClass> vdataList, 409 list<RegisterClass> vaddrList=[], 410 bit isLds = 0> { 411 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 412 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 413 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 414 dag InsNoData = !if(!empty(vaddrList), 415 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 416 offset:$offset, CPol_0:$cpol), 417 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 418 offset:$offset, CPol_0:$cpol) 419 ); 420 dag InsData = !if(!empty(vaddrList), 421 (ins vdata_op:$vdata, SReg_128:$srsrc, 422 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 423 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 424 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 425 ); 426 dag ret = !con( 427 !if(!empty(vdataList), InsNoData, InsData), 428 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 429 ); 430} 431 432class getMUBUFElements<ValueType vt> { 433 int ret = 434 !if(!eq(vt, f16), 1, 435 !if(!eq(vt, v2f16), 2, 436 !if(!eq(vt, v3f16), 3, 437 !if(!eq(vt, v4f16), 4, 438 !if(!eq(vt.Size, 32), 1, 439 !if(!eq(vt.Size, 64), 2, 440 !if(!eq(vt.Size, 96), 3, 441 !if(!eq(vt.Size, 128), 4, 0) 442 ) 443 ) 444 ) 445 ) 446 ) 447 ) 448 ); 449} 450 451class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 452 dag ret = 453 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 454 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 455 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 456 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 457 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 458 (ins)))))); 459} 460 461class getMUBUFAsmOps<int addrKind> { 462 string Pfx = 463 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 464 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 465 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 466 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 467 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 468 ""))))); 469 string ret = Pfx # "$offset"; 470} 471 472class MUBUF_SetupAddr<int addrKind> { 473 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 474 !eq(addrKind, BUFAddrKind.BothEn)); 475 476 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 477 !eq(addrKind, BUFAddrKind.BothEn)); 478 479 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 480 481 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 482} 483 484class MUBUF_Load_Pseudo <string opName, 485 int addrKind, 486 ValueType vdata_vt, 487 bit HasTiedDest = 0, 488 bit isLds = 0, 489 list<dag> pattern=[], 490 // Workaround bug bz30254 491 int addrKindCopy = addrKind, 492 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 493 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 494 : MUBUF_Pseudo<opName, 495 (outs vdata_op:$vdata), 496 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 497 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 498 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 499 !if(isLds, " lds", "$tfe") # "$swz", 500 pattern>, 501 MUBUF_SetupAddr<addrKindCopy> { 502 let PseudoInstr = opName # !if(isLds, "_lds", "") # 503 "_" # getAddrName<addrKindCopy>.ret; 504 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 505 506 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 507 let mayLoad = 1; 508 let mayStore = 0; 509 let maybeAtomic = 1; 510 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 511 let has_tfe = !not(isLds); 512 let lds = isLds; 513 let elements = getMUBUFElements<vdata_vt>.ret; 514} 515 516class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 517 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 518 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 519>; 520 521class MUBUF_Addr64_Load_Pat <Instruction inst, 522 ValueType load_vt = i32, 523 SDPatternOperator ld = null_frag> : Pat < 524 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 525 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 526>; 527 528multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 529 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 530 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 531} 532 533 534// FIXME: tfe can't be an operand because it requires a separate 535// opcode because it needs an N+1 register class dest register. 536multiclass MUBUF_Pseudo_Loads<string opName, 537 ValueType load_vt = i32, 538 bit TiedDest = 0, 539 bit isLds = 0> { 540 541 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 542 543 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 544 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 545 546 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 547 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 548 549 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 550 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 551 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 552 553 let DisableWQM = 1 in { 554 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 555 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 556 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 557 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 558 } 559} 560 561multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 562 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 563 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 564} 565 566class MUBUF_Store_Pseudo <string opName, 567 int addrKind, 568 ValueType store_vt, 569 list<dag> pattern=[], 570 // Workaround bug bz30254 571 int addrKindCopy = addrKind> 572 : MUBUF_Pseudo<opName, 573 (outs), 574 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 575 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 576 pattern>, 577 MUBUF_SetupAddr<addrKindCopy> { 578 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 579 let mayLoad = 0; 580 let mayStore = 1; 581 let maybeAtomic = 1; 582 let elements = getMUBUFElements<store_vt>.ret; 583} 584 585multiclass MUBUF_Pseudo_Stores<string opName, 586 ValueType store_vt = i32, 587 SDPatternOperator st = null_frag> { 588 589 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 590 591 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 592 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 593 i16:$offset))]>, 594 MUBUFAddr64Table<0, NAME>; 595 596 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 597 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 598 i16:$offset))]>, 599 MUBUFAddr64Table<1, NAME>; 600 601 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 602 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 603 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 604 605 let DisableWQM = 1 in { 606 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 607 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 608 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 609 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 610 } 611} 612 613class MUBUF_Pseudo_Store_Lds<string opName> 614 : MUBUF_Pseudo<opName, 615 (outs), 616 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 617 " $srsrc, $soffset$offset lds$cpol$swz"> { 618 let mayLoad = 0; 619 let mayStore = 1; 620 let maybeAtomic = 1; 621 622 let has_vdata = 0; 623 let has_vaddr = 0; 624 let has_tfe = 0; 625 let lds = 1; 626 627 let Uses = [EXEC, M0]; 628 let AsmMatchConverter = "cvtMubufLds"; 629} 630 631class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 632 list<RegisterClass> vaddrList=[]> { 633 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 634 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 635 dag ret = !if(vdata_in, 636 !if(!empty(vaddrList), 637 (ins vdata_op:$vdata_in, 638 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 639 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 640 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 641 ), 642 !if(!empty(vaddrList), 643 (ins vdata_op:$vdata, 644 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 645 (ins vdata_op:$vdata, vaddrClass:$vaddr, 646 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 647 )); 648} 649 650class getMUBUFAtomicIns<int addrKind, 651 RegisterClass vdataClass, 652 bit vdata_in, 653 // Workaround bug bz30254 654 RegisterClass vdataClassCopy=vdataClass> { 655 dag ret = 656 !if(!eq(addrKind, BUFAddrKind.Offset), 657 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 658 !if(!eq(addrKind, BUFAddrKind.OffEn), 659 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 660 !if(!eq(addrKind, BUFAddrKind.IdxEn), 661 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 662 !if(!eq(addrKind, BUFAddrKind.BothEn), 663 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 664 !if(!eq(addrKind, BUFAddrKind.Addr64), 665 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 666 (ins)))))); 667} 668 669class MUBUF_Atomic_Pseudo<string opName, 670 int addrKind, 671 dag outs, 672 dag ins, 673 string asmOps, 674 list<dag> pattern=[], 675 // Workaround bug bz30254 676 int addrKindCopy = addrKind> 677 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 678 MUBUF_SetupAddr<addrKindCopy> { 679 let mayStore = 1; 680 let mayLoad = 1; 681 let hasPostISelHook = 1; 682 let hasSideEffects = 1; 683 let DisableWQM = 1; 684 let has_glc = 0; 685 let has_dlc = 0; 686 let has_tfe = 0; 687 let has_sccb = 1; 688 let maybeAtomic = 1; 689 let AsmMatchConverter = "cvtMubufAtomic"; 690} 691 692class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 693 RegisterClass vdataClass, 694 list<dag> pattern=[], 695 // Workaround bug bz30254 696 int addrKindCopy = addrKind, 697 RegisterClass vdataClassCopy = vdataClass> 698 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 699 (outs), 700 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 701 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 702 pattern>, 703 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 704 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 705 let glc_value = 0; 706 let dlc_value = 0; 707 let sccb_value = 0; 708 let IsAtomicNoRet = 1; 709} 710 711class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 712 RegisterClass vdataClass, 713 list<dag> pattern=[], 714 // Workaround bug bz30254 715 int addrKindCopy = addrKind, 716 RegisterClass vdataClassCopy = vdataClass, 717 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 718 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 719 (outs vdata_op:$vdata), 720 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 721 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 722 pattern>, 723 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 724 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 725 let glc_value = 1; 726 let dlc_value = 0; 727 let sccb_value = 0; 728 let IsAtomicRet = 1; 729 let Constraints = "$vdata = $vdata_in"; 730 let DisableEncoding = "$vdata_in"; 731} 732 733multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 734 RegisterClass vdataClass, 735 ValueType vdataType, 736 bit isFP = isFloatType<vdataType>.ret> { 737 let FPAtomic = isFP in 738 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 739 MUBUFAddr64Table <0, NAME>; 740 741 let FPAtomic = isFP in 742 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 743 MUBUFAddr64Table <1, NAME>; 744 745 let FPAtomic = isFP in 746 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 747 748 let FPAtomic = isFP in 749 750 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 751 752 let FPAtomic = isFP in 753 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 754} 755 756multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 757 RegisterClass vdataClass, 758 ValueType vdataType, 759 SDPatternOperator atomic, 760 bit isFP = isFloatType<vdataType>.ret> { 761 let FPAtomic = isFP in 762 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 763 [(set vdataType:$vdata, 764 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 765 vdataType:$vdata_in))]>, 766 MUBUFAddr64Table <0, NAME # "_RTN">; 767 768 let FPAtomic = isFP in 769 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 770 [(set vdataType:$vdata, 771 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 772 vdataType:$vdata_in))]>, 773 MUBUFAddr64Table <1, NAME # "_RTN">; 774 775 let FPAtomic = isFP in 776 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 777 778 let FPAtomic = isFP in 779 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 780 781 let FPAtomic = isFP in 782 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 783} 784 785multiclass MUBUF_Pseudo_Atomics <string opName, 786 RegisterClass vdataClass, 787 ValueType vdataType, 788 SDPatternOperator atomic = null_frag> : 789 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 790 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 791 792 793//===----------------------------------------------------------------------===// 794// MUBUF Instructions 795//===----------------------------------------------------------------------===// 796 797defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 798 "buffer_load_format_x", f32 799>; 800defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 801 "buffer_load_format_xy", v2f32 802>; 803defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 804 "buffer_load_format_xyz", v3f32 805>; 806defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 807 "buffer_load_format_xyzw", v4f32 808>; 809defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 810 "buffer_store_format_x", f32 811>; 812defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 813 "buffer_store_format_xy", v2f32 814>; 815defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 816 "buffer_store_format_xyz", v3f32 817>; 818defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 819 "buffer_store_format_xyzw", v4f32 820>; 821 822let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 823 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 824 "buffer_load_format_d16_x", i32 825 >; 826 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 827 "buffer_load_format_d16_xy", v2i32 828 >; 829 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 830 "buffer_load_format_d16_xyz", v3i32 831 >; 832 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 833 "buffer_load_format_d16_xyzw", v4i32 834 >; 835 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 836 "buffer_store_format_d16_x", i32 837 >; 838 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 839 "buffer_store_format_d16_xy", v2i32 840 >; 841 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 842 "buffer_store_format_d16_xyz", v3i32 843 >; 844 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_xyzw", v4i32 846 >; 847} // End HasUnpackedD16VMem. 848 849let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 850 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 851 "buffer_load_format_d16_x", f16 852 >; 853 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 854 "buffer_load_format_d16_xy", v2f16 855 >; 856 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 857 "buffer_load_format_d16_xyz", v3f16 858 >; 859 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 860 "buffer_load_format_d16_xyzw", v4f16 861 >; 862 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 863 "buffer_store_format_d16_x", f16 864 >; 865 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 866 "buffer_store_format_d16_xy", v2f16 867 >; 868 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 869 "buffer_store_format_d16_xyz", v3f16 870 >; 871 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 872 "buffer_store_format_d16_xyzw", v4f16 873 >; 874} // End HasPackedD16VMem. 875 876defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 877 "buffer_load_ubyte", i32 878>; 879defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 880 "buffer_load_sbyte", i32 881>; 882defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 883 "buffer_load_ushort", i32 884>; 885defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 886 "buffer_load_sshort", i32 887>; 888defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 889 "buffer_load_dword", i32 890>; 891defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 892 "buffer_load_dwordx2", v2i32 893>; 894defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 895 "buffer_load_dwordx3", v3i32 896>; 897defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 898 "buffer_load_dwordx4", v4i32 899>; 900 901defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 902defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 903defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 904defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 905defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 906defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 907defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 908defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 909defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 910defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 911 912// This is not described in AMD documentation, 913// but 'lds' versions of these opcodes are available 914// in at least GFX8+ chips. See Bug 37653. 915let SubtargetPredicate = isGFX8GFX9 in { 916defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 917 "buffer_load_dwordx2", v2i32, 0, 1 918>; 919defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 920 "buffer_load_dwordx3", v3i32, 0, 1 921>; 922defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 923 "buffer_load_dwordx4", v4i32, 0, 1 924>; 925} 926 927defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 928 "buffer_store_byte", i32, truncstorei8_global 929>; 930defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 931 "buffer_store_short", i32, truncstorei16_global 932>; 933defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 934 "buffer_store_dword", i32, store_global 935>; 936defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 937 "buffer_store_dwordx2", v2i32, store_global 938>; 939defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 940 "buffer_store_dwordx3", v3i32, store_global 941>; 942defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 943 "buffer_store_dwordx4", v4i32, store_global 944>; 945defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 946 "buffer_atomic_swap", VGPR_32, i32 947>; 948defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 949 "buffer_atomic_cmpswap", VReg_64, v2i32 950>; 951defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 952 "buffer_atomic_add", VGPR_32, i32 953>; 954defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 955 "buffer_atomic_sub", VGPR_32, i32 956>; 957defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 958 "buffer_atomic_smin", VGPR_32, i32 959>; 960defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 961 "buffer_atomic_umin", VGPR_32, i32 962>; 963defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 964 "buffer_atomic_smax", VGPR_32, i32 965>; 966defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 967 "buffer_atomic_umax", VGPR_32, i32 968>; 969defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 970 "buffer_atomic_and", VGPR_32, i32 971>; 972defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 973 "buffer_atomic_or", VGPR_32, i32 974>; 975defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 976 "buffer_atomic_xor", VGPR_32, i32 977>; 978defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 979 "buffer_atomic_inc", VGPR_32, i32 980>; 981defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 982 "buffer_atomic_dec", VGPR_32, i32 983>; 984defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 985 "buffer_atomic_swap_x2", VReg_64, i64 986>; 987defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 988 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 989>; 990defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 991 "buffer_atomic_add_x2", VReg_64, i64 992>; 993defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 994 "buffer_atomic_sub_x2", VReg_64, i64 995>; 996defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 997 "buffer_atomic_smin_x2", VReg_64, i64 998>; 999defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1000 "buffer_atomic_umin_x2", VReg_64, i64 1001>; 1002defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1003 "buffer_atomic_smax_x2", VReg_64, i64 1004>; 1005defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1006 "buffer_atomic_umax_x2", VReg_64, i64 1007>; 1008defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1009 "buffer_atomic_and_x2", VReg_64, i64 1010>; 1011defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1012 "buffer_atomic_or_x2", VReg_64, i64 1013>; 1014defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1015 "buffer_atomic_xor_x2", VReg_64, i64 1016>; 1017defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1018 "buffer_atomic_inc_x2", VReg_64, i64 1019>; 1020defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1021 "buffer_atomic_dec_x2", VReg_64, i64 1022>; 1023 1024let SubtargetPredicate = HasGFX10_BEncoding in 1025defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1026 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1027>; 1028 1029let SubtargetPredicate = isGFX8GFX9 in { 1030def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1031} 1032 1033let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1034/* 1035defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1036defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1037*/ 1038 1039def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1040 int_amdgcn_buffer_wbinvl1_sc>; 1041} 1042 1043let SubtargetPredicate = isGFX6GFX7GFX10 in { 1044 1045defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1046 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1047>; 1048defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1049 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1050>; 1051defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1052 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1053>; 1054defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1055 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1056>; 1057defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1058 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1059>; 1060defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1061 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1062>; 1063 1064} 1065 1066let SubtargetPredicate = HasD16LoadStore in { 1067 1068defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1069 "buffer_load_ubyte_d16", i32, 1 1070>; 1071 1072defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1073 "buffer_load_ubyte_d16_hi", i32, 1 1074>; 1075 1076defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1077 "buffer_load_sbyte_d16", i32, 1 1078>; 1079 1080defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1081 "buffer_load_sbyte_d16_hi", i32, 1 1082>; 1083 1084defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1085 "buffer_load_short_d16", i32, 1 1086>; 1087 1088defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1089 "buffer_load_short_d16_hi", i32, 1 1090>; 1091 1092defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1093 "buffer_store_byte_d16_hi", i32 1094>; 1095 1096defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1097 "buffer_store_short_d16_hi", i32 1098>; 1099 1100defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1101 "buffer_load_format_d16_hi_x", i32 1102>; 1103defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1104 "buffer_store_format_d16_hi_x", i32 1105>; 1106 1107} // End HasD16LoadStore 1108 1109def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1110 int_amdgcn_buffer_wbinvl1>; 1111 1112let SubtargetPredicate = HasAtomicFaddInsts in { 1113defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1114 "buffer_atomic_add_f32", VGPR_32, f32 1115>; 1116defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1117 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1118>; 1119 1120let OtherPredicates = [isGFX90APlus] in { 1121defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN < 1122 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1123>; 1124defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1125 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1126>; 1127} 1128} // End SubtargetPredicate = HasAtomicFaddInsts 1129 1130//===----------------------------------------------------------------------===// 1131// MTBUF Instructions 1132//===----------------------------------------------------------------------===// 1133 1134defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1135defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1136defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1137defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1138defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1139defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1140defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1141defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1142 1143let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1144 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1145 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1146 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1147 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1148 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1149 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1150 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1151 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1152} // End HasUnpackedD16VMem. 1153 1154let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1155 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1156 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1157 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1158 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1159 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1160 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1161 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1162 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1163} // End HasPackedD16VMem. 1164 1165let SubtargetPredicate = isGFX7Plus in { 1166 1167//===----------------------------------------------------------------------===// 1168// Instruction definitions for CI and newer. 1169//===----------------------------------------------------------------------===// 1170 1171def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1172 int_amdgcn_buffer_wbinvl1_vol>; 1173 1174} // End let SubtargetPredicate = isGFX7Plus 1175 1176let SubtargetPredicate = isGFX90APlus in { 1177 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1178 let has_glc = 1; 1179 let has_sccb = 1; 1180 let InOperandList = (ins CPol_0:$cpol); 1181 let AsmOperands = "$cpol"; 1182 } 1183 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1184 let SubtargetPredicate = isGFX90AOnly; 1185 } 1186 1187 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 1188 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 1189 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 1190} // End SubtargetPredicate = isGFX90APlus 1191 1192def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1193 let SubtargetPredicate = isGFX940Plus; 1194 let has_glc = 1; 1195 let has_sccb = 1; 1196 let InOperandList = (ins CPol_0:$cpol); 1197 let AsmOperands = "$cpol"; 1198} 1199 1200let SubtargetPredicate = isGFX10Plus in { 1201 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1202 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1203} // End SubtargetPredicate = isGFX10Plus 1204 1205//===----------------------------------------------------------------------===// 1206// MUBUF Patterns 1207//===----------------------------------------------------------------------===// 1208 1209//===----------------------------------------------------------------------===// 1210// buffer_load/store_format patterns 1211//===----------------------------------------------------------------------===// 1212 1213multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1214 string opcode, ValueType memoryVt = vt> { 1215 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1216 1217 def : GCNPat< 1218 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1219 timm:$auxiliary, 0)), 1220 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1221 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1222 >; 1223 1224 def : GCNPat< 1225 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1226 timm:$auxiliary, 0)), 1227 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1228 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1229 >; 1230 1231 def : GCNPat< 1232 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1233 timm:$auxiliary, timm)), 1234 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1235 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1236 >; 1237 1238 def : GCNPat< 1239 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1240 timm:$auxiliary, timm)), 1241 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1242 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1243 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1244 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1245 >; 1246} 1247 1248defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1249defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1250defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1251defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1252defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1253defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1254defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1255defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1256 1257let SubtargetPredicate = HasUnpackedD16VMem in { 1258 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1259 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1260 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1261 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1262 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1263 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1264} // End HasUnpackedD16VMem. 1265 1266let SubtargetPredicate = HasPackedD16VMem in { 1267 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1268 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1269 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1270 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1271 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1272 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1273 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1274 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1275 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1276} // End HasPackedD16VMem. 1277 1278defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1279defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1280defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1281defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1282defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1283defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1288defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1289defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1291defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1292defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1293defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1294 1295multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1296 string opcode, ValueType memoryVt = vt> { 1297 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1298 1299 def : GCNPat< 1300 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1301 timm:$auxiliary, 0), 1302 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1303 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1304 >; 1305 1306 def : GCNPat< 1307 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1308 timm:$auxiliary, 0), 1309 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1310 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1311 >; 1312 1313 def : GCNPat< 1314 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1315 timm:$auxiliary, timm), 1316 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1317 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1318 >; 1319 1320 def : GCNPat< 1321 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1322 timm:$auxiliary, timm), 1323 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1324 getVregSrcForVT<vt>.ret:$vdata, 1325 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1326 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1327 0, (extract_swz $auxiliary)) 1328 >; 1329} 1330 1331defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1332defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1333defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1334defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1335defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1336defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1337defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1338defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1339 1340let SubtargetPredicate = HasUnpackedD16VMem in { 1341 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1342 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1343 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1344 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1345 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1346 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1347} // End HasUnpackedD16VMem. 1348 1349let SubtargetPredicate = HasPackedD16VMem in { 1350 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1351 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1352 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1353 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1354 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1355 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1356 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1357 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1358 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1359} // End HasPackedD16VMem. 1360 1361defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1362defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1363defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1364defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1365defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1366defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1371defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1372defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1373defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1374defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1375 1376//===----------------------------------------------------------------------===// 1377// buffer_atomic patterns 1378//===----------------------------------------------------------------------===// 1379 1380multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst> { 1381 foreach RtnMode = ["ret", "noret"] in { 1382 1383 defvar Op = !cast<SDPatternOperator>(OpPrefix # "_" # RtnMode # "_" # vt.Size); 1384 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1385 1386 def : Pat< 1387 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1388 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1389 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1390 >; 1391 1392 def : Pat< 1393 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1394 vt:$vdata_in)), 1395 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1396 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1397 >; 1398 1399 } // end foreach RtnMode 1400} 1401 1402multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1403 foreach RtnMode = ["ret", "noret"] in { 1404 1405 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode 1406 # "_" # vt.Size); 1407 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1408 1409 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1410 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1411 offset:$offset); 1412 def : Pat< 1413 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1414 !if(!eq(RtnMode, "ret"), 1415 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1416 !if(!eq(vt, i32), sub0, sub0_sub1)), 1417 OffsetResDag) 1418 >; 1419 1420 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1421 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1422 SCSrc_b32:$soffset, offset:$offset); 1423 def : Pat< 1424 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1425 data_vt:$vdata_in)), 1426 !if(!eq(RtnMode, "ret"), 1427 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1428 !if(!eq(vt, i32), sub0, sub0_sub1)), 1429 Addr64ResDag) 1430 >; 1431 1432 } // end foreach RtnMode 1433} 1434 1435foreach Ty = [i32, i64] in { 1436 1437defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1438 1439defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1440defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1441defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1442defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1443defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1444defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1445defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1446defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1447defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1448defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1449defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1450defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1451 1452} // end foreach Ty 1453 1454defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1455defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1456 1457multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1458 list<string> RtnModes = ["ret", "noret"]> { 1459 foreach RtnMode = RtnModes in { 1460 1461 defvar Op = !cast<SDPatternOperator>(!if(!eq(RtnMode, "none"), 1462 OpPrefix, OpPrefix # "_" # RtnMode)); 1463 defvar InstSuffix = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1464 "_RTN", ""); 1465 defvar CachePolicy = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1466 (set_glc $cachepolicy), (timm:$cachepolicy)); 1467 1468 def : GCNPat< 1469 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1470 timm:$offset, timm:$cachepolicy, 0)), 1471 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1472 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1473 (as_i16timm $offset), CachePolicy) 1474 >; 1475 1476 def : GCNPat< 1477 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1478 timm:$offset, timm:$cachepolicy, timm)), 1479 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1480 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1481 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1482 >; 1483 1484 def : GCNPat< 1485 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1486 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1487 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1488 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1489 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1490 >; 1491 1492 def : GCNPat< 1493 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1494 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1495 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1496 getVregSrcForVT<vt>.ret:$vdata_in, 1497 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1498 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1499 >; 1500 1501 } // end foreach RtnMode 1502} 1503 1504defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1505defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1506defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1507defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1508defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1509defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1510defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1511defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1512defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1513defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1514defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1515defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1516defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1517defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["none"]>; 1518defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1519defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1520defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1521defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1522defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1523defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1524defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1525defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1526defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1527defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1528defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1529defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1530 1531let SubtargetPredicate = isGFX6GFX7GFX10 in { 1532 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1533 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1534 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1535 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1536} 1537 1538class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1539 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1540 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1541 [{ return SDValue(N, 0).use_empty(); }]> { 1542 1543 let GISelPredicateCode = [{ 1544 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1545 }]; 1546} 1547 1548multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1549 string opcode> { 1550 def : GCNPat< 1551 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1552 0, i32:$soffset, timm:$offset, 1553 timm:$cachepolicy, 0), 1554 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1555 (as_i16timm $offset), $cachepolicy) 1556 >; 1557 1558 def : GCNPat< 1559 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1560 0, i32:$soffset, timm:$offset, 1561 timm:$cachepolicy, timm), 1562 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1563 (as_i16timm $offset), $cachepolicy) 1564 >; 1565 1566 def : GCNPat< 1567 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1568 i32:$voffset, i32:$soffset, timm:$offset, 1569 timm:$cachepolicy, 0), 1570 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1571 (as_i16timm $offset), $cachepolicy) 1572 >; 1573 1574 def : GCNPat< 1575 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1576 i32:$voffset, i32:$soffset, timm:$offset, 1577 timm:$cachepolicy, timm), 1578 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1579 getVregSrcForVT<vt>.ret:$vdata_in, 1580 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1581 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1582 >; 1583} 1584 1585let SubtargetPredicate = HasAtomicFaddInsts in { 1586defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1587defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1588} 1589 1590let SubtargetPredicate = isGFX90APlus in { 1591 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1592 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1593 1594 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1595 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1596 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1597} // End SubtargetPredicate = isGFX90APlus 1598 1599foreach RtnMode = ["ret", "noret"] in { 1600 1601defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap # "_" # RtnMode); 1602defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1603defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1604 (timm:$cachepolicy)); 1605 1606defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1607 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1608 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1609def : GCNPat< 1610 (Op 1611 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1612 timm:$offset, timm:$cachepolicy, 0), 1613 !if(!eq(RtnMode, "ret"), 1614 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1615 OffsetResDag) 1616>; 1617 1618defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1619 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1620 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1621 CachePolicy); 1622def : GCNPat< 1623 (Op 1624 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1625 0, i32:$soffset, timm:$offset, 1626 timm:$cachepolicy, timm), 1627 !if(!eq(RtnMode, "ret"), 1628 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1629 IdxenResDag) 1630>; 1631 1632defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1633 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1634 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1635 CachePolicy); 1636def : GCNPat< 1637 (Op 1638 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1639 i32:$voffset, i32:$soffset, timm:$offset, 1640 timm:$cachepolicy, 0), 1641 !if(!eq(RtnMode, "ret"), 1642 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1643 OffenResDag) 1644>; 1645 1646defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1647 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1648 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1649 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1650def : GCNPat< 1651 (Op 1652 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1653 i32:$voffset, i32:$soffset, timm:$offset, 1654 timm:$cachepolicy, timm), 1655 !if(!eq(RtnMode, "ret"), 1656 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1657 BothenResDag) 1658>; 1659 1660} // end foreach RtnMode 1661 1662class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1663 PatFrag constant_ld> : GCNPat < 1664 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1665 i16:$offset))), 1666 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1667 >; 1668 1669multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1670 ValueType vt, PatFrag atomic_ld> { 1671 def : GCNPat < 1672 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1673 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1674 >; 1675 1676 def : GCNPat < 1677 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1678 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1679 >; 1680} 1681 1682let SubtargetPredicate = isGFX6GFX7 in { 1683def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1684def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1685def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1686def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1687def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1688def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1689 1690defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1691defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1692} // End SubtargetPredicate = isGFX6GFX7 1693 1694multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1695 PatFrag ld> { 1696 1697 def : GCNPat < 1698 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1699 (Instr_OFFSET $srsrc, $soffset, $offset) 1700 >; 1701} 1702 1703let OtherPredicates = [Has16BitInsts] in { 1704 1705defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1706defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1707defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1708defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1709defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1710defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1711 1712defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1713 1714} // End OtherPredicates = [Has16BitInsts] 1715 1716multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1717 MUBUF_Pseudo InstrOffset, 1718 ValueType vt, PatFrag ld> { 1719 def : GCNPat < 1720 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1721 i32:$soffset, u16imm:$offset))), 1722 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1723 >; 1724 1725 def : GCNPat < 1726 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1727 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1728 >; 1729} 1730 1731// XXX - Is it possible to have a complex pattern in a PatFrag? 1732multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1733 MUBUF_Pseudo InstrOffset, 1734 ValueType vt, PatFrag ld_frag> { 1735 def : GCNPat < 1736 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1737 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1738 >; 1739 1740 def : GCNPat < 1741 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1742 (InstrOffset $srsrc, $soffset, $offset, $in) 1743 >; 1744} 1745 1746let OtherPredicates = [DisableFlatScratch] in { 1747defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1748defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1749defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1750defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1751defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1752defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1753defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1754defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1755defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1756defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1757 1758foreach vt = Reg32Types.types in { 1759defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1760} 1761defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1762defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1763defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1764 1765let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1766defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1767defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1768defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1769defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1770defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1771defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1772 1773defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1774defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1775defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1776defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1777defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1778defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1779} 1780 1781} // End OtherPredicates = [DisableFlatScratch] 1782 1783multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1784 ValueType vt, PatFrag atomic_st> { 1785 // Store follows atomic op convention so address is first 1786 def : GCNPat < 1787 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1788 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1789 >; 1790 1791 def : GCNPat < 1792 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1793 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1794 >; 1795} 1796let SubtargetPredicate = isGFX6GFX7 in { 1797defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1798defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1799} // End Predicates = isGFX6GFX7 1800 1801 1802multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1803 PatFrag st> { 1804 1805 def : GCNPat < 1806 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1807 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1808 >; 1809} 1810 1811defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1812defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1813 1814multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1815 MUBUF_Pseudo InstrOffset, 1816 ValueType vt, PatFrag st, 1817 RegisterClass rc = VGPR_32> { 1818 def : GCNPat < 1819 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1820 i32:$soffset, u16imm:$offset)), 1821 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1822 >; 1823 1824 def : GCNPat < 1825 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1826 u16imm:$offset)), 1827 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1828 >; 1829} 1830 1831let OtherPredicates = [DisableFlatScratch] in { 1832defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1833defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1834defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1835defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1836 1837foreach vt = Reg32Types.types in { 1838defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1839} 1840 1841defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1842defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1843defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1844 1845 1846let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1847 // Hiding the extract high pattern in the PatFrag seems to not 1848 // automatically increase the complexity. 1849let AddedComplexity = 1 in { 1850defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1851defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1852} 1853} 1854} // End OtherPredicates = [DisableFlatScratch] 1855 1856//===----------------------------------------------------------------------===// 1857// MTBUF Patterns 1858//===----------------------------------------------------------------------===// 1859 1860//===----------------------------------------------------------------------===// 1861// tbuffer_load/store_format patterns 1862//===----------------------------------------------------------------------===// 1863 1864multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1865 string opcode, ValueType memoryVt = vt> { 1866 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1867 1868 def : GCNPat< 1869 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1870 timm:$format, timm:$auxiliary, 0)), 1871 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1872 (as_i8timm $format), 1873 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1874 >; 1875 1876 def : GCNPat< 1877 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1878 timm:$format, timm:$auxiliary, timm)), 1879 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1880 (as_i8timm $format), 1881 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1882 >; 1883 1884 def : GCNPat< 1885 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1886 timm:$format, timm:$auxiliary, 0)), 1887 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1888 (as_i8timm $format), 1889 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1890 >; 1891 1892 def : GCNPat< 1893 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1894 timm:$format, timm:$auxiliary, timm)), 1895 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1896 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1897 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1898 (as_i8timm $format), 1899 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1900 >; 1901} 1902 1903defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1904defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1905defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1906defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1907defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1908defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1909defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1910defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1911 1912let SubtargetPredicate = HasUnpackedD16VMem in { 1913 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1914 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1915 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1916 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1917 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1918} // End HasUnpackedD16VMem. 1919 1920let SubtargetPredicate = HasPackedD16VMem in { 1921 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1922 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1923 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1924 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1925 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1926} // End HasPackedD16VMem. 1927 1928multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1929 string opcode, ValueType memoryVt = vt> { 1930 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1931 1932 def : GCNPat< 1933 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1934 timm:$format, timm:$auxiliary, 0), 1935 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1936 (as_i16timm $offset), (as_i8timm $format), 1937 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1938 >; 1939 1940 def : GCNPat< 1941 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1942 timm:$format, timm:$auxiliary, timm), 1943 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1944 (as_i16timm $offset), (as_i8timm $format), 1945 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1946 >; 1947 1948 def : GCNPat< 1949 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1950 timm:$format, timm:$auxiliary, 0), 1951 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1952 (as_i16timm $offset), (as_i8timm $format), 1953 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1954 >; 1955 1956 def : GCNPat< 1957 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1958 timm:$offset, timm:$format, timm:$auxiliary, timm), 1959 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1960 getVregSrcForVT<vt>.ret:$vdata, 1961 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1962 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1963 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1964 >; 1965} 1966 1967defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1968defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1969defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1970defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1971defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1972defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1973defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1974defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1975 1976let SubtargetPredicate = HasUnpackedD16VMem in { 1977 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1978 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1979 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1980 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1981 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1982} // End HasUnpackedD16VMem. 1983 1984let SubtargetPredicate = HasPackedD16VMem in { 1985 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1986 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1987 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1988 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1989 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1990} // End HasPackedD16VMem. 1991 1992//===----------------------------------------------------------------------===// 1993// Target-specific instruction encodings. 1994//===----------------------------------------------------------------------===// 1995 1996//===----------------------------------------------------------------------===// 1997// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1998//===----------------------------------------------------------------------===// 1999 2000class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2001 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2002 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2003 let Inst{12} = ps.offen; 2004 let Inst{13} = ps.idxen; 2005 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2006 let Inst{16} = ps.lds; 2007 let Inst{24-18} = op; 2008 let Inst{31-26} = 0x38; 2009 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2010 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2011 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2012 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2013 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2014 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2015} 2016 2017class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2018 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2019 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2020 let Inst{25} = op{7}; 2021} 2022 2023class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2024 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2025 let Inst{15} = ps.addr64; 2026} 2027 2028//===----------------------------------------------------------------------===// 2029// MUBUF - GFX10. 2030//===----------------------------------------------------------------------===// 2031 2032let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2033 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2034 def _BOTHEN_gfx10 : 2035 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2036 def _IDXEN_gfx10 : 2037 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2038 def _OFFEN_gfx10 : 2039 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2040 def _OFFSET_gfx10 : 2041 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2042 } 2043 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2044 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2045 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 2046 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2047 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 2048 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2049 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 2050 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2051 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 2052 2053 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2054 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 2055 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2056 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 2057 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2058 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 2059 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2060 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 2061 } 2062 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2063 def _BOTHEN_RTN_gfx10 : 2064 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2065 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2066 def _IDXEN_RTN_gfx10 : 2067 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2068 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2069 def _OFFEN_RTN_gfx10 : 2070 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2071 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2072 def _OFFSET_RTN_gfx10 : 2073 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2074 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2075 } 2076 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2077 MUBUF_Real_Atomics_RTN_gfx10<op> { 2078 def _BOTHEN_gfx10 : 2079 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2080 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2081 def _IDXEN_gfx10 : 2082 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2083 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2084 def _OFFEN_gfx10 : 2085 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2086 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2087 def _OFFSET_gfx10 : 2088 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2089 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2090 } 2091} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2092 2093defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2094defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2095defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2096defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2097defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2098defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2099defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2100defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2101// FIXME-GFX10: Add following instructions: 2102//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2103//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2104defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2105defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2106defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2107defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2108defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2109defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2110defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2111defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2112 2113def BUFFER_GL0_INV_gfx10 : 2114 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2115def BUFFER_GL1_INV_gfx10 : 2116 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2117 2118//===----------------------------------------------------------------------===// 2119// MUBUF - GFX6, GFX7, GFX10. 2120//===----------------------------------------------------------------------===// 2121 2122let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2123 multiclass MUBUF_Real_gfx6<bits<8> op> { 2124 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2125 } 2126} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2127 2128let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2129 multiclass MUBUF_Real_gfx7<bits<8> op> { 2130 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2131 } 2132} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2133 2134let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2135 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2136 def _ADDR64_gfx6_gfx7 : 2137 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2138 def _BOTHEN_gfx6_gfx7 : 2139 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2140 def _IDXEN_gfx6_gfx7 : 2141 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2142 def _OFFEN_gfx6_gfx7 : 2143 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2144 def _OFFSET_gfx6_gfx7 : 2145 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2146 } 2147 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2148 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2149 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2150 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2151 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2152 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2153 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2154 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2155 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2156 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2157 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2158 2159 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2160 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2161 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2162 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2163 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2164 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2165 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2166 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2167 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2168 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2169 } 2170 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2171 def _ADDR64_gfx6_gfx7 : 2172 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2173 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2174 def _BOTHEN_gfx6_gfx7 : 2175 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2176 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2177 def _IDXEN_gfx6_gfx7 : 2178 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2179 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2180 def _OFFEN_gfx6_gfx7 : 2181 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2182 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2183 def _OFFSET_gfx6_gfx7 : 2184 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2185 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2186 2187 def _ADDR64_RTN_gfx6_gfx7 : 2188 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2189 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2190 def _BOTHEN_RTN_gfx6_gfx7 : 2191 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2192 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2193 def _IDXEN_RTN_gfx6_gfx7 : 2194 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2195 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2196 def _OFFEN_RTN_gfx6_gfx7 : 2197 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2198 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2199 def _OFFSET_RTN_gfx6_gfx7 : 2200 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2201 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2202 } 2203} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2204 2205multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2206 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2207 2208multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2209 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2210 2211multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2212 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2213 2214// FIXME-GFX6: Following instructions are available only on GFX6. 2215//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2216//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2217 2218defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2219defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2220defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2221defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2222defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2223defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2224defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2225defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2226defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2227defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2228defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2229defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2230defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2231defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2232defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2233defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2234defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2235defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2236defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2237defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2238defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2239defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2240 2241defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2242defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2243defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2244defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2245defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2246defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2247defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2248defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2249defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2250defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2251defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2252defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2253defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2254defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2255defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2256defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2257defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2258defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2259defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2260defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2261defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2262defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2263defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2264defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2265defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2266defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2267defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2268defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2269defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2270// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2271defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2272defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2273defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2274 2275defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2276 2277defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2278defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2279def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2280 2281//===----------------------------------------------------------------------===// 2282// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2283//===----------------------------------------------------------------------===// 2284 2285class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2286 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2287 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2288 let Inst{12} = ps.offen; 2289 let Inst{13} = ps.idxen; 2290 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2291 let Inst{18-16} = op; 2292 let Inst{31-26} = 0x3a; //encoding 2293 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2294 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2295 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2296 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2297 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2298 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2299} 2300 2301//===----------------------------------------------------------------------===// 2302// MTBUF - GFX10. 2303//===----------------------------------------------------------------------===// 2304 2305class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2306 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2307 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2308 let Inst{25-19} = format; 2309 let Inst{53} = op{3}; 2310} 2311 2312let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2313 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2314 def _BOTHEN_gfx10 : 2315 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2316 def _IDXEN_gfx10 : 2317 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2318 def _OFFEN_gfx10 : 2319 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2320 def _OFFSET_gfx10 : 2321 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2322 } 2323} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2324 2325defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2326defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2327defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2328defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2329defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2330defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2331defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2332defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2333 2334//===----------------------------------------------------------------------===// 2335// MTBUF - GFX6, GFX7, GFX10. 2336//===----------------------------------------------------------------------===// 2337 2338class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2339 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2340 let Inst{15} = ps.addr64; 2341 let Inst{22-19} = dfmt; 2342 let Inst{25-23} = nfmt; 2343} 2344 2345let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2346 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2347 def _ADDR64_gfx6_gfx7 : 2348 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2349 def _BOTHEN_gfx6_gfx7 : 2350 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2351 def _IDXEN_gfx6_gfx7 : 2352 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2353 def _OFFEN_gfx6_gfx7 : 2354 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2355 def _OFFSET_gfx6_gfx7 : 2356 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2357 } 2358} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2359 2360multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2361 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2362 2363defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2364defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2365defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2366defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2367defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2368defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2369defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2370defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2371 2372//===----------------------------------------------------------------------===// 2373// GFX8, GFX9 (VI). 2374//===----------------------------------------------------------------------===// 2375 2376class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2377 bit has_sccb = ps.has_sccb> : 2378 MUBUF_Real<ps>, 2379 Enc64, 2380 SIMCInstr<ps.PseudoInstr, Enc>, 2381 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2382 !if(ps.IsAtomicRet, 1, ?))> { 2383 2384 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2385 let Inst{12} = ps.offen; 2386 let Inst{13} = ps.idxen; 2387 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2388 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2389 let Inst{16} = ps.lds; 2390 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2391 let Inst{24-18} = op; 2392 let Inst{31-26} = 0x38; //encoding 2393 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2394 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2395 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2396 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2397} 2398 2399class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2400 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2401 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2402 let DecoderNamespace = "GFX8"; 2403 2404 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2405} 2406 2407class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2408 bit has_sccb = ps.has_sccb> : 2409 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2410 let AssemblerPredicate = isGFX90APlus; 2411 let DecoderNamespace = "GFX90A"; 2412 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2413 !subst("$tfe", "", ps.AsmOperands)); 2414 2415 let Inst{55} = acc; 2416} 2417 2418class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2419 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2420 let AssemblerPredicate = isGFX940Plus; 2421 let DecoderNamespace = "GFX9"; 2422 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2423 2424 let Inst{55} = acc; 2425} 2426 2427multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2428 def _vi : MUBUF_Real_vi<op, ps>; 2429 2430 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2431 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2432 2433 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2434 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2435 let SubtargetPredicate = isGFX90AOnly; 2436 let AssemblerPredicate = isGFX90AOnly; 2437 } 2438 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2439 } 2440} 2441 2442multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2443 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2444 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2445 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2446 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2447} 2448 2449multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2450 2451 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2452 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2453 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2454 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2455 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2456 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2457 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2458 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2459 2460 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2461 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2462 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2463 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2464 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2465 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2466 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2467 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2468 2469 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2470 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2471 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2472 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2473 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2474 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2475 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2476 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2477 2478 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2479 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2480 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2481 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2482 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2483 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2484 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2485 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2486} 2487 2488class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2489 MUBUF_Real<ps>, 2490 Enc64, 2491 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2492 let AssemblerPredicate=HasUnpackedD16VMem; 2493 let DecoderNamespace="GFX80_UNPACKED"; 2494 2495 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2496 let Inst{12} = ps.offen; 2497 let Inst{13} = ps.idxen; 2498 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2499 let Inst{16} = ps.lds; 2500 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2501 let Inst{24-18} = op; 2502 let Inst{31-26} = 0x38; //encoding 2503 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2504 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2505 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2506 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2507 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2508} 2509 2510multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2511 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2512 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2513 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2514 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2515} 2516 2517multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2518 MUBUF_Real_AllAddr_vi<op> { 2519 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2520 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2521 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2522 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2523} 2524 2525defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2526defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2527defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2528defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2529defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2530defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2531defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2532defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2533let SubtargetPredicate = HasUnpackedD16VMem in { 2534 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2535 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2536 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2537 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2538 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2539 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2540 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2541 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2542} // End HasUnpackedD16VMem. 2543let SubtargetPredicate = HasPackedD16VMem in { 2544 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2545 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2546 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2547 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2548 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2549 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2550 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2551 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2552} // End HasPackedD16VMem. 2553defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2554defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2555defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2556defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2557defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2558defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2559defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2560defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2561defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2562defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2563defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2564defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2565defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2566defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2567defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2568defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2569 2570defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2571defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2572defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2573defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2574defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2575defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2576 2577defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2578defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2579 2580defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2581defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2582defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2583defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2584defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2585defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2586defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2587defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2588defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2589defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2590defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2591defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2592defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2593 2594defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2595defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2596defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2597defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2598defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2599defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2600defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2601defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2602defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2603defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2604defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2605defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2606defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2607 2608defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2609 2610let AssemblerPredicate = isGFX8GFX9 in { 2611def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2612def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2613} // End AssemblerPredicate = isGFX8GFX9 2614 2615let SubtargetPredicate = HasAtomicFaddInsts in { 2616 2617defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2618defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2619 2620} // End SubtargetPredicate = HasAtomicFaddInsts 2621 2622let SubtargetPredicate = isGFX90APlus in { 2623 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2624 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2625 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2626} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2627 2628def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2629 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2630 let AssemblerPredicate = isGFX90AOnly; 2631 let SubtargetPredicate = isGFX90AOnly; 2632} 2633def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2634 2635let SubtargetPredicate = isGFX940Plus in { 2636def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2637def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2638} 2639 2640class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2641 MTBUF_Real<ps>, 2642 Enc64, 2643 SIMCInstr<ps.PseudoInstr, Enc> { 2644 2645 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2646 let Inst{12} = ps.offen; 2647 let Inst{13} = ps.idxen; 2648 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2649 let Inst{18-15} = op; 2650 let Inst{22-19} = dfmt; 2651 let Inst{25-23} = nfmt; 2652 let Inst{31-26} = 0x3a; //encoding 2653 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2654 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2655 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2656 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2657 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2658 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2659 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2660} 2661 2662class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2663 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2664 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2665 let DecoderNamespace = "GFX8"; 2666 2667 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2668} 2669 2670class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2671 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2672 let AssemblerPredicate = isGFX90APlus; 2673 let DecoderNamespace = "GFX90A"; 2674 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2675 2676 let Inst{55} = acc; 2677} 2678 2679multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2680 def _vi : MTBUF_Real_vi<op, ps>; 2681 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2682} 2683 2684multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2685 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2686 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2687 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2688 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2689} 2690 2691class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2692 MTBUF_Real<ps>, 2693 Enc64, 2694 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2695 let AssemblerPredicate=HasUnpackedD16VMem; 2696 let DecoderNamespace="GFX80_UNPACKED"; 2697 2698 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2699 let Inst{12} = ps.offen; 2700 let Inst{13} = ps.idxen; 2701 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2702 let Inst{18-15} = op; 2703 let Inst{22-19} = dfmt; 2704 let Inst{25-23} = nfmt; 2705 let Inst{31-26} = 0x3a; //encoding 2706 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2707 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2708 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2709 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2710 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2711 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2712} 2713 2714multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2715 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2716 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2717 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2718 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2719} 2720 2721defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2722defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2723defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2724defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2725defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2726defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2727defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2728defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2729let SubtargetPredicate = HasUnpackedD16VMem in { 2730 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2731 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2732 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2733 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2734 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2735 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2736 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2737 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2738} // End HasUnpackedD16VMem. 2739let SubtargetPredicate = HasPackedD16VMem in { 2740 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2741 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2742 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2743 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2744 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2745 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2746 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2747 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2748} // End HasUnpackedD16VMem. 2749 2750def MUBUFInfoTable : GenericTable { 2751 let FilterClass = "MUBUF_Pseudo"; 2752 let CppTypeName = "MUBUFInfo"; 2753 let Fields = [ 2754 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2755 "IsBufferInv" 2756 ]; 2757 2758 let PrimaryKey = ["Opcode"]; 2759 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2760} 2761 2762def getMUBUFInfoFromOpcode : SearchIndex { 2763 let Table = MUBUFInfoTable; 2764 let Key = ["Opcode"]; 2765} 2766 2767def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2768 let Table = MUBUFInfoTable; 2769 let Key = ["BaseOpcode", "elements"]; 2770} 2771 2772def MTBUFInfoTable : GenericTable { 2773 let FilterClass = "MTBUF_Pseudo"; 2774 let CppTypeName = "MTBUFInfo"; 2775 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2776 2777 let PrimaryKey = ["Opcode"]; 2778 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2779} 2780 2781def getMTBUFInfoFromOpcode : SearchIndex { 2782 let Table = MTBUFInfoTable; 2783 let Key = ["Opcode"]; 2784} 2785 2786def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2787 let Table = MTBUFInfoTable; 2788 let Key = ["BaseOpcode", "elements"]; 2789} 2790