1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MUBUFLdsTable <bit is_lds, string Name> { 39 bit IsLds = is_lds; 40 string OpName = Name; 41} 42 43class MTBUFAddr64Table <bit is_addr64, string Name> { 44 bit IsAddr64 = is_addr64; 45 string OpName = Name; 46} 47 48//===----------------------------------------------------------------------===// 49// MTBUF classes 50//===----------------------------------------------------------------------===// 51 52class MTBUFGetBaseOpcode<string Op> { 53 string ret = !subst("FORMAT_XY", "FORMAT_X", 54 !subst("FORMAT_XYZ", "FORMAT_X", 55 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 56} 57 58 59class MTBUF_Pseudo <string opName, dag outs, dag ins, 60 string asmOps, list<dag> pattern=[]> : 61 InstSI<outs, ins, "", pattern>, 62 SIMCInstr<opName, SIEncodingFamily.NONE> { 63 64 let isPseudo = 1; 65 let isCodeGenOnly = 1; 66 let Size = 8; 67 let UseNamedOperandTable = 1; 68 69 string Mnemonic = opName; 70 string AsmOperands = asmOps; 71 72 Instruction Opcode = !cast<Instruction>(NAME); 73 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 74 75 let VM_CNT = 1; 76 let EXP_CNT = 1; 77 let MTBUF = 1; 78 let Uses = [EXEC]; 79 let hasSideEffects = 0; 80 let SchedRW = [WriteVMEM]; 81 82 let AsmMatchConverter = "cvtMtbuf"; 83 84 bits<1> offen = 0; 85 bits<1> idxen = 0; 86 bits<1> addr64 = 0; 87 bits<1> has_vdata = 1; 88 bits<1> has_vaddr = 1; 89 bits<1> has_glc = 1; 90 bits<1> has_dlc = 1; 91 bits<1> glc_value = 0; // the value for glc if no such operand 92 bits<1> dlc_value = 0; // the value for dlc if no such operand 93 bits<1> has_srsrc = 1; 94 bits<1> has_soffset = 1; 95 bits<1> has_offset = 1; 96 bits<1> has_slc = 1; 97 bits<1> has_tfe = 1; 98 bits<4> elements = 0; 99 bits<1> has_sccb = 1; 100 bits<1> sccb_value = 0; 101} 102 103class MTBUF_Real <MTBUF_Pseudo ps> : 104 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 105 106 let isPseudo = 0; 107 let isCodeGenOnly = 0; 108 109 let VM_CNT = 1; 110 let EXP_CNT = 1; 111 let MTBUF = 1; 112 113 // copy relevant pseudo op flags 114 let UseNamedOperandTable = ps.UseNamedOperandTable; 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 let SchedRW = ps.SchedRW; 121 let mayLoad = ps.mayLoad; 122 let mayStore = ps.mayStore; 123 let IsAtomicRet = ps.IsAtomicRet; 124 let IsAtomicNoRet = ps.IsAtomicNoRet; 125 126 bits<12> offset; 127 bits<5> cpol; 128 bits<7> format; 129 bits<8> vaddr; 130 bits<10> vdata; 131 bits<7> srsrc; 132 bits<1> tfe; 133 bits<8> soffset; 134 135 bits<4> dfmt = format{3-0}; 136 bits<3> nfmt = format{6-4}; 137 138 // GFX90A+ only: instruction uses AccVGPR for data 139 // Bit supersedes tfe. 140 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 141} 142 143class getMTBUFInsDA<list<RegisterClass> vdataList, 144 list<RegisterClass> vaddrList=[]> { 145 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 146 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 147 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 148 dag InsNoData = !if(!empty(vaddrList), 149 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 150 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 151 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 152 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 153 ); 154 dag InsData = !if(!empty(vaddrList), 155 (ins vdata_op:$vdata, SReg_128:$srsrc, 156 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 157 TFE:$tfe, SWZ:$swz), 158 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 159 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 160 TFE:$tfe, SWZ:$swz) 161 ); 162 dag ret = !if(!empty(vdataList), InsNoData, InsData); 163} 164 165class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 166 dag ret = 167 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 168 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 169 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 170 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 171 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 172 (ins)))))); 173} 174 175class getMTBUFAsmOps<int addrKind> { 176 string Pfx = 177 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 178 !if(!eq(addrKind, BUFAddrKind.OffEn), 179 "$vaddr, $srsrc,$format $soffset offen", 180 !if(!eq(addrKind, BUFAddrKind.IdxEn), 181 "$vaddr, $srsrc,$format $soffset idxen", 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 183 "$vaddr, $srsrc,$format $soffset idxen offen", 184 !if(!eq(addrKind, BUFAddrKind.Addr64), 185 "$vaddr, $srsrc,$format $soffset addr64", 186 ""))))); 187 string ret = Pfx # "$offset"; 188} 189 190class MTBUF_SetupAddr<int addrKind> { 191 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 192 !eq(addrKind, BUFAddrKind.BothEn)); 193 194 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 195 !eq(addrKind, BUFAddrKind.BothEn)); 196 197 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 198 199 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 200} 201 202class MTBUF_Load_Pseudo <string opName, 203 int addrKind, 204 RegisterClass vdataClass, 205 int elems, 206 list<dag> pattern=[], 207 // Workaround bug bz30254 208 int addrKindCopy = addrKind> 209 : MTBUF_Pseudo<opName, 210 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 211 getMTBUFIns<addrKindCopy>.ret, 212 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 213 pattern>, 214 MTBUF_SetupAddr<addrKindCopy> { 215 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 216 let mayLoad = 1; 217 let mayStore = 0; 218 let elements = elems; 219} 220 221multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 222 int elems> { 223 224 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 225 MTBUFAddr64Table<0, NAME>; 226 227 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 228 MTBUFAddr64Table<1, NAME>; 229 230 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 231 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 232 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 233 234 let DisableWQM = 1 in { 235 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 236 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 237 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 238 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 239 } 240} 241 242class MTBUF_Store_Pseudo <string opName, 243 int addrKind, 244 RegisterClass vdataClass, 245 int elems, 246 list<dag> pattern=[], 247 // Workaround bug bz30254 248 int addrKindCopy = addrKind, 249 RegisterClass vdataClassCopy = vdataClass> 250 : MTBUF_Pseudo<opName, 251 (outs), 252 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 253 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 254 pattern>, 255 MTBUF_SetupAddr<addrKindCopy> { 256 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 257 let mayLoad = 0; 258 let mayStore = 1; 259 let elements = elems; 260} 261 262multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 263 int elems> { 264 265 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 266 MTBUFAddr64Table<0, NAME>; 267 268 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 269 MTBUFAddr64Table<1, NAME>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUFGetBaseOpcode<string Op> { 289 string ret = !subst("DWORDX2", "DWORD", 290 !subst("DWORDX3", "DWORD", 291 !subst("DWORDX4", "DWORD", Op))); 292} 293 294class MUBUF_Pseudo <string opName, dag outs, dag ins, 295 string asmOps, list<dag> pattern=[]> : 296 InstSI<outs, ins, "", pattern>, 297 SIMCInstr<opName, SIEncodingFamily.NONE> { 298 299 let isPseudo = 1; 300 let isCodeGenOnly = 1; 301 let Size = 8; 302 let UseNamedOperandTable = 1; 303 304 string Mnemonic = opName; 305 string AsmOperands = asmOps; 306 307 Instruction Opcode = !cast<Instruction>(NAME); 308 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 309 310 let VM_CNT = 1; 311 let EXP_CNT = 1; 312 let MUBUF = 1; 313 let Uses = [EXEC]; 314 let hasSideEffects = 0; 315 let SchedRW = [WriteVMEM]; 316 317 let AsmMatchConverter = "cvtMubuf"; 318 319 bits<1> offen = 0; 320 bits<1> idxen = 0; 321 bits<1> addr64 = 0; 322 bits<1> lds = 0; 323 bits<1> has_vdata = !not(lds); 324 bits<1> has_vaddr = 1; 325 bits<1> has_glc = 1; 326 bits<1> has_dlc = 1; 327 bits<1> glc_value = 0; // the value for glc if no such operand 328 bits<1> dlc_value = 0; // the value for dlc if no such operand 329 bits<1> has_srsrc = 1; 330 bits<1> has_soffset = 1; 331 bits<1> has_offset = 1; 332 bits<1> has_slc = 1; 333 bits<1> has_tfe = 1; 334 bits<4> elements = 0; 335 bits<1> has_sccb = 1; 336 bits<1> sccb_value = 0; 337 bits<1> IsBufferInv = 0; 338} 339 340class MUBUF_Real <MUBUF_Pseudo ps> : 341 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 342 343 let isPseudo = 0; 344 let isCodeGenOnly = 0; 345 346 let VM_CNT = 1; 347 let EXP_CNT = 1; 348 let MUBUF = 1; 349 350 // copy relevant pseudo op flags 351 let SubtargetPredicate = ps.SubtargetPredicate; 352 let AsmMatchConverter = ps.AsmMatchConverter; 353 let OtherPredicates = ps.OtherPredicates; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 let UseNamedOperandTable = ps.UseNamedOperandTable; 358 let SchedRW = ps.SchedRW; 359 let mayLoad = ps.mayLoad; 360 let mayStore = ps.mayStore; 361 let IsAtomicRet = ps.IsAtomicRet; 362 let IsAtomicNoRet = ps.IsAtomicNoRet; 363 let VALU = ps.VALU; 364 let LGKM_CNT = ps.LGKM_CNT; 365 366 bits<12> offset; 367 bits<5> cpol; 368 bits<8> vaddr; 369 bits<10> vdata; 370 bits<7> srsrc; 371 bits<1> tfe; 372 bits<8> soffset; 373 374 // GFX90A+ only: instruction uses AccVGPR for data 375 // Bit supersedes tfe. 376 bits<1> acc = !if(ps.has_vdata, vdata{9}, !if(ps.lds, ?, 0)); 377} 378 379 380// For cache invalidation instructions. 381class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 382 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 383 384 let AsmMatchConverter = ""; 385 386 let hasSideEffects = 1; 387 let mayLoad = 0; 388 let mayStore = 0; 389 390 let IsBufferInv = 1; 391 // Set everything else to 0. 392 let offen = 0; 393 let idxen = 0; 394 let addr64 = 0; 395 let has_vdata = 0; 396 let has_vaddr = 0; 397 let has_glc = 0; 398 let has_dlc = 0; 399 let glc_value = 0; 400 let dlc_value = 0; 401 let has_srsrc = 0; 402 let has_soffset = 0; 403 let has_offset = 0; 404 let has_slc = 0; 405 let has_tfe = 0; 406 let has_sccb = 0; 407 let sccb_value = 0; 408} 409 410class getMUBUFInsDA<list<RegisterClass> vdataList, 411 list<RegisterClass> vaddrList=[], 412 bit isLds = 0> { 413 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 414 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 415 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 416 dag InsNoData = !if(!empty(vaddrList), 417 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 418 offset:$offset, CPol_0:$cpol), 419 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 420 offset:$offset, CPol_0:$cpol) 421 ); 422 dag InsData = !if(!empty(vaddrList), 423 (ins vdata_op:$vdata, SReg_128:$srsrc, 424 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 425 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 426 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 427 ); 428 dag ret = !con( 429 !if(!empty(vdataList), InsNoData, InsData), 430 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 431 ); 432} 433 434class getMUBUFElements<ValueType vt> { 435 int ret = 436 !if(!eq(vt, f16), 1, 437 !if(!eq(vt, v2f16), 2, 438 !if(!eq(vt, v3f16), 3, 439 !if(!eq(vt, v4f16), 4, 440 !if(!eq(vt.Size, 32), 1, 441 !if(!eq(vt.Size, 64), 2, 442 !if(!eq(vt.Size, 96), 3, 443 !if(!eq(vt.Size, 128), 4, 0) 444 ) 445 ) 446 ) 447 ) 448 ) 449 ) 450 ); 451} 452 453class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 454 dag ret = 455 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 456 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 457 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 458 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 459 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 460 (ins)))))); 461} 462 463class getMUBUFAsmOps<int addrKind> { 464 string Pfx = 465 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 466 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 467 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 468 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 469 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 470 ""))))); 471 string ret = Pfx # "$offset"; 472} 473 474class MUBUF_SetupAddr<int addrKind> { 475 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 476 !eq(addrKind, BUFAddrKind.BothEn)); 477 478 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 479 !eq(addrKind, BUFAddrKind.BothEn)); 480 481 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 482 483 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 484} 485 486class MUBUF_Load_Pseudo <string opName, 487 int addrKind, 488 ValueType vdata_vt, 489 bit HasTiedDest = 0, 490 bit isLds = 0, 491 list<dag> pattern=[], 492 // Workaround bug bz30254 493 int addrKindCopy = addrKind, 494 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 495 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 496 : MUBUF_Pseudo<opName, 497 !if(isLds, (outs), (outs vdata_op:$vdata)), 498 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 499 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 500 !if(isLds, " ", " $vdata, ") # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 501 !if(isLds, " lds", "$tfe") # "$swz", 502 pattern>, 503 MUBUF_SetupAddr<addrKindCopy> { 504 let PseudoInstr = opName # !if(isLds, "_lds", "") # 505 "_" # getAddrName<addrKindCopy>.ret; 506 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 507 508 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 509 let LGKM_CNT = isLds; 510 let mayLoad = 1; 511 let mayStore = isLds; 512 let maybeAtomic = 1; 513 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 514 let has_tfe = !not(isLds); 515 let lds = isLds; 516 let elements = getMUBUFElements<vdata_vt>.ret; 517 let VALU = isLds; 518} 519 520class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 521 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 522 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 523>; 524 525class MUBUF_Addr64_Load_Pat <Instruction inst, 526 ValueType load_vt = i32, 527 SDPatternOperator ld = null_frag> : Pat < 528 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 529 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 530>; 531 532multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 533 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 534 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 535} 536 537 538// FIXME: tfe can't be an operand because it requires a separate 539// opcode because it needs an N+1 register class dest register. 540multiclass MUBUF_Pseudo_Loads<string opName, 541 ValueType load_vt = i32, 542 bit TiedDest = 0, 543 bit isLds = 0> { 544 545 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 546 547 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 548 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 549 550 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 551 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 552 553 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 554 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 555 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 556 557 let DisableWQM = 1 in { 558 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 559 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 560 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 561 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 562 } 563} 564 565multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 566 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 567 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 568} 569 570class MUBUF_Store_Pseudo <string opName, 571 int addrKind, 572 ValueType store_vt, 573 list<dag> pattern=[], 574 // Workaround bug bz30254 575 int addrKindCopy = addrKind> 576 : MUBUF_Pseudo<opName, 577 (outs), 578 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 579 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 580 pattern>, 581 MUBUF_SetupAddr<addrKindCopy> { 582 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 583 let mayLoad = 0; 584 let mayStore = 1; 585 let maybeAtomic = 1; 586 let elements = getMUBUFElements<store_vt>.ret; 587} 588 589multiclass MUBUF_Pseudo_Stores<string opName, 590 ValueType store_vt = i32, 591 SDPatternOperator st = null_frag> { 592 593 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 594 595 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 596 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 597 i16:$offset))]>, 598 MUBUFAddr64Table<0, NAME>; 599 600 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 601 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 602 i16:$offset))]>, 603 MUBUFAddr64Table<1, NAME>; 604 605 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 606 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 607 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 608 609 let DisableWQM = 1 in { 610 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 611 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 612 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 613 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 614 } 615} 616 617class MUBUF_Pseudo_Store_Lds<string opName> 618 : MUBUF_Pseudo<opName, 619 (outs), 620 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 621 " $srsrc, $soffset$offset lds$cpol$swz"> { 622 let LGKM_CNT = 1; 623 let mayLoad = 1; 624 let mayStore = 1; 625 let maybeAtomic = 1; 626 627 let has_vdata = 0; 628 let has_vaddr = 0; 629 let has_tfe = 0; 630 let lds = 1; 631 let VALU = 1; 632 633 let Uses = [EXEC, M0]; 634 let AsmMatchConverter = "cvtMubufLds"; 635} 636 637class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 638 list<RegisterClass> vaddrList=[]> { 639 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 640 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 641 dag ret = !if(vdata_in, 642 !if(!empty(vaddrList), 643 (ins vdata_op:$vdata_in, 644 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 645 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 646 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 647 ), 648 !if(!empty(vaddrList), 649 (ins vdata_op:$vdata, 650 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 651 (ins vdata_op:$vdata, vaddrClass:$vaddr, 652 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 653 )); 654} 655 656class getMUBUFAtomicIns<int addrKind, 657 RegisterClass vdataClass, 658 bit vdata_in, 659 // Workaround bug bz30254 660 RegisterClass vdataClassCopy=vdataClass> { 661 dag ret = 662 !if(!eq(addrKind, BUFAddrKind.Offset), 663 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 664 !if(!eq(addrKind, BUFAddrKind.OffEn), 665 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 666 !if(!eq(addrKind, BUFAddrKind.IdxEn), 667 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 668 !if(!eq(addrKind, BUFAddrKind.BothEn), 669 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 670 !if(!eq(addrKind, BUFAddrKind.Addr64), 671 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 672 (ins)))))); 673} 674 675class MUBUF_Atomic_Pseudo<string opName, 676 int addrKind, 677 dag outs, 678 dag ins, 679 string asmOps, 680 list<dag> pattern=[], 681 // Workaround bug bz30254 682 int addrKindCopy = addrKind> 683 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 684 MUBUF_SetupAddr<addrKindCopy> { 685 let mayStore = 1; 686 let mayLoad = 1; 687 let hasPostISelHook = 1; 688 let hasSideEffects = 1; 689 let DisableWQM = 1; 690 let has_glc = 0; 691 let has_dlc = 0; 692 let has_tfe = 0; 693 let has_sccb = 1; 694 let maybeAtomic = 1; 695 let AsmMatchConverter = "cvtMubufAtomic"; 696} 697 698class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 699 RegisterClass vdataClass, 700 list<dag> pattern=[], 701 // Workaround bug bz30254 702 int addrKindCopy = addrKind, 703 RegisterClass vdataClassCopy = vdataClass> 704 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 705 (outs), 706 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 707 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 708 pattern>, 709 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 710 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 711 let glc_value = 0; 712 let dlc_value = 0; 713 let sccb_value = 0; 714 let IsAtomicNoRet = 1; 715} 716 717class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 718 RegisterClass vdataClass, 719 list<dag> pattern=[], 720 // Workaround bug bz30254 721 int addrKindCopy = addrKind, 722 RegisterClass vdataClassCopy = vdataClass, 723 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 724 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 725 (outs vdata_op:$vdata), 726 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 727 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 728 pattern>, 729 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 730 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 731 let glc_value = 1; 732 let dlc_value = 0; 733 let sccb_value = 0; 734 let IsAtomicRet = 1; 735 let Constraints = "$vdata = $vdata_in"; 736 let DisableEncoding = "$vdata_in"; 737} 738 739multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 740 RegisterClass vdataClass, 741 ValueType vdataType, 742 bit isFP = isFloatType<vdataType>.ret> { 743 let FPAtomic = isFP in 744 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 745 MUBUFAddr64Table <0, NAME>; 746 747 let FPAtomic = isFP in 748 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 749 MUBUFAddr64Table <1, NAME>; 750 751 let FPAtomic = isFP in 752 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 753 754 let FPAtomic = isFP in 755 756 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 757 758 let FPAtomic = isFP in 759 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 760} 761 762multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 763 RegisterClass vdataClass, 764 ValueType vdataType, 765 SDPatternOperator atomic, 766 bit isFP = isFloatType<vdataType>.ret> { 767 let FPAtomic = isFP in 768 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 769 [(set vdataType:$vdata, 770 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 771 vdataType:$vdata_in))]>, 772 MUBUFAddr64Table <0, NAME # "_RTN">; 773 774 let FPAtomic = isFP in 775 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 776 [(set vdataType:$vdata, 777 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 778 vdataType:$vdata_in))]>, 779 MUBUFAddr64Table <1, NAME # "_RTN">; 780 781 let FPAtomic = isFP in 782 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 783 784 let FPAtomic = isFP in 785 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 786 787 let FPAtomic = isFP in 788 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 789} 790 791multiclass MUBUF_Pseudo_Atomics <string opName, 792 RegisterClass vdataClass, 793 ValueType vdataType, 794 SDPatternOperator atomic = null_frag> : 795 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 796 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 797 798 799//===----------------------------------------------------------------------===// 800// MUBUF Instructions 801//===----------------------------------------------------------------------===// 802 803defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 804 "buffer_load_format_x", f32 805>; 806defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 807 "buffer_load_format_xy", v2f32 808>; 809defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 810 "buffer_load_format_xyz", v3f32 811>; 812defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 813 "buffer_load_format_xyzw", v4f32 814>; 815defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 816 "buffer_store_format_x", f32 817>; 818defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 819 "buffer_store_format_xy", v2f32 820>; 821defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 822 "buffer_store_format_xyz", v3f32 823>; 824defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 825 "buffer_store_format_xyzw", v4f32 826>; 827 828let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 829 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 830 "buffer_load_format_d16_x", i32 831 >; 832 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 833 "buffer_load_format_d16_xy", v2i32 834 >; 835 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 836 "buffer_load_format_d16_xyz", v3i32 837 >; 838 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 839 "buffer_load_format_d16_xyzw", v4i32 840 >; 841 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 842 "buffer_store_format_d16_x", i32 843 >; 844 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_xy", v2i32 846 >; 847 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 848 "buffer_store_format_d16_xyz", v3i32 849 >; 850 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 851 "buffer_store_format_d16_xyzw", v4i32 852 >; 853} // End HasUnpackedD16VMem. 854 855let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 856 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 857 "buffer_load_format_d16_x", f16 858 >; 859 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 860 "buffer_load_format_d16_xy", v2f16 861 >; 862 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 863 "buffer_load_format_d16_xyz", v3f16 864 >; 865 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 866 "buffer_load_format_d16_xyzw", v4f16 867 >; 868 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 869 "buffer_store_format_d16_x", f16 870 >; 871 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 872 "buffer_store_format_d16_xy", v2f16 873 >; 874 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 875 "buffer_store_format_d16_xyz", v3f16 876 >; 877 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 878 "buffer_store_format_d16_xyzw", v4f16 879 >; 880} // End HasPackedD16VMem. 881 882defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 883 "buffer_load_ubyte", i32 884>; 885defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 886 "buffer_load_sbyte", i32 887>; 888defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 889 "buffer_load_ushort", i32 890>; 891defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 892 "buffer_load_sshort", i32 893>; 894defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 895 "buffer_load_dword", i32 896>; 897defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 898 "buffer_load_dwordx2", v2i32 899>; 900defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 901 "buffer_load_dwordx3", v3i32 902>; 903defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 904 "buffer_load_dwordx4", v4i32 905>; 906 907defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 908defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 909defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 910defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 911defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 912defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 913defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 914defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 915defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 916defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 917defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 918defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 919defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 920defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 921 922// This is not described in AMD documentation, 923// but 'lds' versions of these opcodes are available 924// in at least GFX8+ chips. See Bug 37653. 925let SubtargetPredicate = isGFX8GFX9 in { 926defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 927 "buffer_load_dwordx2", v2i32, 0, 1 928>; 929defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 930 "buffer_load_dwordx3", v3i32, 0, 1 931>; 932defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 933 "buffer_load_dwordx4", v4i32, 0, 1 934>; 935} 936 937defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 938 "buffer_store_byte", i32, truncstorei8_global 939>; 940defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 941 "buffer_store_short", i32, truncstorei16_global 942>; 943defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 944 "buffer_store_dword", i32, store_global 945>; 946defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 947 "buffer_store_dwordx2", v2i32, store_global 948>; 949defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 950 "buffer_store_dwordx3", v3i32, store_global 951>; 952defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 953 "buffer_store_dwordx4", v4i32, store_global 954>; 955defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_swap", VGPR_32, i32 957>; 958defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_cmpswap", VReg_64, v2i32 960>; 961defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_add", VGPR_32, i32 963>; 964defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_sub", VGPR_32, i32 966>; 967defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_smin", VGPR_32, i32 969>; 970defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_umin", VGPR_32, i32 972>; 973defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_smax", VGPR_32, i32 975>; 976defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_umax", VGPR_32, i32 978>; 979defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_and", VGPR_32, i32 981>; 982defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_or", VGPR_32, i32 984>; 985defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_xor", VGPR_32, i32 987>; 988defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_inc", VGPR_32, i32 990>; 991defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_dec", VGPR_32, i32 993>; 994defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_swap_x2", VReg_64, i64 996>; 997defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 999>; 1000defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_add_x2", VReg_64, i64 1002>; 1003defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_sub_x2", VReg_64, i64 1005>; 1006defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_smin_x2", VReg_64, i64 1008>; 1009defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_umin_x2", VReg_64, i64 1011>; 1012defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_smax_x2", VReg_64, i64 1014>; 1015defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_umax_x2", VReg_64, i64 1017>; 1018defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_and_x2", VReg_64, i64 1020>; 1021defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_or_x2", VReg_64, i64 1023>; 1024defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_xor_x2", VReg_64, i64 1026>; 1027defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_inc_x2", VReg_64, i64 1029>; 1030defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_dec_x2", VReg_64, i64 1032>; 1033 1034let SubtargetPredicate = HasGFX10_BEncoding in 1035defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1036 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1037>; 1038 1039let SubtargetPredicate = isGFX8GFX9 in { 1040def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1041} 1042 1043let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1044/* 1045defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1046defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1047*/ 1048 1049def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1050 int_amdgcn_buffer_wbinvl1_sc>; 1051} 1052 1053let SubtargetPredicate = isGFX6GFX7GFX10 in { 1054 1055defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1056 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1057>; 1058defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1059 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1060>; 1061defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1062 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1063>; 1064defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1066>; 1067defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1069>; 1070defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1072>; 1073 1074} 1075 1076let SubtargetPredicate = HasD16LoadStore in { 1077 1078defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1079 "buffer_load_ubyte_d16", i32, 1 1080>; 1081 1082defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1083 "buffer_load_ubyte_d16_hi", i32, 1 1084>; 1085 1086defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1087 "buffer_load_sbyte_d16", i32, 1 1088>; 1089 1090defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1091 "buffer_load_sbyte_d16_hi", i32, 1 1092>; 1093 1094defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1095 "buffer_load_short_d16", i32, 1 1096>; 1097 1098defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1099 "buffer_load_short_d16_hi", i32, 1 1100>; 1101 1102defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1103 "buffer_store_byte_d16_hi", i32 1104>; 1105 1106defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1107 "buffer_store_short_d16_hi", i32 1108>; 1109 1110defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1111 "buffer_load_format_d16_hi_x", i32 1112>; 1113defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1114 "buffer_store_format_d16_hi_x", i32 1115>; 1116 1117} // End HasD16LoadStore 1118 1119def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1120 int_amdgcn_buffer_wbinvl1>; 1121 1122let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1123defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1124 "buffer_atomic_add_f32", VGPR_32, f32 1125>; 1126 1127let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1128defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1129 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1130>; 1131 1132let OtherPredicates = [HasAtomicFaddRtnInsts] in 1133defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1134 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1135>; 1136 1137let OtherPredicates = [isGFX90APlus] in 1138defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1139 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1140>; 1141 1142//===----------------------------------------------------------------------===// 1143// MTBUF Instructions 1144//===----------------------------------------------------------------------===// 1145 1146defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1147defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1148defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1149defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1150defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1151defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1152defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1153defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1154 1155let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1156 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1157 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1158 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1159 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1160 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1161 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1162 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1163 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1164} // End HasUnpackedD16VMem. 1165 1166let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1167 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1168 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1169 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1170 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1171 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1172 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1173 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1174 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1175} // End HasPackedD16VMem. 1176 1177let SubtargetPredicate = isGFX7Plus in { 1178 1179//===----------------------------------------------------------------------===// 1180// Instruction definitions for CI and newer. 1181//===----------------------------------------------------------------------===// 1182 1183def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1184 int_amdgcn_buffer_wbinvl1_vol>; 1185 1186} // End let SubtargetPredicate = isGFX7Plus 1187 1188let SubtargetPredicate = isGFX90APlus in { 1189 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1190 let has_glc = 1; 1191 let has_sccb = 1; 1192 let InOperandList = (ins CPol_0:$cpol); 1193 let AsmOperands = "$cpol"; 1194 } 1195 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1196 let SubtargetPredicate = isGFX90AOnly; 1197 } 1198 1199 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1200 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1201 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1202} // End SubtargetPredicate = isGFX90APlus 1203 1204def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1205 let SubtargetPredicate = isGFX940Plus; 1206 let has_glc = 1; 1207 let has_sccb = 1; 1208 let InOperandList = (ins CPol_0:$cpol); 1209 let AsmOperands = "$cpol"; 1210} 1211 1212let SubtargetPredicate = isGFX10Plus in { 1213 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1214 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1215} // End SubtargetPredicate = isGFX10Plus 1216 1217//===----------------------------------------------------------------------===// 1218// MUBUF Patterns 1219//===----------------------------------------------------------------------===// 1220 1221//===----------------------------------------------------------------------===// 1222// buffer_load/store_format patterns 1223//===----------------------------------------------------------------------===// 1224 1225multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1226 string opcode, ValueType memoryVt = vt> { 1227 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1228 1229 def : GCNPat< 1230 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1231 timm:$auxiliary, 0)), 1232 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1233 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1234 >; 1235 1236 def : GCNPat< 1237 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1238 timm:$auxiliary, 0)), 1239 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1240 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1241 >; 1242 1243 def : GCNPat< 1244 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1245 timm:$auxiliary, timm)), 1246 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1247 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1248 >; 1249 1250 def : GCNPat< 1251 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1252 timm:$auxiliary, timm)), 1253 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1254 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1255 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1256 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1257 >; 1258} 1259 1260defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1261defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1262defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1263defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1264defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1265defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1266defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1267defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1268 1269let SubtargetPredicate = HasUnpackedD16VMem in { 1270 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1271 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1272 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1273 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1274 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1275 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1276} // End HasUnpackedD16VMem. 1277 1278let SubtargetPredicate = HasPackedD16VMem in { 1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1282 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1283 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1284 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1285 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1286 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1287 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1288} // End HasPackedD16VMem. 1289 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1291defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1292defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1293defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1294defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1295defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1296defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1297defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1298defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1299defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1300defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1301defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1302defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1303defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1304defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1305defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1306 1307multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1308 string opcode, ValueType memoryVt = vt> { 1309 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1310 1311 def : GCNPat< 1312 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1313 timm:$auxiliary, 0), 1314 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1315 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1316 >; 1317 1318 def : GCNPat< 1319 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1320 timm:$auxiliary, 0), 1321 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1322 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1323 >; 1324 1325 def : GCNPat< 1326 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1327 timm:$auxiliary, timm), 1328 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1329 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1330 >; 1331 1332 def : GCNPat< 1333 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1334 timm:$auxiliary, timm), 1335 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1336 getVregSrcForVT<vt>.ret:$vdata, 1337 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1338 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1339 0, (extract_swz $auxiliary)) 1340 >; 1341} 1342 1343defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1344defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1345defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1346defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1347defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1348defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1349defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1350defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1351 1352let SubtargetPredicate = HasUnpackedD16VMem in { 1353 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1354 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1355 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1356 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1357 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1358 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1359} // End HasUnpackedD16VMem. 1360 1361let SubtargetPredicate = HasPackedD16VMem in { 1362 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1363 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1365 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1366 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1367 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1368 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1369 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1370 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1371} // End HasPackedD16VMem. 1372 1373defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1374defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1375defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1376defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1377defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1378defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1379defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1380defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1381defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1382defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1383defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1384defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1385defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1386defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1387 1388//===----------------------------------------------------------------------===// 1389// buffer_atomic patterns 1390//===----------------------------------------------------------------------===// 1391 1392multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1393 foreach RtnMode = ["ret", "noret"] in { 1394 1395 defvar Op = !cast<SDPatternOperator>(OpPrefix # "_" # RtnMode 1396 # !if(isIntr, "", "_" # vt.Size)); 1397 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1398 1399 def : GCNPat< 1400 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1401 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1402 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1403 >; 1404 1405 def : GCNPat< 1406 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1407 vt:$vdata_in)), 1408 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1409 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1410 >; 1411 1412 } // end foreach RtnMode 1413} 1414 1415multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1416 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1417} 1418 1419multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1420 foreach RtnMode = ["ret", "noret"] in { 1421 1422 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode 1423 # "_" # vt.Size); 1424 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1425 1426 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1427 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1428 offset:$offset); 1429 def : GCNPat< 1430 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1431 !if(!eq(RtnMode, "ret"), 1432 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1433 !if(!eq(vt, i32), sub0, sub0_sub1)), 1434 OffsetResDag) 1435 >; 1436 1437 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1438 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1439 SCSrc_b32:$soffset, offset:$offset); 1440 def : GCNPat< 1441 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1442 data_vt:$vdata_in)), 1443 !if(!eq(RtnMode, "ret"), 1444 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1445 !if(!eq(vt, i32), sub0, sub0_sub1)), 1446 Addr64ResDag) 1447 >; 1448 1449 } // end foreach RtnMode 1450} 1451 1452foreach Ty = [i32, i64] in { 1453 1454defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1455 1456defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1457defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1458defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1459defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1460defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1461defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1462defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1463defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1464defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1465defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1466defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1467defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1468 1469} // end foreach Ty 1470 1471defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1472defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1473 1474multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1475 list<string> RtnModes = ["ret", "noret"]> { 1476 foreach RtnMode = RtnModes in { 1477 1478 defvar Op = !cast<SDPatternOperator>(!if(!eq(RtnMode, "none"), 1479 OpPrefix, OpPrefix # "_" # RtnMode)); 1480 defvar InstSuffix = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1481 "_RTN", ""); 1482 defvar CachePolicy = !if(!or(!eq(RtnMode, "none"), !eq(RtnMode, "ret")), 1483 (set_glc $cachepolicy), (timm:$cachepolicy)); 1484 1485 def : GCNPat< 1486 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1487 timm:$offset, timm:$cachepolicy, 0)), 1488 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1489 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1490 (as_i16timm $offset), CachePolicy) 1491 >; 1492 1493 def : GCNPat< 1494 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1495 timm:$offset, timm:$cachepolicy, timm)), 1496 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1497 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1498 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1499 >; 1500 1501 def : GCNPat< 1502 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1503 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1504 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1505 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1506 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1507 >; 1508 1509 def : GCNPat< 1510 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1511 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1512 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1513 getVregSrcForVT<vt>.ret:$vdata_in, 1514 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1515 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1516 >; 1517 1518 } // end foreach RtnMode 1519} 1520 1521defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1522defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1523defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1524defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1525defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1526defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1527defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1528defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1529defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1530defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1531defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1532defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1533defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1534defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["none"]>; 1535defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1536defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1537defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1538defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1539defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1540defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1545defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1546defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1547 1548let SubtargetPredicate = isGFX6GFX7GFX10 in { 1549 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1550 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1551 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1552 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1553} 1554 1555class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1556 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1557 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1558 [{ return SDValue(N, 0).use_empty(); }]> { 1559 1560 let GISelPredicateCode = [{ 1561 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1562 }]; 1563} 1564 1565multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1566 string opcode> { 1567 def : GCNPat< 1568 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1569 0, i32:$soffset, timm:$offset, 1570 timm:$cachepolicy, 0), 1571 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1572 (as_i16timm $offset), $cachepolicy) 1573 >; 1574 1575 def : GCNPat< 1576 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1577 0, i32:$soffset, timm:$offset, 1578 timm:$cachepolicy, timm), 1579 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1580 (as_i16timm $offset), $cachepolicy) 1581 >; 1582 1583 def : GCNPat< 1584 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1585 i32:$voffset, i32:$soffset, timm:$offset, 1586 timm:$cachepolicy, 0), 1587 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1588 (as_i16timm $offset), $cachepolicy) 1589 >; 1590 1591 def : GCNPat< 1592 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1593 i32:$voffset, i32:$soffset, timm:$offset, 1594 timm:$cachepolicy, timm), 1595 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1596 getVregSrcForVT<vt>.ret:$vdata_in, 1597 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1598 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1599 >; 1600} 1601 1602let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1603defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1604 1605let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1606defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1607 1608let SubtargetPredicate = HasAtomicFaddRtnInsts in 1609 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1610 1611let SubtargetPredicate = isGFX90APlus in { 1612 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1613 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1614 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1615 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1616 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1617 1618 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1619 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1620 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1621} // End SubtargetPredicate = isGFX90APlus 1622 1623foreach RtnMode = ["ret", "noret"] in { 1624 1625defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap # "_" # RtnMode); 1626defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1627defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1628 (timm:$cachepolicy)); 1629 1630defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1631 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1632 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1633def : GCNPat< 1634 (Op 1635 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1636 timm:$offset, timm:$cachepolicy, 0), 1637 !if(!eq(RtnMode, "ret"), 1638 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1639 OffsetResDag) 1640>; 1641 1642defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1643 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1644 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1645 CachePolicy); 1646def : GCNPat< 1647 (Op 1648 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1649 0, i32:$soffset, timm:$offset, 1650 timm:$cachepolicy, timm), 1651 !if(!eq(RtnMode, "ret"), 1652 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1653 IdxenResDag) 1654>; 1655 1656defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1657 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1658 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1659 CachePolicy); 1660def : GCNPat< 1661 (Op 1662 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1663 i32:$voffset, i32:$soffset, timm:$offset, 1664 timm:$cachepolicy, 0), 1665 !if(!eq(RtnMode, "ret"), 1666 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1667 OffenResDag) 1668>; 1669 1670defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1671 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1672 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1673 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1674def : GCNPat< 1675 (Op 1676 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1677 i32:$voffset, i32:$soffset, timm:$offset, 1678 timm:$cachepolicy, timm), 1679 !if(!eq(RtnMode, "ret"), 1680 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1681 BothenResDag) 1682>; 1683 1684} // end foreach RtnMode 1685 1686class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1687 PatFrag constant_ld> : GCNPat < 1688 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1689 i16:$offset))), 1690 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1691 >; 1692 1693multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1694 ValueType vt, PatFrag atomic_ld> { 1695 def : GCNPat < 1696 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1697 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1698 >; 1699 1700 def : GCNPat < 1701 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1702 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1703 >; 1704} 1705 1706let SubtargetPredicate = isGFX6GFX7 in { 1707def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1708def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1709def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1710def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1711def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1712def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1713 1714defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1715defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1716} // End SubtargetPredicate = isGFX6GFX7 1717 1718multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1719 PatFrag ld> { 1720 1721 def : GCNPat < 1722 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1723 (Instr_OFFSET $srsrc, $soffset, $offset) 1724 >; 1725} 1726 1727let OtherPredicates = [Has16BitInsts] in { 1728 1729defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1730defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1731defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1732defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1733defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1734defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1735 1736defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1737 1738} // End OtherPredicates = [Has16BitInsts] 1739 1740multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1741 MUBUF_Pseudo InstrOffset, 1742 ValueType vt, PatFrag ld> { 1743 def : GCNPat < 1744 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1745 i32:$soffset, u16imm:$offset))), 1746 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1747 >; 1748 1749 def : GCNPat < 1750 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1751 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1752 >; 1753} 1754 1755// XXX - Is it possible to have a complex pattern in a PatFrag? 1756multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1757 MUBUF_Pseudo InstrOffset, 1758 ValueType vt, PatFrag ld_frag> { 1759 def : GCNPat < 1760 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1761 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1762 >; 1763 1764 def : GCNPat < 1765 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1766 (InstrOffset $srsrc, $soffset, $offset, $in) 1767 >; 1768} 1769 1770let OtherPredicates = [DisableFlatScratch] in { 1771defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1772defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1773defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1774defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1775defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1776defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1777defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1778defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1779defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1780defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1781 1782foreach vt = Reg32Types.types in { 1783defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1784} 1785defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1786defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1787defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1788 1789let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1790defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1791defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1792defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1793defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1794defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1795defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1796 1797defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1798defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1799defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1800defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1801defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1802defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1803} 1804 1805} // End OtherPredicates = [DisableFlatScratch] 1806 1807multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1808 ValueType vt, PatFrag atomic_st> { 1809 // Store follows atomic op convention so address is first 1810 def : GCNPat < 1811 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1812 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1813 >; 1814 1815 def : GCNPat < 1816 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1817 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1818 >; 1819} 1820let SubtargetPredicate = isGFX6GFX7 in { 1821defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1822defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1823defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1824defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1825defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1826defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1827} // End Predicates = isGFX6GFX7 1828 1829 1830multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1831 PatFrag st> { 1832 1833 def : GCNPat < 1834 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1835 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1836 >; 1837} 1838 1839defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1840defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1841 1842multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1843 MUBUF_Pseudo InstrOffset, 1844 ValueType vt, PatFrag st, 1845 RegisterClass rc = VGPR_32> { 1846 def : GCNPat < 1847 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1848 i32:$soffset, u16imm:$offset)), 1849 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1850 >; 1851 1852 def : GCNPat < 1853 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1854 u16imm:$offset)), 1855 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1856 >; 1857} 1858 1859let OtherPredicates = [DisableFlatScratch] in { 1860defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1861defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1862defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1863defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1864 1865foreach vt = Reg32Types.types in { 1866defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1867} 1868 1869defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1870defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1871defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1872 1873 1874let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1875 // Hiding the extract high pattern in the PatFrag seems to not 1876 // automatically increase the complexity. 1877let AddedComplexity = 1 in { 1878defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1879defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1880} 1881} 1882} // End OtherPredicates = [DisableFlatScratch] 1883 1884//===----------------------------------------------------------------------===// 1885// MTBUF Patterns 1886//===----------------------------------------------------------------------===// 1887 1888//===----------------------------------------------------------------------===// 1889// tbuffer_load/store_format patterns 1890//===----------------------------------------------------------------------===// 1891 1892multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1893 string opcode, ValueType memoryVt = vt> { 1894 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1895 1896 def : GCNPat< 1897 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1898 timm:$format, timm:$auxiliary, 0)), 1899 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1900 (as_i8timm $format), 1901 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1902 >; 1903 1904 def : GCNPat< 1905 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1906 timm:$format, timm:$auxiliary, timm)), 1907 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1908 (as_i8timm $format), 1909 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1910 >; 1911 1912 def : GCNPat< 1913 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1914 timm:$format, timm:$auxiliary, 0)), 1915 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1916 (as_i8timm $format), 1917 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1918 >; 1919 1920 def : GCNPat< 1921 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1922 timm:$format, timm:$auxiliary, timm)), 1923 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1924 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1925 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1926 (as_i8timm $format), 1927 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1928 >; 1929} 1930 1931defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1932defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1933defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1934defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1935defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1936defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1937defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1938defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1939 1940let SubtargetPredicate = HasUnpackedD16VMem in { 1941 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1942 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1943 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1944 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1945 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1946} // End HasUnpackedD16VMem. 1947 1948let SubtargetPredicate = HasPackedD16VMem in { 1949 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1950 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1951 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1952 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1953 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1954} // End HasPackedD16VMem. 1955 1956multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1957 string opcode, ValueType memoryVt = vt> { 1958 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1959 1960 def : GCNPat< 1961 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1962 timm:$format, timm:$auxiliary, 0), 1963 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1964 (as_i16timm $offset), (as_i8timm $format), 1965 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1966 >; 1967 1968 def : GCNPat< 1969 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1970 timm:$format, timm:$auxiliary, timm), 1971 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1972 (as_i16timm $offset), (as_i8timm $format), 1973 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1974 >; 1975 1976 def : GCNPat< 1977 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1978 timm:$format, timm:$auxiliary, 0), 1979 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1980 (as_i16timm $offset), (as_i8timm $format), 1981 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1982 >; 1983 1984 def : GCNPat< 1985 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1986 timm:$offset, timm:$format, timm:$auxiliary, timm), 1987 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1988 getVregSrcForVT<vt>.ret:$vdata, 1989 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1990 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1991 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1992 >; 1993} 1994 1995defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1996defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1997defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1998defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1999defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2000defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2001defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2002defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2003 2004let SubtargetPredicate = HasUnpackedD16VMem in { 2005 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2006 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2007 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2008 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2009 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2010} // End HasUnpackedD16VMem. 2011 2012let SubtargetPredicate = HasPackedD16VMem in { 2013 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2014 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2015 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2016 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2017 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2018} // End HasPackedD16VMem. 2019 2020//===----------------------------------------------------------------------===// 2021// Target-specific instruction encodings. 2022//===----------------------------------------------------------------------===// 2023 2024//===----------------------------------------------------------------------===// 2025// Base ENC_MUBUF for GFX6, GFX7, GFX10. 2026//===----------------------------------------------------------------------===// 2027 2028class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2029 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2030 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2031 let Inst{12} = ps.offen; 2032 let Inst{13} = ps.idxen; 2033 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2034 let Inst{16} = ps.lds; 2035 let Inst{24-18} = op; 2036 let Inst{31-26} = 0x38; 2037 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2038 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2039 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2040 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2041 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2042 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2043} 2044 2045class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2046 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2047 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2048 let Inst{25} = op{7}; 2049} 2050 2051class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2052 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2053 let Inst{15} = ps.addr64; 2054} 2055 2056//===----------------------------------------------------------------------===// 2057// MUBUF - GFX10. 2058//===----------------------------------------------------------------------===// 2059 2060let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2061 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2062 def _BOTHEN_gfx10 : 2063 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2064 def _IDXEN_gfx10 : 2065 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2066 def _OFFEN_gfx10 : 2067 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2068 def _OFFSET_gfx10 : 2069 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2070 } 2071 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2072 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2073 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 2074 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2075 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 2076 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2077 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 2078 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2079 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 2080 2081 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2082 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 2083 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2084 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 2085 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2086 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 2087 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2088 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 2089 } 2090 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2091 def _BOTHEN_RTN_gfx10 : 2092 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2093 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2094 def _IDXEN_RTN_gfx10 : 2095 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2096 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2097 def _OFFEN_RTN_gfx10 : 2098 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2099 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2100 def _OFFSET_RTN_gfx10 : 2101 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2102 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2103 } 2104 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2105 MUBUF_Real_Atomics_RTN_gfx10<op> { 2106 def _BOTHEN_gfx10 : 2107 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2108 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2109 def _IDXEN_gfx10 : 2110 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2111 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2112 def _OFFEN_gfx10 : 2113 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2114 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2115 def _OFFSET_gfx10 : 2116 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2117 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2118 } 2119} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2120 2121defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2122defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2123defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2124defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2125defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2126defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2127defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2128defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2129// FIXME-GFX10: Add following instructions: 2130//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2131//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2132defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2133defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2134defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2135defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2136defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2137defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2138defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2139defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2140 2141def BUFFER_GL0_INV_gfx10 : 2142 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2143def BUFFER_GL1_INV_gfx10 : 2144 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2145 2146//===----------------------------------------------------------------------===// 2147// MUBUF - GFX6, GFX7, GFX10. 2148//===----------------------------------------------------------------------===// 2149 2150let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2151 multiclass MUBUF_Real_gfx6<bits<8> op> { 2152 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2153 } 2154} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2155 2156let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2157 multiclass MUBUF_Real_gfx7<bits<8> op> { 2158 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2159 } 2160} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2161 2162let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2163 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2164 def _ADDR64_gfx6_gfx7 : 2165 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2166 def _BOTHEN_gfx6_gfx7 : 2167 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2168 def _IDXEN_gfx6_gfx7 : 2169 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2170 def _OFFEN_gfx6_gfx7 : 2171 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2172 def _OFFSET_gfx6_gfx7 : 2173 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2174 } 2175 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2176 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2177 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2178 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2179 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2180 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2181 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2182 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2183 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2184 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2185 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2186 2187 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2188 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2189 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2190 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2191 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2192 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2193 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2194 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2195 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2196 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2197 } 2198 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2199 def _ADDR64_gfx6_gfx7 : 2200 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2201 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2202 def _BOTHEN_gfx6_gfx7 : 2203 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2204 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2205 def _IDXEN_gfx6_gfx7 : 2206 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2207 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2208 def _OFFEN_gfx6_gfx7 : 2209 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2210 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2211 def _OFFSET_gfx6_gfx7 : 2212 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2213 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2214 2215 def _ADDR64_RTN_gfx6_gfx7 : 2216 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2217 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2218 def _BOTHEN_RTN_gfx6_gfx7 : 2219 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2220 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2221 def _IDXEN_RTN_gfx6_gfx7 : 2222 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2223 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2224 def _OFFEN_RTN_gfx6_gfx7 : 2225 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2226 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2227 def _OFFSET_RTN_gfx6_gfx7 : 2228 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2229 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2230 } 2231} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2232 2233multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2234 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2235 2236multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2237 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2238 2239multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2240 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2241 2242// FIXME-GFX6: Following instructions are available only on GFX6. 2243//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2244//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2245 2246defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2247defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2248defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2249defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2250defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2251defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2252defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2253defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2254defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2255defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2256defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2257defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2258defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2259defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2260defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2261defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2262defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2263defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2264defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2265defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2266defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2267defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2268 2269defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2270defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2271defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2272defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2273defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2274defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2275defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2276defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2277defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2278defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2279defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2280defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2281defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2282defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2283defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2284defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2285defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2286defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2287defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2288defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2289defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2290defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2291defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2292defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2293defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2294defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2295defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2296defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2297defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2298// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2299defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2300defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2301defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2302 2303defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2304 2305defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2306defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2307def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2308 2309//===----------------------------------------------------------------------===// 2310// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2311//===----------------------------------------------------------------------===// 2312 2313class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2314 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2315 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2316 let Inst{12} = ps.offen; 2317 let Inst{13} = ps.idxen; 2318 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2319 let Inst{18-16} = op; 2320 let Inst{31-26} = 0x3a; //encoding 2321 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2322 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2323 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2324 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2325 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2326 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2327} 2328 2329//===----------------------------------------------------------------------===// 2330// MTBUF - GFX10. 2331//===----------------------------------------------------------------------===// 2332 2333class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2334 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2335 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2336 let Inst{25-19} = format; 2337 let Inst{53} = op{3}; 2338} 2339 2340let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2341 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2342 def _BOTHEN_gfx10 : 2343 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2344 def _IDXEN_gfx10 : 2345 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2346 def _OFFEN_gfx10 : 2347 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2348 def _OFFSET_gfx10 : 2349 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2350 } 2351} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2352 2353defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2354defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2355defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2356defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2357defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2358defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2359defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2360defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2361 2362//===----------------------------------------------------------------------===// 2363// MTBUF - GFX6, GFX7, GFX10. 2364//===----------------------------------------------------------------------===// 2365 2366class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2367 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2368 let Inst{15} = ps.addr64; 2369 let Inst{22-19} = dfmt; 2370 let Inst{25-23} = nfmt; 2371} 2372 2373let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2374 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2375 def _ADDR64_gfx6_gfx7 : 2376 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2377 def _BOTHEN_gfx6_gfx7 : 2378 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2379 def _IDXEN_gfx6_gfx7 : 2380 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2381 def _OFFEN_gfx6_gfx7 : 2382 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2383 def _OFFSET_gfx6_gfx7 : 2384 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2385 } 2386} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2387 2388multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2389 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2390 2391defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2392defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2393defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2394defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2395defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2396defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2397defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2398defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2399 2400//===----------------------------------------------------------------------===// 2401// GFX8, GFX9 (VI). 2402//===----------------------------------------------------------------------===// 2403 2404class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2405 bit has_sccb = ps.has_sccb> : 2406 MUBUF_Real<ps>, 2407 Enc64, 2408 SIMCInstr<ps.PseudoInstr, Enc>, 2409 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2410 !if(ps.IsAtomicRet, 1, ?))> { 2411 2412 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2413 let Inst{12} = ps.offen; 2414 let Inst{13} = ps.idxen; 2415 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2416 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2417 let Inst{16} = ps.lds; 2418 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2419 let Inst{24-18} = op; 2420 let Inst{31-26} = 0x38; //encoding 2421 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2422 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2423 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2424 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2425} 2426 2427class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2428 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2429 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2430 let DecoderNamespace = "GFX8"; 2431 2432 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2433} 2434 2435class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2436 bit has_sccb = ps.has_sccb> : 2437 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2438 let AssemblerPredicate = isGFX90APlus; 2439 let DecoderNamespace = "GFX90A"; 2440 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2441 !subst("$tfe", "", ps.AsmOperands)); 2442 2443 let Inst{55} = acc; 2444} 2445 2446class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2447 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2448 let AssemblerPredicate = isGFX940Plus; 2449 let DecoderNamespace = "GFX9"; 2450 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2451 2452 let Inst{55} = acc; 2453} 2454 2455multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2456 def _vi : MUBUF_Real_vi<op, ps>; 2457 2458 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2459 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2460 2461 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2462 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2463 let SubtargetPredicate = isGFX90AOnly; 2464 let AssemblerPredicate = isGFX90AOnly; 2465 } 2466 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2467 } 2468} 2469 2470multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2471 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2472 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2473 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2474 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2475} 2476 2477multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2478 2479 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2480 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2481 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2482 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2483 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2484 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2485 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2486 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2487 2488 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2489 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2490 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2491 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2492 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2493 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2494 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2495 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2496 2497 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2498 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2499 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2500 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2501 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2502 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2503 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2504 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2505 2506 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2507 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2508 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2509 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2510 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2511 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2512 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2513 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2514} 2515 2516class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2517 MUBUF_Real<ps>, 2518 Enc64, 2519 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2520 let AssemblerPredicate=HasUnpackedD16VMem; 2521 let DecoderNamespace="GFX80_UNPACKED"; 2522 2523 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2524 let Inst{12} = ps.offen; 2525 let Inst{13} = ps.idxen; 2526 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2527 let Inst{16} = ps.lds; 2528 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2529 let Inst{24-18} = op; 2530 let Inst{31-26} = 0x38; //encoding 2531 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2532 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2533 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2534 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2535 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2536} 2537 2538multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2539 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2540 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2541 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2542 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2543} 2544 2545multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2546 MUBUF_Real_AllAddr_vi<op> { 2547 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2548 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2549 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2550 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2551} 2552 2553defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2554defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2555defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2556defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2557defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2558defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2559defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2560defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2561let SubtargetPredicate = HasUnpackedD16VMem in { 2562 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2563 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2564 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2565 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2566 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2567 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2568 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2569 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2570} // End HasUnpackedD16VMem. 2571let SubtargetPredicate = HasPackedD16VMem in { 2572 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2573 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2574 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2575 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2576 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2577 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2578 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2579 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2580} // End HasPackedD16VMem. 2581defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2582defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2583defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2584defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2585defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2586defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2587defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2588defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2589defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2590defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2591defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2592defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2593defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2594defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2595defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2596defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2597 2598defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2599defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2600defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2601defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2602defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2603defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2604 2605defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2606defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2607 2608defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2609defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2610defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2611defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2612defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2613defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2614defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2615defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2616defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2617defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2618defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2619defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2620defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2621 2622defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2623defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2624defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2625defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2626defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2627defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2628defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2629defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2630defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2631defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2632defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2633defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2634defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2635 2636defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2637 2638let AssemblerPredicate = isGFX8GFX9 in { 2639def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2640def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2641} // End AssemblerPredicate = isGFX8GFX9 2642 2643let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2644 2645defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2646defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2647 2648} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2649 2650let SubtargetPredicate = isGFX90APlus in { 2651 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2652 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2653 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2654} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2655 2656def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2657 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2658 let AssemblerPredicate = isGFX90AOnly; 2659 let SubtargetPredicate = isGFX90AOnly; 2660} 2661def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2662 2663let SubtargetPredicate = isGFX940Plus in { 2664def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2665def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2666} 2667 2668class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2669 MTBUF_Real<ps>, 2670 Enc64, 2671 SIMCInstr<ps.PseudoInstr, Enc> { 2672 2673 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2674 let Inst{12} = ps.offen; 2675 let Inst{13} = ps.idxen; 2676 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2677 let Inst{18-15} = op; 2678 let Inst{22-19} = dfmt; 2679 let Inst{25-23} = nfmt; 2680 let Inst{31-26} = 0x3a; //encoding 2681 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2682 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2683 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2684 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2685 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2686 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2687 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2688} 2689 2690class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2691 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2692 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2693 let DecoderNamespace = "GFX8"; 2694 2695 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2696} 2697 2698class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2699 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2700 let AssemblerPredicate = isGFX90APlus; 2701 let DecoderNamespace = "GFX90A"; 2702 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2703 2704 let Inst{55} = acc; 2705} 2706 2707multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2708 def _vi : MTBUF_Real_vi<op, ps>; 2709 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2710} 2711 2712multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2713 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2714 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2715 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2716 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2717} 2718 2719class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2720 MTBUF_Real<ps>, 2721 Enc64, 2722 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2723 let AssemblerPredicate=HasUnpackedD16VMem; 2724 let DecoderNamespace="GFX80_UNPACKED"; 2725 2726 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2727 let Inst{12} = ps.offen; 2728 let Inst{13} = ps.idxen; 2729 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2730 let Inst{18-15} = op; 2731 let Inst{22-19} = dfmt; 2732 let Inst{25-23} = nfmt; 2733 let Inst{31-26} = 0x3a; //encoding 2734 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2735 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2736 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2737 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2738 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2739 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2740} 2741 2742multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2743 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2744 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2745 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2746 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2747} 2748 2749defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2750defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2751defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2752defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2753defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2754defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2755defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2756defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2757let SubtargetPredicate = HasUnpackedD16VMem in { 2758 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2759 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2760 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2761 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2762 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2763 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2764 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2765 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2766} // End HasUnpackedD16VMem. 2767let SubtargetPredicate = HasPackedD16VMem in { 2768 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2769 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2770 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2771 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2772 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2773 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2774 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2775 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2776} // End HasUnpackedD16VMem. 2777 2778def MUBUFInfoTable : GenericTable { 2779 let FilterClass = "MUBUF_Pseudo"; 2780 let CppTypeName = "MUBUFInfo"; 2781 let Fields = [ 2782 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2783 "IsBufferInv" 2784 ]; 2785 2786 let PrimaryKey = ["Opcode"]; 2787 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2788} 2789 2790def getMUBUFInfoFromOpcode : SearchIndex { 2791 let Table = MUBUFInfoTable; 2792 let Key = ["Opcode"]; 2793} 2794 2795def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2796 let Table = MUBUFInfoTable; 2797 let Key = ["BaseOpcode", "elements"]; 2798} 2799 2800def MTBUFInfoTable : GenericTable { 2801 let FilterClass = "MTBUF_Pseudo"; 2802 let CppTypeName = "MTBUFInfo"; 2803 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2804 2805 let PrimaryKey = ["Opcode"]; 2806 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2807} 2808 2809def getMTBUFInfoFromOpcode : SearchIndex { 2810 let Table = MTBUFInfoTable; 2811 let Key = ["Opcode"]; 2812} 2813 2814def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2815 let Table = MTBUFInfoTable; 2816 let Key = ["BaseOpcode", "elements"]; 2817} 2818