1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 10, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 9, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108 bits<1> has_sccb = 1; 109 bits<1> sccb_value = 0; 110} 111 112class MTBUF_Real <MTBUF_Pseudo ps> : 113 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 114 115 let isPseudo = 0; 116 let isCodeGenOnly = 0; 117 118 // copy relevant pseudo op flags 119 let UseNamedOperandTable = ps.UseNamedOperandTable; 120 let SubtargetPredicate = ps.SubtargetPredicate; 121 let AsmMatchConverter = ps.AsmMatchConverter; 122 let Constraints = ps.Constraints; 123 let DisableEncoding = ps.DisableEncoding; 124 let TSFlags = ps.TSFlags; 125 126 bits<12> offset; 127 bits<1> glc; 128 bits<1> dlc; 129 bits<7> format; 130 bits<8> vaddr; 131 bits<10> vdata; 132 bits<7> srsrc; 133 bits<1> slc; 134 bits<1> tfe; 135 bits<8> soffset; 136 137 bits<4> dfmt = format{3-0}; 138 bits<3> nfmt = format{6-4}; 139 140 bits<1> sccb; 141 // GFX90A+ only: instruction uses AccVGPR for data 142 // Bit superceedes tfe. 143 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 144} 145 146class getMTBUFInsDA<list<RegisterClass> vdataList, 147 list<RegisterClass> vaddrList=[]> { 148 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 149 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 150 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 151 dag InsNoData = !if(!empty(vaddrList), 152 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 153 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz, SCCB_0:$sccb), 154 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 155 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz, SCCB_0:$sccb) 156 ); 157 dag InsData = !if(!empty(vaddrList), 158 (ins vdata_op:$vdata, SReg_128:$srsrc, 159 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 160 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz, SCCB_0:$sccb), 161 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 162 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 163 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz, SCCB_0:$sccb) 164 ); 165 dag ret = !if(!empty(vdataList), InsNoData, InsData); 166} 167 168class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 169 dag ret = 170 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 171 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 172 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 173 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 174 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 175 (ins)))))); 176} 177 178class getMTBUFAsmOps<int addrKind> { 179 string Pfx = 180 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 181 !if(!eq(addrKind, BUFAddrKind.OffEn), 182 "$vaddr, $srsrc,$format $soffset offen", 183 !if(!eq(addrKind, BUFAddrKind.IdxEn), 184 "$vaddr, $srsrc,$format $soffset idxen", 185 !if(!eq(addrKind, BUFAddrKind.BothEn), 186 "$vaddr, $srsrc,$format $soffset idxen offen", 187 !if(!eq(addrKind, BUFAddrKind.Addr64), 188 "$vaddr, $srsrc,$format $soffset addr64", 189 ""))))); 190 string ret = Pfx # "$offset"; 191} 192 193class MTBUF_SetupAddr<int addrKind> { 194 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 195 !eq(addrKind, BUFAddrKind.BothEn)); 196 197 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 198 !eq(addrKind, BUFAddrKind.BothEn)); 199 200 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 201 202 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 203} 204 205class MTBUF_Load_Pseudo <string opName, 206 int addrKind, 207 RegisterClass vdataClass, 208 int elems, 209 list<dag> pattern=[], 210 // Workaround bug bz30254 211 int addrKindCopy = addrKind> 212 : MTBUF_Pseudo<opName, 213 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 214 getMTBUFIns<addrKindCopy>.ret, 215 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz$sccb", 216 pattern>, 217 MTBUF_SetupAddr<addrKindCopy> { 218 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 219 let mayLoad = 1; 220 let mayStore = 0; 221 let elements = elems; 222} 223 224multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 225 int elems, ValueType load_vt = i32, 226 SDPatternOperator ld = null_frag> { 227 228 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 229 [(set load_vt:$vdata, 230 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 231 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 232 MTBUFAddr64Table<0, NAME>; 233 234 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 235 [(set load_vt:$vdata, 236 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 237 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 238 MTBUFAddr64Table<1, NAME>; 239 240 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 241 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 242 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 243 244 let DisableWQM = 1 in { 245 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 246 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 247 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 248 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 249 } 250} 251 252class MTBUF_Store_Pseudo <string opName, 253 int addrKind, 254 RegisterClass vdataClass, 255 int elems, 256 list<dag> pattern=[], 257 // Workaround bug bz30254 258 int addrKindCopy = addrKind, 259 RegisterClass vdataClassCopy = vdataClass> 260 : MTBUF_Pseudo<opName, 261 (outs), 262 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 263 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz$sccb", 264 pattern>, 265 MTBUF_SetupAddr<addrKindCopy> { 266 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 267 let mayLoad = 0; 268 let mayStore = 1; 269 let elements = elems; 270} 271 272multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 273 int elems, ValueType store_vt = i32, 274 SDPatternOperator st = null_frag> { 275 276 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 277 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 278 i16:$offset, i8:$format, i1:$glc, 279 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 280 MTBUFAddr64Table<0, NAME>; 281 282 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 283 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 284 i16:$offset, i8:$format, i1:$glc, 285 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 286 MTBUFAddr64Table<1, NAME>; 287 288 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 289 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 290 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 291 292 let DisableWQM = 1 in { 293 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 294 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 295 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 296 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 297 } 298} 299 300 301//===----------------------------------------------------------------------===// 302// MUBUF classes 303//===----------------------------------------------------------------------===// 304 305class MUBUFGetBaseOpcode<string Op> { 306 string ret = !subst("DWORDX2", "DWORD", 307 !subst("DWORDX3", "DWORD", 308 !subst("DWORDX4", "DWORD", Op))); 309} 310 311class MUBUF_Pseudo <string opName, dag outs, dag ins, 312 string asmOps, list<dag> pattern=[]> : 313 InstSI<outs, ins, "", pattern>, 314 SIMCInstr<opName, SIEncodingFamily.NONE> { 315 316 let isPseudo = 1; 317 let isCodeGenOnly = 1; 318 let Size = 8; 319 let UseNamedOperandTable = 1; 320 321 string Mnemonic = opName; 322 string AsmOperands = asmOps; 323 324 Instruction Opcode = !cast<Instruction>(NAME); 325 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 326 327 let VM_CNT = 1; 328 let EXP_CNT = 1; 329 let MUBUF = 1; 330 let Uses = [EXEC]; 331 let hasSideEffects = 0; 332 let SchedRW = [WriteVMEM]; 333 334 let AsmMatchConverter = "cvtMubuf"; 335 336 bits<1> offen = 0; 337 bits<1> idxen = 0; 338 bits<1> addr64 = 0; 339 bits<1> lds = 0; 340 bits<1> has_vdata = 1; 341 bits<1> has_vaddr = 1; 342 bits<1> has_glc = 1; 343 bits<1> has_dlc = 1; 344 bits<1> glc_value = 0; // the value for glc if no such operand 345 bits<1> dlc_value = 0; // the value for dlc if no such operand 346 bits<1> has_srsrc = 1; 347 bits<1> has_soffset = 1; 348 bits<1> has_offset = 1; 349 bits<1> has_slc = 1; 350 bits<1> has_tfe = 1; 351 bits<4> elements = 0; 352 bits<1> has_sccb = 1; 353 bits<1> sccb_value = 0; 354} 355 356class MUBUF_Real <MUBUF_Pseudo ps> : 357 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 358 359 let isPseudo = 0; 360 let isCodeGenOnly = 0; 361 362 // copy relevant pseudo op flags 363 let SubtargetPredicate = ps.SubtargetPredicate; 364 let AsmMatchConverter = ps.AsmMatchConverter; 365 let OtherPredicates = ps.OtherPredicates; 366 let Constraints = ps.Constraints; 367 let DisableEncoding = ps.DisableEncoding; 368 let TSFlags = ps.TSFlags; 369 let UseNamedOperandTable = ps.UseNamedOperandTable; 370 371 bits<12> offset; 372 bits<1> glc; 373 bits<1> dlc; 374 bits<8> vaddr; 375 bits<10> vdata; 376 bits<7> srsrc; 377 bits<1> slc; 378 bits<1> tfe; 379 bits<8> soffset; 380 381 bits<1> sccb; 382 // GFX90A+ only: instruction uses AccVGPR for data 383 // Bit superceedes tfe. 384 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 385} 386 387 388// For cache invalidation instructions. 389class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 390 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 391 392 let AsmMatchConverter = ""; 393 394 let hasSideEffects = 1; 395 let mayLoad = 0; 396 let mayStore = 0; 397 398 // Set everything to 0. 399 let offen = 0; 400 let idxen = 0; 401 let addr64 = 0; 402 let has_vdata = 0; 403 let has_vaddr = 0; 404 let has_glc = 0; 405 let has_dlc = 0; 406 let glc_value = 0; 407 let dlc_value = 0; 408 let has_srsrc = 0; 409 let has_soffset = 0; 410 let has_offset = 0; 411 let has_slc = 0; 412 let has_tfe = 0; 413 let has_sccb = 0; 414 let sccb_value = 0; 415} 416 417class getMUBUFInsDA<list<RegisterClass> vdataList, 418 list<RegisterClass> vaddrList=[], 419 bit isLds = 0> { 420 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 421 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 422 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 423 dag InsNoData = !if(!empty(vaddrList), 424 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 425 offset:$offset, GLC:$glc, SLC:$slc), 426 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 427 offset:$offset, GLC:$glc, SLC:$slc) 428 ); 429 dag InsData = !if(!empty(vaddrList), 430 (ins vdata_op:$vdata, SReg_128:$srsrc, 431 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 432 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 433 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 434 ); 435 dag ret = !con( 436 !if(!empty(vdataList), InsNoData, InsData), 437 !if(isLds, (ins DLC:$dlc, SWZ:$swz, SCCB_0:$sccb), (ins TFE:$tfe, DLC:$dlc, SWZ:$swz, SCCB_0:$sccb)) 438 ); 439} 440 441class getMUBUFElements<ValueType vt> { 442 // eq does not support ValueType for some reason. 443 string vtAsStr = !cast<string>(vt); 444 445 int ret = 446 !if(!eq(vtAsStr, "f16"), 1, 447 !if(!eq(vtAsStr, "v2f16"), 2, 448 !if(!eq(vtAsStr, "v3f16"), 3, 449 !if(!eq(vtAsStr, "v4f16"), 4, 450 !if(!eq(vt.Size, 32), 1, 451 !if(!eq(vt.Size, 64), 2, 452 !if(!eq(vt.Size, 96), 3, 453 !if(!eq(vt.Size, 128), 4, 0) 454 ) 455 ) 456 ) 457 ) 458 ) 459 ) 460 ); 461} 462 463class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 464 dag ret = 465 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 466 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 467 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 468 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 469 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 470 (ins)))))); 471} 472 473class getMUBUFAsmOps<int addrKind> { 474 string Pfx = 475 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 476 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 477 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 478 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 479 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 480 ""))))); 481 string ret = Pfx # "$offset"; 482} 483 484class MUBUF_SetupAddr<int addrKind> { 485 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 486 !eq(addrKind, BUFAddrKind.BothEn)); 487 488 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 489 !eq(addrKind, BUFAddrKind.BothEn)); 490 491 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 492 493 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 494} 495 496class MUBUF_Load_Pseudo <string opName, 497 int addrKind, 498 ValueType vdata_vt, 499 bit HasTiedDest = 0, 500 bit isLds = 0, 501 list<dag> pattern=[], 502 // Workaround bug bz30254 503 int addrKindCopy = addrKind, 504 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 505 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 506 : MUBUF_Pseudo<opName, 507 (outs vdata_op:$vdata), 508 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 509 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 510 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 511 !if(isLds, " lds", "$tfe") # "$dlc$swz$sccb", 512 pattern>, 513 MUBUF_SetupAddr<addrKindCopy> { 514 let PseudoInstr = opName # !if(isLds, "_lds", "") # 515 "_" # getAddrName<addrKindCopy>.ret; 516 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 517 518 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 519 let mayLoad = 1; 520 let mayStore = 0; 521 let maybeAtomic = 1; 522 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 523 let has_tfe = !not(isLds); 524 let lds = isLds; 525 let elements = getMUBUFElements<vdata_vt>.ret; 526} 527 528class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 529 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 530 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 531>; 532 533class MUBUF_Addr64_Load_Pat <Instruction inst, 534 ValueType load_vt = i32, 535 SDPatternOperator ld = null_frag> : Pat < 536 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 537 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 538>; 539 540multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 541 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 542 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 543} 544 545 546// FIXME: tfe can't be an operand because it requires a separate 547// opcode because it needs an N+1 register class dest register. 548multiclass MUBUF_Pseudo_Loads<string opName, 549 ValueType load_vt = i32, 550 SDPatternOperator ld = null_frag, 551 bit TiedDest = 0, 552 bit isLds = 0> { 553 554 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 555 556 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 557 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 558 559 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 560 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 561 562 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 563 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 564 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 565 566 let DisableWQM = 1 in { 567 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 568 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 569 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 570 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 571 } 572} 573 574multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 575 SDPatternOperator ld_nolds = null_frag, 576 SDPatternOperator ld_lds = null_frag> { 577 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 578 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 579} 580 581class MUBUF_Store_Pseudo <string opName, 582 int addrKind, 583 ValueType store_vt, 584 list<dag> pattern=[], 585 // Workaround bug bz30254 586 int addrKindCopy = addrKind> 587 : MUBUF_Pseudo<opName, 588 (outs), 589 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 590 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz$sccb", 591 pattern>, 592 MUBUF_SetupAddr<addrKindCopy> { 593 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 594 let mayLoad = 0; 595 let mayStore = 1; 596 let maybeAtomic = 1; 597 let elements = getMUBUFElements<store_vt>.ret; 598} 599 600multiclass MUBUF_Pseudo_Stores<string opName, 601 ValueType store_vt = i32, 602 SDPatternOperator st = null_frag> { 603 604 defvar legal_store_vt = !if(!eq(!cast<string>(store_vt), !cast<string>(v3f16)), v4f16, store_vt); 605 606 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 607 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 608 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 609 MUBUFAddr64Table<0, NAME>; 610 611 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 612 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 613 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 614 MUBUFAddr64Table<1, NAME>; 615 616 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 617 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 618 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 619 620 let DisableWQM = 1 in { 621 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 622 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 623 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 624 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 625 } 626} 627 628class MUBUF_Pseudo_Store_Lds<string opName> 629 : MUBUF_Pseudo<opName, 630 (outs), 631 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz, SCCB_0:$sccb), 632 " $srsrc, $soffset$offset lds$glc$slc$swz$sccb"> { 633 let mayLoad = 0; 634 let mayStore = 1; 635 let maybeAtomic = 1; 636 637 let has_vdata = 0; 638 let has_vaddr = 0; 639 let has_tfe = 0; 640 let lds = 1; 641 642 let Uses = [EXEC, M0]; 643 let AsmMatchConverter = "cvtMubufLds"; 644} 645 646class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 647 list<RegisterClass> vaddrList=[]> { 648 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 649 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 650 dag ret = !if(vdata_in, 651 !if(!empty(vaddrList), 652 (ins vdata_op:$vdata_in, 653 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC_1:$glc1, SLC:$slc), 654 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 655 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC_1:$glc1, SLC:$slc) 656 ), 657 !if(!empty(vaddrList), 658 (ins vdata_op:$vdata, 659 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 660 (ins vdata_op:$vdata, vaddrClass:$vaddr, 661 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 662 )); 663} 664 665class getMUBUFAtomicIns<int addrKind, 666 RegisterClass vdataClass, 667 bit vdata_in, 668 // Workaround bug bz30254 669 RegisterClass vdataClassCopy=vdataClass> { 670 dag ret = 671 !if(!eq(addrKind, BUFAddrKind.Offset), 672 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 673 !if(!eq(addrKind, BUFAddrKind.OffEn), 674 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 675 !if(!eq(addrKind, BUFAddrKind.IdxEn), 676 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 677 !if(!eq(addrKind, BUFAddrKind.BothEn), 678 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 679 !if(!eq(addrKind, BUFAddrKind.Addr64), 680 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 681 (ins)))))); 682} 683 684class MUBUF_Atomic_Pseudo<string opName, 685 int addrKind, 686 dag outs, 687 dag ins, 688 string asmOps, 689 list<dag> pattern=[], 690 // Workaround bug bz30254 691 int addrKindCopy = addrKind> 692 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 693 MUBUF_SetupAddr<addrKindCopy> { 694 let mayStore = 1; 695 let mayLoad = 1; 696 let hasPostISelHook = 1; 697 let hasSideEffects = 1; 698 let DisableWQM = 1; 699 let has_glc = 0; 700 let has_dlc = 0; 701 let has_tfe = 0; 702 let has_sccb = 0; 703 let maybeAtomic = 1; 704} 705 706class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 707 RegisterClass vdataClass, 708 list<dag> pattern=[], 709 // Workaround bug bz30254 710 int addrKindCopy = addrKind, 711 RegisterClass vdataClassCopy = vdataClass> 712 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 713 (outs), 714 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 715 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 716 pattern>, 717 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 718 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 719 let glc_value = 0; 720 let dlc_value = 0; 721 let sccb_value = 0; 722 let IsAtomicNoRet = 1; 723 let AsmMatchConverter = "cvtMubufAtomic"; 724} 725 726class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 727 RegisterClass vdataClass, 728 list<dag> pattern=[], 729 // Workaround bug bz30254 730 int addrKindCopy = addrKind, 731 RegisterClass vdataClassCopy = vdataClass, 732 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 733 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 734 (outs vdata_op:$vdata), 735 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 736 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc1$slc", 737 pattern>, 738 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 739 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 740 let glc_value = 1; 741 let dlc_value = 0; 742 let sccb_value = 0; 743 let IsAtomicRet = 1; 744 let Constraints = "$vdata = $vdata_in"; 745 let DisableEncoding = "$vdata_in"; 746 let AsmMatchConverter = "cvtMubufAtomicReturn"; 747} 748 749multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 750 RegisterClass vdataClass, 751 ValueType vdataType, 752 SDPatternOperator atomic, 753 bit isFP = isFloatType<vdataType>.ret> { 754 let FPAtomic = isFP in 755 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 756 MUBUFAddr64Table <0, NAME>; 757 758 let FPAtomic = isFP in 759 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 760 MUBUFAddr64Table <1, NAME>; 761 762 let FPAtomic = isFP in 763 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 764 765 let FPAtomic = isFP in 766 767 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 768 769 let FPAtomic = isFP in 770 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 771} 772 773multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 774 RegisterClass vdataClass, 775 ValueType vdataType, 776 SDPatternOperator atomic, 777 bit isFP = isFloatType<vdataType>.ret> { 778 let FPAtomic = isFP in 779 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 780 [(set vdataType:$vdata, 781 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 782 vdataType:$vdata_in))]>, 783 MUBUFAddr64Table <0, NAME # "_RTN">; 784 785 let FPAtomic = isFP in 786 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 787 [(set vdataType:$vdata, 788 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 789 vdataType:$vdata_in))]>, 790 MUBUFAddr64Table <1, NAME # "_RTN">; 791 792 let FPAtomic = isFP in 793 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 794 795 let FPAtomic = isFP in 796 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 797 798 let FPAtomic = isFP in 799 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 800} 801 802multiclass MUBUF_Pseudo_Atomics <string opName, 803 RegisterClass vdataClass, 804 ValueType vdataType, 805 SDPatternOperator atomic> : 806 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 807 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 808 809 810//===----------------------------------------------------------------------===// 811// MUBUF Instructions 812//===----------------------------------------------------------------------===// 813 814defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 815 "buffer_load_format_x", f32 816>; 817defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 818 "buffer_load_format_xy", v2f32 819>; 820defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 821 "buffer_load_format_xyz", v3f32 822>; 823defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 824 "buffer_load_format_xyzw", v4f32 825>; 826defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 827 "buffer_store_format_x", f32 828>; 829defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 830 "buffer_store_format_xy", v2f32 831>; 832defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 833 "buffer_store_format_xyz", v3f32 834>; 835defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 836 "buffer_store_format_xyzw", v4f32 837>; 838 839let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 840 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_x", i32 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xy", v2i32 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xyz", v3i32 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyzw", v4i32 851 >; 852 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_x", i32 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xy", v2i32 857 >; 858 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 859 "buffer_store_format_d16_xyz", v3i32 860 >; 861 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 862 "buffer_store_format_d16_xyzw", v4i32 863 >; 864} // End HasUnpackedD16VMem. 865 866let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 867 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 868 "buffer_load_format_d16_x", f16 869 >; 870 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 871 "buffer_load_format_d16_xy", v2f16 872 >; 873 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 874 "buffer_load_format_d16_xyz", v3f16 875 >; 876 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 877 "buffer_load_format_d16_xyzw", v4f16 878 >; 879 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 880 "buffer_store_format_d16_x", f16 881 >; 882 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 883 "buffer_store_format_d16_xy", v2f16 884 >; 885 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 886 "buffer_store_format_d16_xyz", v3f16 887 >; 888 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 889 "buffer_store_format_d16_xyzw", v4f16 890 >; 891} // End HasPackedD16VMem. 892 893defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 894 "buffer_load_ubyte", i32 895>; 896defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 897 "buffer_load_sbyte", i32 898>; 899defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 900 "buffer_load_ushort", i32 901>; 902defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 903 "buffer_load_sshort", i32 904>; 905defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 906 "buffer_load_dword", i32 907>; 908defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 909 "buffer_load_dwordx2", v2i32 910>; 911defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 912 "buffer_load_dwordx3", v3i32 913>; 914defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 915 "buffer_load_dwordx4", v4i32 916>; 917 918defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 919defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 920defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 921defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 922defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 923defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 924defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 925defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 926defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 927defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 928 929// This is not described in AMD documentation, 930// but 'lds' versions of these opcodes are available 931// in at least GFX8+ chips. See Bug 37653. 932let SubtargetPredicate = isGFX8GFX9 in { 933defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 934 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 935>; 936defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 937 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 938>; 939defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 940 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 941>; 942} 943 944defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 945 "buffer_store_byte", i32, truncstorei8_global 946>; 947defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 948 "buffer_store_short", i32, truncstorei16_global 949>; 950defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 951 "buffer_store_dword", i32, store_global 952>; 953defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 954 "buffer_store_dwordx2", v2i32, store_global 955>; 956defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 957 "buffer_store_dwordx3", v3i32, store_global 958>; 959defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 960 "buffer_store_dwordx4", v4i32, store_global 961>; 962defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 963 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 964>; 965defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 966 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 967>; 968defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 969 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 970>; 971defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 972 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 973>; 974defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 975 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 976>; 977defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 978 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 979>; 980defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 981 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 982>; 983defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 984 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 985>; 986defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 987 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 988>; 989defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 990 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 991>; 992defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 993 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 994>; 995defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 996 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 997>; 998defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 999 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 1000>; 1001defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1002 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 1003>; 1004defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1005 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 1006>; 1007defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1008 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 1009>; 1010defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1011 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 1012>; 1013defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1014 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 1015>; 1016defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1017 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 1018>; 1019defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1020 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 1021>; 1022defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1023 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 1024>; 1025defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1026 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 1027>; 1028defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1029 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 1030>; 1031defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1032 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 1033>; 1034defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1035 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1036>; 1037defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1038 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1039>; 1040 1041let SubtargetPredicate = HasGFX10_BEncoding in 1042defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1043 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1044>; 1045 1046let SubtargetPredicate = isGFX8GFX9 in { 1047def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1048} 1049 1050let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1051/* 1052defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1053defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1054*/ 1055 1056def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1057 int_amdgcn_buffer_wbinvl1_sc>; 1058} 1059 1060let SubtargetPredicate = isGFX6GFX7GFX10 in { 1061 1062defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1063 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1064>; 1065defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1066 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1067>; 1068defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1069 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1070>; 1071defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1072 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1073>; 1074defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1075 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1076>; 1077defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1078 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1079>; 1080 1081} 1082 1083let SubtargetPredicate = HasD16LoadStore in { 1084 1085defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1086 "buffer_load_ubyte_d16", i32, null_frag, 1 1087>; 1088 1089defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1090 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1091>; 1092 1093defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1094 "buffer_load_sbyte_d16", i32, null_frag, 1 1095>; 1096 1097defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1098 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1099>; 1100 1101defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1102 "buffer_load_short_d16", i32, null_frag, 1 1103>; 1104 1105defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1106 "buffer_load_short_d16_hi", i32, null_frag, 1 1107>; 1108 1109defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1110 "buffer_store_byte_d16_hi", i32 1111>; 1112 1113defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1114 "buffer_store_short_d16_hi", i32 1115>; 1116 1117defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1118 "buffer_load_format_d16_hi_x", i32 1119>; 1120defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1121 "buffer_store_format_d16_hi_x", i32 1122>; 1123 1124} // End HasD16LoadStore 1125 1126def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1127 int_amdgcn_buffer_wbinvl1>; 1128 1129let SubtargetPredicate = HasAtomicFaddInsts in { 1130defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1131 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_noret_32 1132>; 1133defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1134 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_noret_32 1135>; 1136 1137let OtherPredicates = [isGFX90APlus] in { 1138defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN < 1139 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1140>; 1141defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1142 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1143>; 1144} 1145} // End SubtargetPredicate = HasAtomicFaddInsts 1146 1147//===----------------------------------------------------------------------===// 1148// MTBUF Instructions 1149//===----------------------------------------------------------------------===// 1150 1151defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1152defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1153defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1154defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1155defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1156defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1157defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1158defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1159 1160let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1161 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1162 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1163 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1164 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1165 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1166 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1167 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1168 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1169} // End HasUnpackedD16VMem. 1170 1171let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1172 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1173 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1174 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1175 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1176 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1177 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1178 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1179 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1180} // End HasPackedD16VMem. 1181 1182let SubtargetPredicate = isGFX7Plus in { 1183 1184//===----------------------------------------------------------------------===// 1185// Instruction definitions for CI and newer. 1186//===----------------------------------------------------------------------===// 1187 1188def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1189 int_amdgcn_buffer_wbinvl1_vol>; 1190 1191} // End let SubtargetPredicate = isGFX7Plus 1192 1193let SubtargetPredicate = isGFX90APlus in { 1194 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1195 } 1196 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1197 } 1198 1199 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 1200 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 1201 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 1202} // End SubtargetPredicate = isGFX90APlus 1203 1204let SubtargetPredicate = isGFX10Plus in { 1205 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1206 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1207} // End SubtargetPredicate = isGFX10Plus 1208 1209//===----------------------------------------------------------------------===// 1210// MUBUF Patterns 1211//===----------------------------------------------------------------------===// 1212 1213//===----------------------------------------------------------------------===// 1214// buffer_load/store_format patterns 1215//===----------------------------------------------------------------------===// 1216 1217multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1218 string opcode, ValueType memoryVt = vt> { 1219 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_load<name, memoryVt>); 1220 1221 def : GCNPat< 1222 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1223 timm:$auxiliary, 0)), 1224 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1225 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1226 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1227 >; 1228 1229 def : GCNPat< 1230 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1231 timm:$auxiliary, 0)), 1232 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1233 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1234 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1235 >; 1236 1237 def : GCNPat< 1238 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1239 timm:$auxiliary, timm)), 1240 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1241 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1242 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1243 >; 1244 1245 def : GCNPat< 1246 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1247 timm:$auxiliary, timm)), 1248 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1249 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1250 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1251 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1252 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1253 >; 1254} 1255 1256defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1257defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1258defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1259defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1260defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1261defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1262defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1263defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1264 1265let SubtargetPredicate = HasUnpackedD16VMem in { 1266 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1267 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1268 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1269 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1270 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1271 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1272} // End HasUnpackedD16VMem. 1273 1274let SubtargetPredicate = HasPackedD16VMem in { 1275 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1276 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1277 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1278 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1282 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1283 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1284} // End HasPackedD16VMem. 1285 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1288defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1289defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1291defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1292defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1293defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1294defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1295defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1296defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1297defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1298defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1299defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1300defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1301defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1302 1303multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1304 string opcode, ValueType memoryVt = vt> { 1305 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_store<name, memoryVt>); 1306 1307 def : GCNPat< 1308 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1309 timm:$auxiliary, 0), 1310 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1311 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1312 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1313 >; 1314 1315 def : GCNPat< 1316 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1317 timm:$auxiliary, 0), 1318 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1319 (as_i16timm $offset), (extract_glc $auxiliary), 1320 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1321 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1322 >; 1323 1324 def : GCNPat< 1325 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1326 timm:$auxiliary, timm), 1327 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1328 (as_i16timm $offset), (extract_glc $auxiliary), 1329 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1330 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1331 >; 1332 1333 def : GCNPat< 1334 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1335 timm:$auxiliary, timm), 1336 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1337 getVregSrcForVT<vt>.ret:$vdata, 1338 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1339 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary), 1340 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1341 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1342 >; 1343} 1344 1345defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1346defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1347defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1348defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1349defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1350defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1351defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1352defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1353 1354let SubtargetPredicate = HasUnpackedD16VMem in { 1355 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1356 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1357 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1358 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1359 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1360 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1361} // End HasUnpackedD16VMem. 1362 1363let SubtargetPredicate = HasPackedD16VMem in { 1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1365 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1366 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1367 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1368 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1369 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1370 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1371 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1372 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1373} // End HasPackedD16VMem. 1374 1375defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1376defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1377defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1378defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1379defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1380defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1381defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1382defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1383defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1384defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1385defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1386defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1387defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1388defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1389 1390//===----------------------------------------------------------------------===// 1391// buffer_atomic patterns 1392//===----------------------------------------------------------------------===// 1393 1394multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1395 string opcode> { 1396 def : GCNPat< 1397 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1398 timm:$offset, timm:$cachepolicy, 0)), 1399 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1400 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1401 (as_i16timm $offset), (extract_slc $cachepolicy)) 1402 >; 1403 1404 def : GCNPat< 1405 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1406 timm:$offset, timm:$cachepolicy, timm)), 1407 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1408 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1409 (as_i16timm $offset), (extract_slc $cachepolicy)) 1410 >; 1411 1412 def : GCNPat< 1413 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1414 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1415 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1416 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1417 (as_i16timm $offset), (extract_slc $cachepolicy)) 1418 >; 1419 1420 def : GCNPat< 1421 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1422 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1423 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1424 getVregSrcForVT<vt>.ret:$vdata_in, 1425 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1426 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1427 (extract_slc $cachepolicy)) 1428 >; 1429} 1430 1431defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1432defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">; 1433defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1434defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1435defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1436defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1437defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1438defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1439defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1440defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1441defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1442defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1443defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1444defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1445defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1446defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1447defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1448defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1449defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1450defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1451defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1452defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1453defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1454defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1455defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1456defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1457 1458class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1459 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1460 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1461 [{ return SDValue(N, 0).use_empty(); }]> { 1462 1463 let GISelPredicateCode = [{ 1464 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1465 }]; 1466} 1467 1468multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1469 string opcode> { 1470 def : GCNPat< 1471 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1472 0, i32:$soffset, timm:$offset, 1473 timm:$cachepolicy, 0), 1474 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1475 (as_i16timm $offset), (extract_slc $cachepolicy)) 1476 >; 1477 1478 def : GCNPat< 1479 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1480 0, i32:$soffset, timm:$offset, 1481 timm:$cachepolicy, timm), 1482 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1483 (as_i16timm $offset), (extract_slc $cachepolicy)) 1484 >; 1485 1486 def : GCNPat< 1487 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1488 i32:$voffset, i32:$soffset, timm:$offset, 1489 timm:$cachepolicy, 0), 1490 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1491 (as_i16timm $offset), (extract_slc $cachepolicy)) 1492 >; 1493 1494 def : GCNPat< 1495 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1496 i32:$voffset, i32:$soffset, timm:$offset, 1497 timm:$cachepolicy, timm), 1498 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1499 getVregSrcForVT<vt>.ret:$vdata_in, 1500 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1501 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)) 1502 >; 1503} 1504 1505let SubtargetPredicate = HasAtomicFaddInsts in { 1506defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1507defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1508} 1509 1510let SubtargetPredicate = isGFX90APlus in { 1511 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1512 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1513 1514 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f64, "BUFFER_ATOMIC_ADD_F64">; 1515 defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f64, "BUFFER_ATOMIC_MIN_F64">; 1516 defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f64, "BUFFER_ATOMIC_MAX_F64">; 1517} // End SubtargetPredicate = isGFX90APlus 1518 1519def : GCNPat< 1520 (SIbuffer_atomic_cmpswap 1521 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1522 timm:$offset, timm:$cachepolicy, 0), 1523 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1524 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1525 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1526 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1527 (extract_slc $cachepolicy)), VReg_64)), sub0) 1528>; 1529 1530def : GCNPat< 1531 (SIbuffer_atomic_cmpswap 1532 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1533 0, i32:$soffset, timm:$offset, 1534 timm:$cachepolicy, timm), 1535 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1536 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1537 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1538 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), VReg_64)), 1539 sub0) 1540>; 1541 1542def : GCNPat< 1543 (SIbuffer_atomic_cmpswap 1544 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1545 i32:$voffset, i32:$soffset, timm:$offset, 1546 timm:$cachepolicy, 0), 1547 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1548 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1549 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1550 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), VReg_64)), 1551 sub0) 1552>; 1553 1554def : GCNPat< 1555 (SIbuffer_atomic_cmpswap 1556 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1557 i32:$voffset, i32:$soffset, timm:$offset, 1558 timm:$cachepolicy, timm), 1559 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1560 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1561 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1562 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1563 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), VReg_64)), 1564 sub0) 1565>; 1566 1567class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1568 PatFrag constant_ld> : GCNPat < 1569 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1570 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1571 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1572 >; 1573 1574multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1575 ValueType vt, PatFrag atomic_ld> { 1576 def : GCNPat < 1577 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1578 i16:$offset, i1:$slc))), 1579 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0, 0) 1580 >; 1581 1582 def : GCNPat < 1583 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1584 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0, 0) 1585 >; 1586} 1587 1588let SubtargetPredicate = isGFX6GFX7 in { 1589def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1590def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1591def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1592def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1593def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1594def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1595 1596defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1597defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1598} // End SubtargetPredicate = isGFX6GFX7 1599 1600multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1601 PatFrag ld> { 1602 1603 def : GCNPat < 1604 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1605 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1606 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1607 >; 1608} 1609 1610let OtherPredicates = [Has16BitInsts] in { 1611 1612defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1613defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1614defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1615defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1616defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1617defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1618 1619defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1620 1621} // End OtherPredicates = [Has16BitInsts] 1622 1623multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1624 MUBUF_Pseudo InstrOffset, 1625 ValueType vt, PatFrag ld> { 1626 def : GCNPat < 1627 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1628 i32:$soffset, u16imm:$offset))), 1629 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, 0) 1630 >; 1631 1632 def : GCNPat < 1633 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1634 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, 0) 1635 >; 1636} 1637 1638// XXX - Is it possible to have a complex pattern in a PatFrag? 1639multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1640 MUBUF_Pseudo InstrOffset, 1641 ValueType vt, PatFrag ld_frag> { 1642 def : GCNPat < 1643 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1644 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1645 >; 1646 1647 def : GCNPat < 1648 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1649 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1650 >; 1651} 1652 1653let OtherPredicates = [DisableFlatScratch] in { 1654defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1655defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1656defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1657defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1658defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1659defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1660defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1661defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1662defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1663defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1664 1665foreach vt = Reg32Types.types in { 1666defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1667} 1668defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1669defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1670defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1671 1672let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1673defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1674defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1675defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1676defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1677defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1678defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1679 1680defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1681defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1682defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1683defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1684defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1685defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1686} 1687 1688} // End OtherPredicates = [DisableFlatScratch] 1689 1690multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1691 ValueType vt, PatFrag atomic_st> { 1692 // Store follows atomic op convention so address is first 1693 def : GCNPat < 1694 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1695 i16:$offset, i1:$slc), vt:$val), 1696 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0, 0) 1697 >; 1698 1699 def : GCNPat < 1700 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1701 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0, 0) 1702 >; 1703} 1704let SubtargetPredicate = isGFX6GFX7 in { 1705defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1706defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1707} // End Predicates = isGFX6GFX7 1708 1709 1710multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1711 PatFrag st> { 1712 1713 def : GCNPat < 1714 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1715 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1716 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1717 >; 1718} 1719 1720defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1721defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1722 1723multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1724 MUBUF_Pseudo InstrOffset, 1725 ValueType vt, PatFrag st, 1726 RegisterClass rc = VGPR_32> { 1727 def : GCNPat < 1728 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1729 i32:$soffset, u16imm:$offset)), 1730 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, 0) 1731 >; 1732 1733 def : GCNPat < 1734 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1735 u16imm:$offset)), 1736 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, 0) 1737 >; 1738} 1739 1740let OtherPredicates = [DisableFlatScratch] in { 1741defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1742defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1743defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1744defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1745 1746foreach vt = Reg32Types.types in { 1747defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1748} 1749 1750defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1751defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1752defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1753 1754 1755let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1756 // Hiding the extract high pattern in the PatFrag seems to not 1757 // automatically increase the complexity. 1758let AddedComplexity = 1 in { 1759defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1760defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1761} 1762} 1763} // End OtherPredicates = [DisableFlatScratch] 1764 1765//===----------------------------------------------------------------------===// 1766// MTBUF Patterns 1767//===----------------------------------------------------------------------===// 1768 1769//===----------------------------------------------------------------------===// 1770// tbuffer_load/store_format patterns 1771//===----------------------------------------------------------------------===// 1772 1773multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1774 string opcode, ValueType memoryVt = vt> { 1775 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_load<name, memoryVt>); 1776 1777 def : GCNPat< 1778 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1779 timm:$format, timm:$auxiliary, 0)), 1780 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1781 (as_i8timm $format), 1782 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1783 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1784 >; 1785 1786 def : GCNPat< 1787 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1788 timm:$format, timm:$auxiliary, timm)), 1789 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1790 (as_i8timm $format), 1791 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1792 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1793 >; 1794 1795 def : GCNPat< 1796 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1797 timm:$format, timm:$auxiliary, 0)), 1798 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1799 (as_i8timm $format), 1800 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1801 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1802 >; 1803 1804 def : GCNPat< 1805 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1806 timm:$format, timm:$auxiliary, timm)), 1807 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1808 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1809 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1810 (as_i8timm $format), 1811 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1812 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1813 >; 1814} 1815 1816defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1817defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1818defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1819defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1820defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1821defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1822defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1823defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1824 1825let SubtargetPredicate = HasUnpackedD16VMem in { 1826 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1827 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1828 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1829 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1830 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1831} // End HasUnpackedD16VMem. 1832 1833let SubtargetPredicate = HasPackedD16VMem in { 1834 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1835 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1836 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1837 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1838 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1839} // End HasPackedD16VMem. 1840 1841multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1842 string opcode, ValueType memoryVt = vt> { 1843 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_store<name, memoryVt>); 1844 1845 def : GCNPat< 1846 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1847 timm:$format, timm:$auxiliary, 0), 1848 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1849 (as_i16timm $offset), (as_i8timm $format), 1850 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1851 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1852 >; 1853 1854 def : GCNPat< 1855 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1856 timm:$format, timm:$auxiliary, timm), 1857 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1858 (as_i16timm $offset), (as_i8timm $format), 1859 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1860 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1861 >; 1862 1863 def : GCNPat< 1864 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1865 timm:$format, timm:$auxiliary, 0), 1866 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1867 (as_i16timm $offset), (as_i8timm $format), 1868 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1869 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1870 >; 1871 1872 def : GCNPat< 1873 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1874 timm:$offset, timm:$format, timm:$auxiliary, timm), 1875 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1876 getVregSrcForVT<vt>.ret:$vdata, 1877 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1878 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1879 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1880 (extract_swz $auxiliary), (extract_sccb $auxiliary)) 1881 >; 1882} 1883 1884defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1885defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1886defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1887defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1888defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1889defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1890defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1891defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1892 1893let SubtargetPredicate = HasUnpackedD16VMem in { 1894 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1895 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1896 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1897 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1898 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1899} // End HasUnpackedD16VMem. 1900 1901let SubtargetPredicate = HasPackedD16VMem in { 1902 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1903 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1904 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1905 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1906 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1907} // End HasPackedD16VMem. 1908 1909//===----------------------------------------------------------------------===// 1910// Target-specific instruction encodings. 1911//===----------------------------------------------------------------------===// 1912 1913//===----------------------------------------------------------------------===// 1914// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1915//===----------------------------------------------------------------------===// 1916 1917class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1918 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1919 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1920 let Inst{12} = ps.offen; 1921 let Inst{13} = ps.idxen; 1922 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1923 let Inst{16} = ps.lds; 1924 let Inst{24-18} = op; 1925 let Inst{31-26} = 0x38; 1926 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1927 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 1928 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1929 let Inst{54} = !if(ps.has_slc, slc, ?); 1930 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1931 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1932} 1933 1934class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1935 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1936 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1937 let Inst{25} = op{7}; 1938} 1939 1940class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1941 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1942 let Inst{15} = ps.addr64; 1943} 1944 1945//===----------------------------------------------------------------------===// 1946// MUBUF - GFX10. 1947//===----------------------------------------------------------------------===// 1948 1949let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1950 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1951 string asmName> { 1952 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1953 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1954 let AsmString = asmName # ps.AsmOperands; 1955 } 1956 } 1957 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1958 def _BOTHEN_gfx10 : 1959 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1960 def _IDXEN_gfx10 : 1961 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1962 def _OFFEN_gfx10 : 1963 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1964 def _OFFSET_gfx10 : 1965 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1966 } 1967 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1968 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1969 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1970 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1971 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1972 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1973 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1974 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1975 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1976 1977 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1978 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1979 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1980 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1981 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1982 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1983 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1984 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1985 } 1986 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1987 def _BOTHEN_RTN_gfx10 : 1988 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1989 def _IDXEN_RTN_gfx10 : 1990 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1991 def _OFFEN_RTN_gfx10 : 1992 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1993 def _OFFSET_RTN_gfx10 : 1994 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1995 } 1996 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1997 MUBUF_Real_AllAddr_gfx10<op>, MUBUF_Real_Atomics_RTN_gfx10<op>; 1998} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1999 2000defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2001defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2002defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2003defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2004defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2005defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2006defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2007defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2008// FIXME-GFX10: Add following instructions: 2009//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2010//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2011defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2012defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2013defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2014defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2015defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2016defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2017defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2018defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2019 2020def BUFFER_GL0_INV_gfx10 : 2021 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2022def BUFFER_GL1_INV_gfx10 : 2023 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2024 2025//===----------------------------------------------------------------------===// 2026// MUBUF - GFX6, GFX7, GFX10. 2027//===----------------------------------------------------------------------===// 2028 2029let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2030 multiclass MUBUF_Real_gfx6<bits<8> op> { 2031 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2032 } 2033} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2034 2035let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2036 multiclass MUBUF_Real_gfx7<bits<8> op> { 2037 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2038 } 2039} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2040 2041let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2042 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2043 def _ADDR64_gfx6_gfx7 : 2044 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2045 def _BOTHEN_gfx6_gfx7 : 2046 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2047 def _IDXEN_gfx6_gfx7 : 2048 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2049 def _OFFEN_gfx6_gfx7 : 2050 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2051 def _OFFSET_gfx6_gfx7 : 2052 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2053 } 2054 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2055 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2056 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2057 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2058 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2059 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2060 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2061 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2062 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2063 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2064 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2065 2066 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2067 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2068 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2069 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2070 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2071 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2072 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2073 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2074 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2075 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2076 } 2077 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 2078 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 2079 def _ADDR64_RTN_gfx6_gfx7 : 2080 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 2081 def _BOTHEN_RTN_gfx6_gfx7 : 2082 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2083 def _IDXEN_RTN_gfx6_gfx7 : 2084 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2085 def _OFFEN_RTN_gfx6_gfx7 : 2086 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2087 def _OFFSET_RTN_gfx6_gfx7 : 2088 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2089 } 2090} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2091 2092multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2093 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2094 2095multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2096 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2097 2098multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2099 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2100 2101// FIXME-GFX6: Following instructions are available only on GFX6. 2102//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2103//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2104 2105defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2106defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2107defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2108defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2109defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2110defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2111defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2112defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2113defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2114defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2115defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2116defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2117defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2118defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2119defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2120defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2121defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2122defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2123defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2124defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2125defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2126defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2127 2128defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2129defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2130defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2131defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2132defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2133defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2134defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2135defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2136defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2137defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2138defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2139defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2140defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2141defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2142defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2143defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2144defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2145defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2146defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2147defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2148defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2149defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2150defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2151defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2152defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2153defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2154defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2155defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2156defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2157// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2158defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2159defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2160defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2161 2162defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2163 2164defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2165defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2166def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2167 2168//===----------------------------------------------------------------------===// 2169// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2170//===----------------------------------------------------------------------===// 2171 2172class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2173 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2174 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2175 let Inst{12} = ps.offen; 2176 let Inst{13} = ps.idxen; 2177 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2178 let Inst{18-16} = op; 2179 let Inst{31-26} = 0x3a; //encoding 2180 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2181 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2182 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2183 let Inst{54} = !if(ps.has_slc, slc, ?); 2184 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2185 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2186} 2187 2188//===----------------------------------------------------------------------===// 2189// MTBUF - GFX10. 2190//===----------------------------------------------------------------------===// 2191 2192class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2193 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2194 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2195 let Inst{25-19} = format; 2196 let Inst{53} = op{3}; 2197} 2198 2199let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2200 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2201 def _BOTHEN_gfx10 : 2202 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2203 def _IDXEN_gfx10 : 2204 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2205 def _OFFEN_gfx10 : 2206 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2207 def _OFFSET_gfx10 : 2208 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2209 } 2210} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2211 2212defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2213defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2214defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2215defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2216defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2217defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2218defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2219defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2220 2221//===----------------------------------------------------------------------===// 2222// MTBUF - GFX6, GFX7, GFX10. 2223//===----------------------------------------------------------------------===// 2224 2225class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2226 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2227 let Inst{15} = ps.addr64; 2228 let Inst{22-19} = dfmt; 2229 let Inst{25-23} = nfmt; 2230} 2231 2232let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2233 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2234 def _ADDR64_gfx6_gfx7 : 2235 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2236 def _BOTHEN_gfx6_gfx7 : 2237 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2238 def _IDXEN_gfx6_gfx7 : 2239 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2240 def _OFFEN_gfx6_gfx7 : 2241 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2242 def _OFFSET_gfx6_gfx7 : 2243 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2244 } 2245} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2246 2247multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2248 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2249 2250defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2251defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2252defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2253defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2254defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2255defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2256defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2257defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2258 2259//===----------------------------------------------------------------------===// 2260// GFX8, GFX9 (VI). 2261//===----------------------------------------------------------------------===// 2262 2263class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc> : 2264 MUBUF_Real<ps>, 2265 Enc64, 2266 SIMCInstr<ps.PseudoInstr, Enc> { 2267 2268 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2269 let Inst{12} = ps.offen; 2270 let Inst{13} = ps.idxen; 2271 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2272 let Inst{15} = !if(ps.has_sccb, sccb, ps.sccb_value); 2273 let Inst{16} = ps.lds; 2274 let Inst{17} = !if(ps.has_slc, slc, ?); 2275 let Inst{24-18} = op; 2276 let Inst{31-26} = 0x38; //encoding 2277 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2278 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2279 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2280 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2281} 2282 2283class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2284 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI> { 2285 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2286 let DecoderNamespace = "GFX8"; 2287 2288 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2289} 2290 2291class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps> : 2292 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A> { 2293 let AssemblerPredicate = isGFX90APlus; 2294 let DecoderNamespace = "GFX90A"; 2295 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2296 2297 let Inst{55} = acc; 2298} 2299 2300multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2301 def _vi : MUBUF_Real_vi<op, ps>; 2302 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2303} 2304 2305multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2306 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2307 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2308 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2309 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2310} 2311 2312multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2313 2314 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2315 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2316 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2317 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2318 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2319 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2320 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2321 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2322 2323 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2324 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2325 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2326 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2327 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2328 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2329 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2330 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2331 2332 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2333 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2334 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2335 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2336 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2337 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2338 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2339 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2340 2341 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2342 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2343 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2344 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2345 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2346 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2347 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2348 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2349} 2350 2351class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2352 MUBUF_Real<ps>, 2353 Enc64, 2354 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2355 let AssemblerPredicate=HasUnpackedD16VMem; 2356 let DecoderNamespace="GFX80_UNPACKED"; 2357 2358 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2359 let Inst{12} = ps.offen; 2360 let Inst{13} = ps.idxen; 2361 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2362 let Inst{16} = ps.lds; 2363 let Inst{17} = !if(ps.has_slc, slc, ?); 2364 let Inst{24-18} = op; 2365 let Inst{31-26} = 0x38; //encoding 2366 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2367 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2368 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2369 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2370 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2371} 2372 2373multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2374 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2375 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2376 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2377 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2378} 2379 2380multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2381 MUBUF_Real_AllAddr_vi<op> { 2382 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2383 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2384 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2385 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2386} 2387 2388defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2389defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2390defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2391defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2392defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2393defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2394defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2395defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2396let SubtargetPredicate = HasUnpackedD16VMem in { 2397 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2398 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2399 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2400 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2401 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2402 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2403 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2404 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2405} // End HasUnpackedD16VMem. 2406let SubtargetPredicate = HasPackedD16VMem in { 2407 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2408 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2409 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2410 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2411 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2412 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2413 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2414 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2415} // End HasPackedD16VMem. 2416defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2417defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2418defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2419defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2420defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2421defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2422defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2423defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2424defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2425defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2426defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2427defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2428defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2429defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2430defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2431defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2432 2433defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2434defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2435defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2436defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2437defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2438defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2439 2440defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2441defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2442 2443defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2444defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2445defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2446defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2447defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2448defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2449defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2450defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2451defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2452defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2453defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2454defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2455defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2456 2457defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2458defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2459defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2460defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2461defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2462defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2463defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2464defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2465defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2466defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2467defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2468defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2469defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2470 2471defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2472 2473let AssemblerPredicate = isGFX8GFX9 in { 2474def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2475def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2476} // End AssemblerPredicate = isGFX8GFX9 2477 2478let SubtargetPredicate = HasAtomicFaddInsts in { 2479 2480defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2481defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2482 2483} // End SubtargetPredicate = HasAtomicFaddInsts 2484 2485let SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus in { 2486 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2487 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2488 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2489} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2490 2491def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2492} 2493def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2494 2495class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2496 MTBUF_Real<ps>, 2497 Enc64, 2498 SIMCInstr<ps.PseudoInstr, Enc> { 2499 2500 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2501 let Inst{12} = ps.offen; 2502 let Inst{13} = ps.idxen; 2503 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2504 let Inst{18-15} = op; 2505 let Inst{22-19} = dfmt; 2506 let Inst{25-23} = nfmt; 2507 let Inst{31-26} = 0x3a; //encoding 2508 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2509 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2510 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2511 let Inst{53} = !if(ps.has_sccb, sccb, ps.sccb_value); 2512 let Inst{54} = !if(ps.has_slc, slc, ?); 2513 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2514} 2515 2516class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2517 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2518 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2519 let DecoderNamespace = "GFX8"; 2520 2521 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2522} 2523 2524class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2525 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2526 let AssemblerPredicate = isGFX90APlus; 2527 let DecoderNamespace = "GFX90A"; 2528 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2529 2530 let Inst{55} = acc; 2531} 2532 2533multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2534 def _vi : MTBUF_Real_vi<op, ps>; 2535 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2536} 2537 2538multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2539 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2540 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2541 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2542 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2543} 2544 2545class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2546 MTBUF_Real<ps>, 2547 Enc64, 2548 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2549 let AssemblerPredicate=HasUnpackedD16VMem; 2550 let DecoderNamespace="GFX80_UNPACKED"; 2551 2552 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2553 let Inst{12} = ps.offen; 2554 let Inst{13} = ps.idxen; 2555 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2556 let Inst{18-15} = op; 2557 let Inst{22-19} = dfmt; 2558 let Inst{25-23} = nfmt; 2559 let Inst{31-26} = 0x3a; //encoding 2560 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2561 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2562 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2563 let Inst{54} = !if(ps.has_slc, slc, ?); 2564 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2565 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2566} 2567 2568multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2569 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2570 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2571 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2572 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2573} 2574 2575defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2576defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2577defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2578defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2579defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2580defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2581defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2582defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2583let SubtargetPredicate = HasUnpackedD16VMem in { 2584 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2585 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2586 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2587 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2588 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2589 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2590 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2591 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2592} // End HasUnpackedD16VMem. 2593let SubtargetPredicate = HasPackedD16VMem in { 2594 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2595 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2596 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2597 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2598 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2599 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2600 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2601 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2602} // End HasUnpackedD16VMem. 2603 2604def MUBUFInfoTable : GenericTable { 2605 let FilterClass = "MUBUF_Pseudo"; 2606 let CppTypeName = "MUBUFInfo"; 2607 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2608 2609 let PrimaryKey = ["Opcode"]; 2610 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2611} 2612 2613def getMUBUFInfoFromOpcode : SearchIndex { 2614 let Table = MUBUFInfoTable; 2615 let Key = ["Opcode"]; 2616} 2617 2618def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2619 let Table = MUBUFInfoTable; 2620 let Key = ["BaseOpcode", "elements"]; 2621} 2622 2623def MTBUFInfoTable : GenericTable { 2624 let FilterClass = "MTBUF_Pseudo"; 2625 let CppTypeName = "MTBUFInfo"; 2626 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2627 2628 let PrimaryKey = ["Opcode"]; 2629 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2630} 2631 2632def getMTBUFInfoFromOpcode : SearchIndex { 2633 let Table = MTBUFInfoTable; 2634 let Key = ["Opcode"]; 2635} 2636 2637def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2638 let Table = MTBUFInfoTable; 2639 let Key = ["BaseOpcode", "elements"]; 2640} 2641