1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108} 109 110class MTBUF_Real <MTBUF_Pseudo ps> : 111 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 112 113 let isPseudo = 0; 114 let isCodeGenOnly = 0; 115 116 // copy relevant pseudo op flags 117 let SubtargetPredicate = ps.SubtargetPredicate; 118 let AsmMatchConverter = ps.AsmMatchConverter; 119 let Constraints = ps.Constraints; 120 let DisableEncoding = ps.DisableEncoding; 121 let TSFlags = ps.TSFlags; 122 123 bits<12> offset; 124 bits<1> glc; 125 bits<1> dlc; 126 bits<7> format; 127 bits<8> vaddr; 128 bits<8> vdata; 129 bits<7> srsrc; 130 bits<1> slc; 131 bits<1> tfe; 132 bits<8> soffset; 133 134 bits<4> dfmt = format{3-0}; 135 bits<3> nfmt = format{6-4}; 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 dag InsNoData = !if(!empty(vaddrList), 143 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 144 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 145 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 146 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 147 ); 148 dag InsData = !if(!empty(vaddrList), 149 (ins vdataClass:$vdata, SReg_128:$srsrc, 150 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 151 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 152 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 153 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 154 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 155 ); 156 dag ret = !if(!empty(vdataList), InsNoData, InsData); 157} 158 159class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 160 dag ret = 161 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 162 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 163 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 (ins)))))); 167} 168 169class getMTBUFAsmOps<int addrKind> { 170 string Pfx = 171 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 172 !if(!eq(addrKind, BUFAddrKind.OffEn), 173 "$vaddr, $srsrc, $format, $soffset offen", 174 !if(!eq(addrKind, BUFAddrKind.IdxEn), 175 "$vaddr, $srsrc, $format, $soffset idxen", 176 !if(!eq(addrKind, BUFAddrKind.BothEn), 177 "$vaddr, $srsrc, $format, $soffset idxen offen", 178 !if(!eq(addrKind, BUFAddrKind.Addr64), 179 "$vaddr, $srsrc, $format, $soffset addr64", 180 ""))))); 181 string ret = Pfx # "$offset"; 182} 183 184class MTBUF_SetupAddr<int addrKind> { 185 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 187 188 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 189 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 190 191 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 192 193 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 194} 195 196class MTBUF_Load_Pseudo <string opName, 197 int addrKind, 198 RegisterClass vdataClass, 199 int elems, 200 list<dag> pattern=[], 201 // Workaround bug bz30254 202 int addrKindCopy = addrKind> 203 : MTBUF_Pseudo<opName, 204 (outs vdataClass:$vdata), 205 getMTBUFIns<addrKindCopy>.ret, 206 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 207 pattern>, 208 MTBUF_SetupAddr<addrKindCopy> { 209 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 210 let mayLoad = 1; 211 let mayStore = 0; 212 let elements = elems; 213} 214 215multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 216 int elems, ValueType load_vt = i32, 217 SDPatternOperator ld = null_frag> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 220 [(set load_vt:$vdata, 221 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 222 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 223 MTBUFAddr64Table<0, NAME>; 224 225 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 226 [(set load_vt:$vdata, 227 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 228 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 229 MTBUFAddr64Table<1, NAME>; 230 231 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 235 let DisableWQM = 1 in { 236 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 237 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 238 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 239 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 240 } 241} 242 243class MTBUF_Store_Pseudo <string opName, 244 int addrKind, 245 RegisterClass vdataClass, 246 int elems, 247 list<dag> pattern=[], 248 // Workaround bug bz30254 249 int addrKindCopy = addrKind, 250 RegisterClass vdataClassCopy = vdataClass> 251 : MTBUF_Pseudo<opName, 252 (outs), 253 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 254 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 255 pattern>, 256 MTBUF_SetupAddr<addrKindCopy> { 257 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 258 let mayLoad = 0; 259 let mayStore = 1; 260 let elements = elems; 261} 262 263multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 264 int elems, ValueType store_vt = i32, 265 SDPatternOperator st = null_frag> { 266 267 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 268 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 269 i16:$offset, i8:$format, i1:$glc, 270 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 271 MTBUFAddr64Table<0, NAME>; 272 273 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 274 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 275 i16:$offset, i8:$format, i1:$glc, 276 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 277 MTBUFAddr64Table<1, NAME>; 278 279 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 280 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 281 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 282 283 let DisableWQM = 1 in { 284 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 285 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 286 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 287 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 288 } 289} 290 291 292//===----------------------------------------------------------------------===// 293// MUBUF classes 294//===----------------------------------------------------------------------===// 295 296class MUBUFGetBaseOpcode<string Op> { 297 string ret = !subst("DWORDX2", "DWORD", 298 !subst("DWORDX3", "DWORD", 299 !subst("DWORDX4", "DWORD", Op))); 300} 301 302class MUBUF_Pseudo <string opName, dag outs, dag ins, 303 string asmOps, list<dag> pattern=[]> : 304 InstSI<outs, ins, "", pattern>, 305 SIMCInstr<opName, SIEncodingFamily.NONE> { 306 307 let isPseudo = 1; 308 let isCodeGenOnly = 1; 309 let Size = 8; 310 let UseNamedOperandTable = 1; 311 312 string Mnemonic = opName; 313 string AsmOperands = asmOps; 314 315 Instruction Opcode = !cast<Instruction>(NAME); 316 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 317 318 let VM_CNT = 1; 319 let EXP_CNT = 1; 320 let MUBUF = 1; 321 let Uses = [EXEC]; 322 let hasSideEffects = 0; 323 let SchedRW = [WriteVMEM]; 324 325 let AsmMatchConverter = "cvtMubuf"; 326 327 bits<1> offen = 0; 328 bits<1> idxen = 0; 329 bits<1> addr64 = 0; 330 bits<1> lds = 0; 331 bits<1> has_vdata = 1; 332 bits<1> has_vaddr = 1; 333 bits<1> has_glc = 1; 334 bits<1> has_dlc = 1; 335 bits<1> glc_value = 0; // the value for glc if no such operand 336 bits<1> dlc_value = 0; // the value for dlc if no such operand 337 bits<1> has_srsrc = 1; 338 bits<1> has_soffset = 1; 339 bits<1> has_offset = 1; 340 bits<1> has_slc = 1; 341 bits<1> has_tfe = 1; 342 bits<4> elements = 0; 343} 344 345class MUBUF_Real <MUBUF_Pseudo ps> : 346 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 347 348 let isPseudo = 0; 349 let isCodeGenOnly = 0; 350 351 // copy relevant pseudo op flags 352 let SubtargetPredicate = ps.SubtargetPredicate; 353 let AsmMatchConverter = ps.AsmMatchConverter; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 358 bits<12> offset; 359 bits<1> glc; 360 bits<1> dlc; 361 bits<8> vaddr; 362 bits<8> vdata; 363 bits<7> srsrc; 364 bits<1> slc; 365 bits<1> tfe; 366 bits<8> soffset; 367} 368 369 370// For cache invalidation instructions. 371class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 372 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 373 374 let AsmMatchConverter = ""; 375 376 let hasSideEffects = 1; 377 let mayStore = 1; 378 379 // Set everything to 0. 380 let offen = 0; 381 let idxen = 0; 382 let addr64 = 0; 383 let has_vdata = 0; 384 let has_vaddr = 0; 385 let has_glc = 0; 386 let has_dlc = 0; 387 let glc_value = 0; 388 let dlc_value = 0; 389 let has_srsrc = 0; 390 let has_soffset = 0; 391 let has_offset = 0; 392 let has_slc = 0; 393 let has_tfe = 0; 394} 395 396class getMUBUFInsDA<list<RegisterClass> vdataList, 397 list<RegisterClass> vaddrList=[], 398 bit isLds = 0> { 399 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 400 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 401 dag InsNoData = !if(!empty(vaddrList), 402 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 403 offset:$offset, GLC:$glc, SLC:$slc), 404 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 405 offset:$offset, GLC:$glc, SLC:$slc) 406 ); 407 dag InsData = !if(!empty(vaddrList), 408 (ins vdataClass:$vdata, SReg_128:$srsrc, 409 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 410 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 411 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 412 ); 413 dag ret = !con( 414 !if(!empty(vdataList), InsNoData, InsData), 415 !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz)) 416 ); 417} 418 419class getMUBUFElements<ValueType vt> { 420 // eq does not support ValueType for some reason. 421 string vtAsStr = !cast<string>(vt); 422 423 int ret = 424 !if(!eq(vtAsStr, "f16"), 1, 425 !if(!eq(vtAsStr, "v2f16"), 2, 426 !if(!eq(vtAsStr, "v3f16"), 3, 427 !if(!eq(vtAsStr, "v4f16"), 4, 428 !if(!eq(vt.Size, 32), 1, 429 !if(!eq(vt.Size, 64), 2, 430 !if(!eq(vt.Size, 96), 3, 431 !if(!eq(vt.Size, 128), 4, 0) 432 ) 433 ) 434 ) 435 ) 436 ) 437 ) 438 ); 439} 440 441class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 442 dag ret = 443 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 444 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 445 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 446 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 447 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 448 (ins)))))); 449} 450 451class getMUBUFAsmOps<int addrKind> { 452 string Pfx = 453 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 454 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 455 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 456 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 457 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 458 ""))))); 459 string ret = Pfx # "$offset"; 460} 461 462class MUBUF_SetupAddr<int addrKind> { 463 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 464 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 465 466 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 467 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 468 469 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 470 471 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 472} 473 474class MUBUF_Load_Pseudo <string opName, 475 int addrKind, 476 ValueType vdata_vt, 477 bit HasTiedDest = 0, 478 bit isLds = 0, 479 list<dag> pattern=[], 480 // Workaround bug bz30254 481 int addrKindCopy = addrKind> 482 : MUBUF_Pseudo<opName, 483 (outs getVregSrcForVT<vdata_vt>.ret:$vdata), 484 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 485 !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))), 486 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 487 !if(isLds, " lds", "$tfe") # "$dlc" # "$swz", 488 pattern>, 489 MUBUF_SetupAddr<addrKindCopy> { 490 let PseudoInstr = opName # !if(isLds, "_lds", "") # 491 "_" # getAddrName<addrKindCopy>.ret; 492 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 493 494 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 495 let mayLoad = 1; 496 let mayStore = 0; 497 let maybeAtomic = 1; 498 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 499 let has_tfe = !if(isLds, 0, 1); 500 let lds = isLds; 501 let elements = getMUBUFElements<vdata_vt>.ret; 502} 503 504class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 505 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 506 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 507>; 508 509class MUBUF_Addr64_Load_Pat <Instruction inst, 510 ValueType load_vt = i32, 511 SDPatternOperator ld = null_frag> : Pat < 512 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 513 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 514>; 515 516multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 517 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 518 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 519} 520 521 522// FIXME: tfe can't be an operand because it requires a separate 523// opcode because it needs an N+1 register class dest register. 524multiclass MUBUF_Pseudo_Loads<string opName, 525 ValueType load_vt = i32, 526 SDPatternOperator ld = null_frag, 527 bit TiedDest = 0, 528 bit isLds = 0> { 529 530 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>, 531 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 532 533 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>, 534 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 535 536 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 537 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 538 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 539 540 let DisableWQM = 1 in { 541 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>; 542 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 543 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 544 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 545 } 546} 547 548multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 549 SDPatternOperator ld_nolds = null_frag, 550 SDPatternOperator ld_lds = null_frag> { 551 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 552 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 553} 554 555class MUBUF_Store_Pseudo <string opName, 556 int addrKind, 557 ValueType store_vt, 558 list<dag> pattern=[], 559 // Workaround bug bz30254 560 int addrKindCopy = addrKind> 561 : MUBUF_Pseudo<opName, 562 (outs), 563 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 564 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 565 pattern>, 566 MUBUF_SetupAddr<addrKindCopy> { 567 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 568 let mayLoad = 0; 569 let mayStore = 1; 570 let maybeAtomic = 1; 571 let elements = getMUBUFElements<store_vt>.ret; 572} 573 574multiclass MUBUF_Pseudo_Stores<string opName, 575 ValueType store_vt = i32, 576 SDPatternOperator st = null_frag> { 577 578 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, 579 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 580 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 581 MUBUFAddr64Table<0, NAME>; 582 583 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, 584 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 585 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 586 MUBUFAddr64Table<1, NAME>; 587 588 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 589 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 590 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 591 592 let DisableWQM = 1 in { 593 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>; 594 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 595 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 596 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 597 } 598} 599 600class MUBUF_Pseudo_Store_Lds<string opName> 601 : MUBUF_Pseudo<opName, 602 (outs), 603 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz), 604 " $srsrc, $soffset$offset lds$glc$slc$swz"> { 605 let mayLoad = 0; 606 let mayStore = 1; 607 let maybeAtomic = 1; 608 609 let has_vdata = 0; 610 let has_vaddr = 0; 611 let has_tfe = 0; 612 let lds = 1; 613 614 let Uses = [EXEC, M0]; 615 let AsmMatchConverter = "cvtMubufLds"; 616} 617 618class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 619 list<RegisterClass> vaddrList=[]> { 620 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 621 dag ret = !if(vdata_in, 622 !if(!empty(vaddrList), 623 (ins vdataClass:$vdata_in, 624 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 625 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 626 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 627 ), 628 !if(!empty(vaddrList), 629 (ins vdataClass:$vdata, 630 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 631 (ins vdataClass:$vdata, vaddrClass:$vaddr, 632 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 633 )); 634} 635 636class getMUBUFAtomicIns<int addrKind, 637 RegisterClass vdataClass, 638 bit vdata_in, 639 // Workaround bug bz30254 640 RegisterClass vdataClassCopy=vdataClass> { 641 dag ret = 642 !if(!eq(addrKind, BUFAddrKind.Offset), 643 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 644 !if(!eq(addrKind, BUFAddrKind.OffEn), 645 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 646 !if(!eq(addrKind, BUFAddrKind.IdxEn), 647 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 648 !if(!eq(addrKind, BUFAddrKind.BothEn), 649 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 650 !if(!eq(addrKind, BUFAddrKind.Addr64), 651 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 652 (ins)))))); 653} 654 655class MUBUF_Atomic_Pseudo<string opName, 656 int addrKind, 657 dag outs, 658 dag ins, 659 string asmOps, 660 list<dag> pattern=[], 661 // Workaround bug bz30254 662 int addrKindCopy = addrKind> 663 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 664 MUBUF_SetupAddr<addrKindCopy> { 665 let mayStore = 1; 666 let mayLoad = 1; 667 let hasPostISelHook = 1; 668 let hasSideEffects = 1; 669 let DisableWQM = 1; 670 let has_glc = 0; 671 let has_dlc = 0; 672 let has_tfe = 0; 673 let maybeAtomic = 1; 674} 675 676class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 677 RegisterClass vdataClass, 678 list<dag> pattern=[], 679 // Workaround bug bz30254 680 int addrKindCopy = addrKind, 681 RegisterClass vdataClassCopy = vdataClass> 682 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 683 (outs), 684 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 685 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 686 pattern>, 687 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 688 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 689 let glc_value = 0; 690 let dlc_value = 0; 691 let AsmMatchConverter = "cvtMubufAtomic"; 692} 693 694class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 695 RegisterClass vdataClass, 696 list<dag> pattern=[], 697 // Workaround bug bz30254 698 int addrKindCopy = addrKind, 699 RegisterClass vdataClassCopy = vdataClass> 700 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 701 (outs vdataClassCopy:$vdata), 702 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 703 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 704 pattern>, 705 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 706 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 707 let glc_value = 1; 708 let dlc_value = 0; 709 let Constraints = "$vdata = $vdata_in"; 710 let DisableEncoding = "$vdata_in"; 711 let AsmMatchConverter = "cvtMubufAtomicReturn"; 712} 713 714multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 715 RegisterClass vdataClass, 716 ValueType vdataType, 717 SDPatternOperator atomic, 718 bit isFP = isFloatType<vdataType>.ret> { 719 let FPAtomic = isFP in 720 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 721 MUBUFAddr64Table <0, NAME>; 722 723 let FPAtomic = isFP in 724 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 725 MUBUFAddr64Table <1, NAME>; 726 727 let FPAtomic = isFP in 728 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 729 730 let FPAtomic = isFP in 731 732 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 733 734 let FPAtomic = isFP in 735 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 736} 737 738multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 739 RegisterClass vdataClass, 740 ValueType vdataType, 741 SDPatternOperator atomic, 742 bit isFP = isFloatType<vdataType>.ret> { 743 let FPAtomic = isFP in 744 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 745 [(set vdataType:$vdata, 746 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 747 vdataType:$vdata_in))]>, 748 MUBUFAddr64Table <0, NAME # "_RTN">; 749 750 let FPAtomic = isFP in 751 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 752 [(set vdataType:$vdata, 753 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 754 vdataType:$vdata_in))]>, 755 MUBUFAddr64Table <1, NAME # "_RTN">; 756 757 let FPAtomic = isFP in 758 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 759 760 let FPAtomic = isFP in 761 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 762 763 let FPAtomic = isFP in 764 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 765} 766 767multiclass MUBUF_Pseudo_Atomics <string opName, 768 RegisterClass vdataClass, 769 ValueType vdataType, 770 SDPatternOperator atomic> : 771 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 772 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 773 774 775//===----------------------------------------------------------------------===// 776// MUBUF Instructions 777//===----------------------------------------------------------------------===// 778 779defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 780 "buffer_load_format_x", f32 781>; 782defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 783 "buffer_load_format_xy", v2f32 784>; 785defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 786 "buffer_load_format_xyz", v3f32 787>; 788defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 789 "buffer_load_format_xyzw", v4f32 790>; 791defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 792 "buffer_store_format_x", f32 793>; 794defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 795 "buffer_store_format_xy", v2f32 796>; 797defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 798 "buffer_store_format_xyz", v3f32 799>; 800defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 801 "buffer_store_format_xyzw", v4f32 802>; 803 804let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 805 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 806 "buffer_load_format_d16_x", i32 807 >; 808 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 809 "buffer_load_format_d16_xy", v2i32 810 >; 811 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 812 "buffer_load_format_d16_xyz", v3i32 813 >; 814 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 815 "buffer_load_format_d16_xyzw", v4i32 816 >; 817 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 818 "buffer_store_format_d16_x", i32 819 >; 820 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 821 "buffer_store_format_d16_xy", v2i32 822 >; 823 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 824 "buffer_store_format_d16_xyz", v3i32 825 >; 826 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 827 "buffer_store_format_d16_xyzw", v4i32 828 >; 829} // End HasUnpackedD16VMem. 830 831let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 832 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 833 "buffer_load_format_d16_x", f16 834 >; 835 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 836 "buffer_load_format_d16_xy", v2f16 837 >; 838 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 839 "buffer_load_format_d16_xyz", v3f16 840 >; 841 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 842 "buffer_load_format_d16_xyzw", v4f16 843 >; 844 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_x", f16 846 >; 847 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 848 "buffer_store_format_d16_xy", v2f16 849 >; 850 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 851 "buffer_store_format_d16_xyz", v3f16 852 >; 853 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 854 "buffer_store_format_d16_xyzw", v4f16 855 >; 856} // End HasPackedD16VMem. 857 858defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 859 "buffer_load_ubyte", i32 860>; 861defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 862 "buffer_load_sbyte", i32 863>; 864defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 865 "buffer_load_ushort", i32 866>; 867defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 868 "buffer_load_sshort", i32 869>; 870defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 871 "buffer_load_dword", i32 872>; 873defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 874 "buffer_load_dwordx2", v2i32 875>; 876defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 877 "buffer_load_dwordx3", v3i32 878>; 879defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 880 "buffer_load_dwordx4", v4i32 881>; 882 883defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 884defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 885defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 886defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 887defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 888defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 889defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 890defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 891defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 892defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 893 894// This is not described in AMD documentation, 895// but 'lds' versions of these opcodes are available 896// in at least GFX8+ chips. See Bug 37653. 897let SubtargetPredicate = isGFX8GFX9 in { 898defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 899 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 900>; 901defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 902 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 903>; 904defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 905 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 906>; 907} 908 909defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 910 "buffer_store_byte", i32, truncstorei8_global 911>; 912defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 913 "buffer_store_short", i32, truncstorei16_global 914>; 915defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 916 "buffer_store_dword", i32, store_global 917>; 918defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 919 "buffer_store_dwordx2", v2i32, store_global 920>; 921defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 922 "buffer_store_dwordx3", v3i32, store_global 923>; 924defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 925 "buffer_store_dwordx4", v4i32, store_global 926>; 927defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 928 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 929>; 930defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 931 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 932>; 933defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 934 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 935>; 936defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 937 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 938>; 939defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 940 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 941>; 942defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 943 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 944>; 945defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 946 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 947>; 948defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 949 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 950>; 951defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 952 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 953>; 954defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 955 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 956>; 957defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 958 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 959>; 960defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 961 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 962>; 963defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 964 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 965>; 966defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 967 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 968>; 969defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 970 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 971>; 972defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 973 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 974>; 975defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 976 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 977>; 978defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 979 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 980>; 981defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 982 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 983>; 984defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 985 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 986>; 987defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 988 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 989>; 990defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 991 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 992>; 993defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 994 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 995>; 996defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 997 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 998>; 999defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1000 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1001>; 1002defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1003 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1004>; 1005 1006let SubtargetPredicate = isGFX8GFX9 in { 1007def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1008} 1009 1010let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1011/* 1012defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1013defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1014*/ 1015 1016def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1017 int_amdgcn_buffer_wbinvl1_sc>; 1018} 1019 1020let SubtargetPredicate = isGFX6GFX7GFX10 in { 1021 1022defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1023 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1024>; 1025defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1026 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1027>; 1028defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1029 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1030>; 1031defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1032 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1033>; 1034defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1035 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1036>; 1037defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1038 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1039>; 1040 1041} 1042 1043let SubtargetPredicate = HasD16LoadStore in { 1044 1045defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1046 "buffer_load_ubyte_d16", i32, null_frag, 1 1047>; 1048 1049defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1050 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1051>; 1052 1053defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1054 "buffer_load_sbyte_d16", i32, null_frag, 1 1055>; 1056 1057defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1058 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1059>; 1060 1061defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1062 "buffer_load_short_d16", i32, null_frag, 1 1063>; 1064 1065defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1066 "buffer_load_short_d16_hi", i32, null_frag, 1 1067>; 1068 1069defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1070 "buffer_store_byte_d16_hi", i32 1071>; 1072 1073defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1074 "buffer_store_short_d16_hi", i32 1075>; 1076 1077defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1078 "buffer_load_format_d16_hi_x", i32 1079>; 1080defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1081 "buffer_store_format_d16_hi_x", i32 1082>; 1083 1084} // End HasD16LoadStore 1085 1086def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1087 int_amdgcn_buffer_wbinvl1>; 1088 1089let SubtargetPredicate = HasAtomicFaddInsts in { 1090 1091defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1092 "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret 1093>; 1094defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1095 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret 1096>; 1097 1098} // End SubtargetPredicate = HasAtomicFaddInsts 1099 1100//===----------------------------------------------------------------------===// 1101// MTBUF Instructions 1102//===----------------------------------------------------------------------===// 1103 1104defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1105defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1106defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1107defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1108defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1109defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1110defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1111defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1112 1113let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1114 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1115 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1116 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1117 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1118 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1119 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1120 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1121 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1122} // End HasUnpackedD16VMem. 1123 1124let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1125 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1126 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1127 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1128 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1129 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1130 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1131 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1132 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1133} // End HasPackedD16VMem. 1134 1135let SubtargetPredicate = isGFX7Plus in { 1136 1137//===----------------------------------------------------------------------===// 1138// Instruction definitions for CI and newer. 1139//===----------------------------------------------------------------------===// 1140 1141def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1142 int_amdgcn_buffer_wbinvl1_vol>; 1143 1144} // End let SubtargetPredicate = isGFX7Plus 1145 1146let SubtargetPredicate = isGFX10Plus in { 1147 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1148 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1149} // End SubtargetPredicate = isGFX10Plus 1150 1151//===----------------------------------------------------------------------===// 1152// MUBUF Patterns 1153//===----------------------------------------------------------------------===// 1154 1155//===----------------------------------------------------------------------===// 1156// buffer_load/store_format patterns 1157//===----------------------------------------------------------------------===// 1158 1159multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1160 string opcode> { 1161 def : GCNPat< 1162 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1163 timm:$auxiliary, 0)), 1164 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1165 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1166 (extract_swz $auxiliary)) 1167 >; 1168 1169 def : GCNPat< 1170 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1171 timm:$auxiliary, 0)), 1172 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1173 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1174 (extract_swz $auxiliary)) 1175 >; 1176 1177 def : GCNPat< 1178 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1179 timm:$auxiliary, timm)), 1180 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1181 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1182 (extract_swz $auxiliary)) 1183 >; 1184 1185 def : GCNPat< 1186 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1187 timm:$auxiliary, timm)), 1188 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1189 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1190 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1191 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1192 (extract_swz $auxiliary)) 1193 >; 1194} 1195 1196defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1197defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1198defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1199defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1200defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1201defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1202defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1203defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1204 1205let SubtargetPredicate = HasUnpackedD16VMem in { 1206 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1207 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1208 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1209 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1210 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1211} // End HasUnpackedD16VMem. 1212 1213let SubtargetPredicate = HasPackedD16VMem in { 1214 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1215 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1216 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1217 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1218 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1219 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1220 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1221} // End HasPackedD16VMem. 1222 1223defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1224defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1225defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1226defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1227defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1228defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1229defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1230defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1231defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1232defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1233defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1234defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1235defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1236defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1237defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1238defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1239 1240multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1241 string opcode> { 1242 def : GCNPat< 1243 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1244 timm:$auxiliary, 0), 1245 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1246 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1247 (extract_swz $auxiliary)) 1248 >; 1249 1250 def : GCNPat< 1251 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1252 timm:$auxiliary, 0), 1253 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1254 (as_i16timm $offset), (extract_glc $auxiliary), 1255 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1256 (extract_swz $auxiliary)) 1257 >; 1258 1259 def : GCNPat< 1260 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1261 timm:$auxiliary, timm), 1262 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1263 (as_i16timm $offset), (extract_glc $auxiliary), 1264 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1265 (extract_swz $auxiliary)) 1266 >; 1267 1268 def : GCNPat< 1269 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1270 timm:$auxiliary, timm), 1271 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1272 getVregSrcForVT<vt>.ret:$vdata, 1273 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1274 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary), 1275 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1276 (extract_swz $auxiliary)) 1277 >; 1278} 1279 1280defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1281defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1282defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1283defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1284defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1285defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1286defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1287defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1288 1289let SubtargetPredicate = HasUnpackedD16VMem in { 1290 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1291 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1292 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1293 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1294 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1295} // End HasUnpackedD16VMem. 1296 1297let SubtargetPredicate = HasPackedD16VMem in { 1298 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1299 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1300 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1301 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1302 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1303 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1304 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1305} // End HasPackedD16VMem. 1306 1307defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1308defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1309defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1310defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1311defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1312defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1313defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1314defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1315defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1316defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1317defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1318defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1319defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1320defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1321 1322//===----------------------------------------------------------------------===// 1323// buffer_atomic patterns 1324//===----------------------------------------------------------------------===// 1325 1326multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1327 string opcode> { 1328 def : GCNPat< 1329 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1330 timm:$offset, timm:$cachepolicy, 0)), 1331 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1332 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1333 (as_i16timm $offset), (extract_slc $cachepolicy)) 1334 >; 1335 1336 def : GCNPat< 1337 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1338 timm:$offset, timm:$cachepolicy, timm)), 1339 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1340 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1341 (as_i16timm $offset), (extract_slc $cachepolicy)) 1342 >; 1343 1344 def : GCNPat< 1345 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1346 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1347 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1348 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1349 (as_i16timm $offset), (extract_slc $cachepolicy)) 1350 >; 1351 1352 def : GCNPat< 1353 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1354 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1355 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1356 getVregSrcForVT<vt>.ret:$vdata_in, 1357 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1358 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1359 (extract_slc $cachepolicy)) 1360 >; 1361} 1362 1363defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1364defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1365defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1366defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1367defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1368defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1369defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1370defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1371defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1372defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1373defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1374defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1375defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1376defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1377defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1378defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1379defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1380defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1381defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1382defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1383defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1384defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1385defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1386defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1387 1388multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1389 string opcode> { 1390 def : GCNPat< 1391 (name vt:$vdata_in, v4i32:$rsrc, 0, 1392 0, i32:$soffset, timm:$offset, 1393 timm:$cachepolicy, 0), 1394 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset, 1395 (as_i16imm $offset), (extract_slc $cachepolicy)) 1396 >; 1397 1398 def : GCNPat< 1399 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1400 0, i32:$soffset, timm:$offset, 1401 timm:$cachepolicy, timm), 1402 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset, 1403 (as_i16imm $offset), (extract_slc $cachepolicy)) 1404 >; 1405 1406 def : GCNPat< 1407 (name vt:$vdata_in, v4i32:$rsrc, 0, 1408 i32:$voffset, i32:$soffset, timm:$offset, 1409 timm:$cachepolicy, 0), 1410 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset, 1411 (as_i16imm $offset), (extract_slc $cachepolicy)) 1412 >; 1413 1414 def : GCNPat< 1415 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1416 i32:$voffset, i32:$soffset, timm:$offset, 1417 timm:$cachepolicy, timm), 1418 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1419 $vdata_in, 1420 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1421 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1422 >; 1423} 1424 1425defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1426defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1427 1428def : GCNPat< 1429 (SIbuffer_atomic_cmpswap 1430 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1431 timm:$offset, timm:$cachepolicy, 0), 1432 (EXTRACT_SUBREG 1433 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1434 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1435 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1436 (extract_slc $cachepolicy)), sub0) 1437>; 1438 1439def : GCNPat< 1440 (SIbuffer_atomic_cmpswap 1441 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1442 0, i32:$soffset, timm:$offset, 1443 timm:$cachepolicy, timm), 1444 (EXTRACT_SUBREG 1445 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1446 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1447 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1448 sub0) 1449>; 1450 1451def : GCNPat< 1452 (SIbuffer_atomic_cmpswap 1453 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1454 i32:$voffset, i32:$soffset, timm:$offset, 1455 timm:$cachepolicy, 0), 1456 (EXTRACT_SUBREG 1457 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1458 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1459 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1460 sub0) 1461>; 1462 1463def : GCNPat< 1464 (SIbuffer_atomic_cmpswap 1465 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1466 i32:$voffset, i32:$soffset, timm:$offset, 1467 timm:$cachepolicy, timm), 1468 (EXTRACT_SUBREG 1469 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1470 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1471 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1472 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1473 sub0) 1474>; 1475 1476class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1477 PatFrag constant_ld> : GCNPat < 1478 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1479 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1480 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1481 >; 1482 1483multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1484 ValueType vt, PatFrag atomic_ld> { 1485 def : GCNPat < 1486 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1487 i16:$offset, i1:$slc))), 1488 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1489 >; 1490 1491 def : GCNPat < 1492 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1493 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1494 >; 1495} 1496 1497let SubtargetPredicate = isGFX6GFX7 in { 1498def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1499def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1500def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1501def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1502def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1503def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1504 1505defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1506defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1507} // End SubtargetPredicate = isGFX6GFX7 1508 1509multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1510 PatFrag ld> { 1511 1512 def : GCNPat < 1513 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1514 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1515 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1516 >; 1517} 1518 1519let OtherPredicates = [Has16BitInsts] in { 1520 1521defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1522defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1523defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1524defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1525defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1526defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1527 1528defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1529 1530} // End OtherPredicates = [Has16BitInsts] 1531 1532multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1533 MUBUF_Pseudo InstrOffset, 1534 ValueType vt, PatFrag ld> { 1535 def : GCNPat < 1536 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1537 i32:$soffset, u16imm:$offset))), 1538 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1539 >; 1540 1541 def : GCNPat < 1542 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1543 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1544 >; 1545} 1546 1547// XXX - Is it possible to have a complex pattern in a PatFrag? 1548multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1549 MUBUF_Pseudo InstrOffset, 1550 ValueType vt, PatFrag ld_frag> { 1551 def : GCNPat < 1552 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1553 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1554 >; 1555 1556 def : GCNPat < 1557 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1558 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1559 >; 1560} 1561 1562defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1563defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1564defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1565defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1566defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1567defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1568defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1569defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1570defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1571defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1572 1573foreach vt = Reg32Types.types in { 1574defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1575} 1576defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1577defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1578defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1579 1580let OtherPredicates = [D16PreservesUnusedBits] in { 1581defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1582defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1583defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1584defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1585defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1586defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1587 1588defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1589defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1590defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1591defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1592defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1593defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1594} 1595 1596multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1597 ValueType vt, PatFrag atomic_st> { 1598 // Store follows atomic op convention so address is first 1599 def : GCNPat < 1600 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1601 i16:$offset, i1:$slc), vt:$val), 1602 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1603 >; 1604 1605 def : GCNPat < 1606 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1607 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1608 >; 1609} 1610let SubtargetPredicate = isGFX6GFX7 in { 1611defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1612defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1613} // End Predicates = isGFX6GFX7 1614 1615 1616multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1617 PatFrag st> { 1618 1619 def : GCNPat < 1620 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1621 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1622 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1623 >; 1624} 1625 1626defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1627defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1628 1629multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1630 MUBUF_Pseudo InstrOffset, 1631 ValueType vt, PatFrag st, 1632 RegisterClass rc = VGPR_32> { 1633 def : GCNPat < 1634 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1635 i32:$soffset, u16imm:$offset)), 1636 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1637 >; 1638 1639 def : GCNPat < 1640 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1641 u16imm:$offset)), 1642 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1643 >; 1644} 1645 1646defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1647defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1648defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1649defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1650 1651foreach vt = Reg32Types.types in { 1652defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1653} 1654 1655defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1656defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1657defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1658 1659 1660let OtherPredicates = [D16PreservesUnusedBits] in { 1661 // Hiding the extract high pattern in the PatFrag seems to not 1662 // automatically increase the complexity. 1663let AddedComplexity = 1 in { 1664defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1665defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1666} 1667} 1668 1669//===----------------------------------------------------------------------===// 1670// MTBUF Patterns 1671//===----------------------------------------------------------------------===// 1672 1673//===----------------------------------------------------------------------===// 1674// tbuffer_load/store_format patterns 1675//===----------------------------------------------------------------------===// 1676 1677multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1678 string opcode> { 1679 def : GCNPat< 1680 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1681 timm:$format, timm:$auxiliary, 0)), 1682 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1683 (as_i8timm $format), 1684 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1685 (extract_swz $auxiliary)) 1686 >; 1687 1688 def : GCNPat< 1689 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1690 timm:$format, timm:$auxiliary, timm)), 1691 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1692 (as_i8timm $format), 1693 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1694 (extract_swz $auxiliary)) 1695 >; 1696 1697 def : GCNPat< 1698 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1699 timm:$format, timm:$auxiliary, 0)), 1700 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1701 (as_i8timm $format), 1702 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1703 (extract_swz $auxiliary)) 1704 >; 1705 1706 def : GCNPat< 1707 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1708 timm:$format, timm:$auxiliary, timm)), 1709 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1710 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1711 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1712 (as_i8timm $format), 1713 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1714 (extract_swz $auxiliary)) 1715 >; 1716} 1717 1718defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1719defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1720defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1721defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1722defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1723defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1724defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1725defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1726 1727let SubtargetPredicate = HasUnpackedD16VMem in { 1728 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1729 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1730 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1731 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1732} // End HasUnpackedD16VMem. 1733 1734let SubtargetPredicate = HasPackedD16VMem in { 1735 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1736 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1737 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1738 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1739} // End HasPackedD16VMem. 1740 1741multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1742 string opcode> { 1743 def : GCNPat< 1744 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1745 timm:$format, timm:$auxiliary, 0), 1746 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1747 (as_i16timm $offset), (as_i8timm $format), 1748 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1749 (extract_swz $auxiliary)) 1750 >; 1751 1752 def : GCNPat< 1753 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1754 timm:$format, timm:$auxiliary, timm), 1755 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1756 (as_i16timm $offset), (as_i8timm $format), 1757 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1758 (extract_swz $auxiliary)) 1759 >; 1760 1761 def : GCNPat< 1762 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1763 timm:$format, timm:$auxiliary, 0), 1764 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1765 (as_i16timm $offset), (as_i8timm $format), 1766 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1767 (extract_swz $auxiliary)) 1768 >; 1769 1770 def : GCNPat< 1771 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1772 timm:$offset, timm:$format, timm:$auxiliary, timm), 1773 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1774 getVregSrcForVT<vt>.ret:$vdata, 1775 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1776 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1777 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1778 (extract_swz $auxiliary)) 1779 >; 1780} 1781 1782defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1783defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1784defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1785defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1786defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1787defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1788defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1789defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1790 1791let SubtargetPredicate = HasUnpackedD16VMem in { 1792 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1793 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1794 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1795} // End HasUnpackedD16VMem. 1796 1797let SubtargetPredicate = HasPackedD16VMem in { 1798 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1799 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1800 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1801} // End HasPackedD16VMem. 1802 1803//===----------------------------------------------------------------------===// 1804// Target-specific instruction encodings. 1805//===----------------------------------------------------------------------===// 1806 1807//===----------------------------------------------------------------------===// 1808// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1809//===----------------------------------------------------------------------===// 1810 1811class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1812 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1813 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1814 let Inst{12} = ps.offen; 1815 let Inst{13} = ps.idxen; 1816 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1817 let Inst{16} = !if(ps.lds, 1, 0); 1818 let Inst{24-18} = op; 1819 let Inst{31-26} = 0x38; 1820 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1821 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1822 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1823 let Inst{54} = !if(ps.has_slc, slc, ?); 1824 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1825 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1826} 1827 1828class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1829 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1830 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1831 let Inst{25} = op{7}; 1832} 1833 1834class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1835 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1836 let Inst{15} = ps.addr64; 1837} 1838 1839//===----------------------------------------------------------------------===// 1840// MUBUF - GFX10. 1841//===----------------------------------------------------------------------===// 1842 1843let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1844 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1845 string asmName> { 1846 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1847 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1848 let AsmString = asmName # ps.AsmOperands; 1849 } 1850 } 1851 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1852 def _BOTHEN_gfx10 : 1853 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1854 def _IDXEN_gfx10 : 1855 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1856 def _OFFEN_gfx10 : 1857 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1858 def _OFFSET_gfx10 : 1859 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1860 } 1861 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1862 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1863 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1864 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1865 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1866 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1867 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1868 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1869 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1870 1871 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1872 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1873 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1874 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1875 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1876 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1877 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1878 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1879 } 1880 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1881 MUBUF_Real_AllAddr_gfx10<op> { 1882 def _BOTHEN_RTN_gfx10 : 1883 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1884 def _IDXEN_RTN_gfx10 : 1885 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1886 def _OFFEN_RTN_gfx10 : 1887 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1888 def _OFFSET_RTN_gfx10 : 1889 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1890 } 1891} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1892 1893defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1894defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1895defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1896defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1897defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1898defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1899defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1900defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1901// FIXME-GFX10: Add following instructions: 1902//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1903//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1904defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1905defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1906defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1907defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1908defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1909defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1910defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1911defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1912 1913def BUFFER_GL0_INV_gfx10 : 1914 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1915def BUFFER_GL1_INV_gfx10 : 1916 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1917 1918//===----------------------------------------------------------------------===// 1919// MUBUF - GFX6, GFX7, GFX10. 1920//===----------------------------------------------------------------------===// 1921 1922let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1923 multiclass MUBUF_Real_gfx6<bits<8> op> { 1924 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1925 } 1926} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1927 1928let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1929 multiclass MUBUF_Real_gfx7<bits<8> op> { 1930 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1931 } 1932} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1933 1934let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1935 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1936 def _ADDR64_gfx6_gfx7 : 1937 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1938 def _BOTHEN_gfx6_gfx7 : 1939 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1940 def _IDXEN_gfx6_gfx7 : 1941 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1942 def _OFFEN_gfx6_gfx7 : 1943 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1944 def _OFFSET_gfx6_gfx7 : 1945 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1946 } 1947 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1948 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1949 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 1950 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1951 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 1952 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1953 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 1954 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1955 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 1956 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1957 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 1958 1959 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1960 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 1961 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1962 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 1963 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1964 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 1965 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1966 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 1967 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1968 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 1969 } 1970 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 1971 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 1972 def _ADDR64_RTN_gfx6_gfx7 : 1973 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1974 def _BOTHEN_RTN_gfx6_gfx7 : 1975 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1976 def _IDXEN_RTN_gfx6_gfx7 : 1977 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1978 def _OFFEN_RTN_gfx6_gfx7 : 1979 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1980 def _OFFSET_RTN_gfx6_gfx7 : 1981 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1982 } 1983} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 1984 1985multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 1986 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 1987 1988multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 1989 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 1990 1991multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 1992 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 1993 1994// FIXME-GFX6: Following instructions are available only on GFX6. 1995//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 1996//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 1997 1998defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 1999defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2000defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2001defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2002defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2003defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2004defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2005defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2006defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2007defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2008defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2009defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2010defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2011defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2012defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2013defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2014defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2015defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2016defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2017defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2018defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2019defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2020 2021defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2022defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2023defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2024defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2025defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2026defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2027defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2028defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2029defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2030defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2031defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2032defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2033defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2034defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2035defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2036defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2037defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2038defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2039defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2040defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2041defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2042defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2043defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2044defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2045defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2046defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2047defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2048defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2049defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2050// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2051defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2052defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2053defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2054 2055defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2056defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2057def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2058 2059//===----------------------------------------------------------------------===// 2060// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2061//===----------------------------------------------------------------------===// 2062 2063class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2064 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2065 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2066 let Inst{12} = ps.offen; 2067 let Inst{13} = ps.idxen; 2068 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2069 let Inst{18-16} = op; 2070 let Inst{31-26} = 0x3a; //encoding 2071 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2072 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2073 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2074 let Inst{54} = !if(ps.has_slc, slc, ?); 2075 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2076 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2077} 2078 2079//===----------------------------------------------------------------------===// 2080// MTBUF - GFX10. 2081//===----------------------------------------------------------------------===// 2082 2083class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2084 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2085 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2086 let Inst{25-19} = format; 2087 let Inst{53} = op{3}; 2088} 2089 2090let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2091 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2092 def _BOTHEN_gfx10 : 2093 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2094 def _IDXEN_gfx10 : 2095 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2096 def _OFFEN_gfx10 : 2097 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2098 def _OFFSET_gfx10 : 2099 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2100 } 2101} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2102 2103defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2104defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2105defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2106defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2107defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2108defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2109defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2110defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2111 2112//===----------------------------------------------------------------------===// 2113// MTBUF - GFX6, GFX7, GFX10. 2114//===----------------------------------------------------------------------===// 2115 2116class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2117 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2118 let Inst{15} = ps.addr64; 2119 let Inst{22-19} = dfmt; 2120 let Inst{25-23} = nfmt; 2121} 2122 2123let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2124 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2125 def _ADDR64_gfx6_gfx7 : 2126 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2127 def _BOTHEN_gfx6_gfx7 : 2128 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2129 def _IDXEN_gfx6_gfx7 : 2130 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2131 def _OFFEN_gfx6_gfx7 : 2132 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2133 def _OFFSET_gfx6_gfx7 : 2134 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2135 } 2136} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2137 2138multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2139 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2140 2141defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2142defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2143defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2144defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2145defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2146defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2147defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2148defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2149 2150//===----------------------------------------------------------------------===// 2151// GFX8, GFX9 (VI). 2152//===----------------------------------------------------------------------===// 2153 2154class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2155 MUBUF_Real<ps>, 2156 Enc64, 2157 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2158 let AssemblerPredicate = isGFX8GFX9; 2159 let DecoderNamespace = "GFX8"; 2160 2161 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2162 let Inst{12} = ps.offen; 2163 let Inst{13} = ps.idxen; 2164 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2165 let Inst{16} = !if(ps.lds, 1, 0); 2166 let Inst{17} = !if(ps.has_slc, slc, ?); 2167 let Inst{24-18} = op; 2168 let Inst{31-26} = 0x38; //encoding 2169 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2170 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2171 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2172 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2173 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2174} 2175 2176multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2177 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2178 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2179 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2180 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2181} 2182 2183multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2184 2185 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2186 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2187 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2188 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2189 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2190 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2191 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2192 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2193 2194 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2195 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2196 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2197 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2198 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2199 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2200 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2201 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2202} 2203 2204class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2205 MUBUF_Real<ps>, 2206 Enc64, 2207 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2208 let AssemblerPredicate=HasUnpackedD16VMem; 2209 let DecoderNamespace="GFX80_UNPACKED"; 2210 2211 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2212 let Inst{12} = ps.offen; 2213 let Inst{13} = ps.idxen; 2214 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2215 let Inst{16} = !if(ps.lds, 1, 0); 2216 let Inst{17} = !if(ps.has_slc, slc, ?); 2217 let Inst{24-18} = op; 2218 let Inst{31-26} = 0x38; //encoding 2219 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2220 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2221 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2222 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2223 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2224} 2225 2226multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2227 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2228 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2229 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2230 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2231} 2232 2233multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2234 MUBUF_Real_AllAddr_vi<op> { 2235 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2236 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2237 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2238 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2239} 2240 2241defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2242defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2243defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2244defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2245defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2246defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2247defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2248defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2249let SubtargetPredicate = HasUnpackedD16VMem in { 2250 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2251 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2252 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2253 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2254 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2255 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2256 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2257 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2258} // End HasUnpackedD16VMem. 2259let SubtargetPredicate = HasPackedD16VMem in { 2260 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2261 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2262 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2263 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2264 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2265 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2266 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2267 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2268} // End HasPackedD16VMem. 2269defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2270defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2271defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2272defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2273defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2274defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2275defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2276defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2277defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2278defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2279defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2280defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2281defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2282defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2283defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2284defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2285 2286defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2287defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2288defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2289defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2290defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2291defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2292 2293defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2294defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2295 2296defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2297defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2298defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2299defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2300defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2301defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2302defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2303defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2304defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2305defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2306defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2307defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2308defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2309 2310defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2311defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2312defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2313defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2314defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2315defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2316defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2317defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2318defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2319defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2320defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2321defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2322defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2323 2324def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2325 2326def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2327def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2328 2329let SubtargetPredicate = HasAtomicFaddInsts in { 2330 2331defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2332defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2333 2334} // End SubtargetPredicate = HasAtomicFaddInsts 2335 2336class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2337 MTBUF_Real<ps>, 2338 Enc64, 2339 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2340 let AssemblerPredicate = isGFX8GFX9; 2341 let DecoderNamespace = "GFX8"; 2342 2343 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2344 let Inst{12} = ps.offen; 2345 let Inst{13} = ps.idxen; 2346 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2347 let Inst{18-15} = op; 2348 let Inst{22-19} = dfmt; 2349 let Inst{25-23} = nfmt; 2350 let Inst{31-26} = 0x3a; //encoding 2351 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2352 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2353 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2354 let Inst{54} = !if(ps.has_slc, slc, ?); 2355 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2356 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2357} 2358 2359multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2360 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2361 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2362 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2363 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2364} 2365 2366class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2367 MTBUF_Real<ps>, 2368 Enc64, 2369 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2370 let AssemblerPredicate=HasUnpackedD16VMem; 2371 let DecoderNamespace="GFX80_UNPACKED"; 2372 2373 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2374 let Inst{12} = ps.offen; 2375 let Inst{13} = ps.idxen; 2376 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2377 let Inst{18-15} = op; 2378 let Inst{22-19} = dfmt; 2379 let Inst{25-23} = nfmt; 2380 let Inst{31-26} = 0x3a; //encoding 2381 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2382 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2383 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2384 let Inst{54} = !if(ps.has_slc, slc, ?); 2385 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2386 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2387} 2388 2389multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2390 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2391 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2392 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2393 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2394} 2395 2396defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2397defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2398defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2399defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2400defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2401defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2402defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2403defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2404let SubtargetPredicate = HasUnpackedD16VMem in { 2405 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2406 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2407 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2408 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2409 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2410 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2411 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2412 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2413} // End HasUnpackedD16VMem. 2414let SubtargetPredicate = HasPackedD16VMem in { 2415 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2416 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2417 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2418 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2419 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2420 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2421 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2422 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2423} // End HasUnpackedD16VMem. 2424 2425def MUBUFInfoTable : GenericTable { 2426 let FilterClass = "MUBUF_Pseudo"; 2427 let CppTypeName = "MUBUFInfo"; 2428 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2429 2430 let PrimaryKey = ["Opcode"]; 2431 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2432} 2433 2434def getMUBUFInfoFromOpcode : SearchIndex { 2435 let Table = MUBUFInfoTable; 2436 let Key = ["Opcode"]; 2437} 2438 2439def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2440 let Table = MUBUFInfoTable; 2441 let Key = ["BaseOpcode", "elements"]; 2442} 2443 2444def MTBUFInfoTable : GenericTable { 2445 let FilterClass = "MTBUF_Pseudo"; 2446 let CppTypeName = "MTBUFInfo"; 2447 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2448 2449 let PrimaryKey = ["Opcode"]; 2450 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2451} 2452 2453def getMTBUFInfoFromOpcode : SearchIndex { 2454 let Table = MTBUFInfoTable; 2455 let Key = ["Opcode"]; 2456} 2457 2458def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2459 let Table = MTBUFInfoTable; 2460 let Key = ["BaseOpcode", "elements"]; 2461} 2462