1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108} 109 110class MTBUF_Real <MTBUF_Pseudo ps> : 111 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 112 113 let isPseudo = 0; 114 let isCodeGenOnly = 0; 115 116 // copy relevant pseudo op flags 117 let UseNamedOperandTable = ps.UseNamedOperandTable; 118 let SubtargetPredicate = ps.SubtargetPredicate; 119 let AsmMatchConverter = ps.AsmMatchConverter; 120 let Constraints = ps.Constraints; 121 let DisableEncoding = ps.DisableEncoding; 122 let TSFlags = ps.TSFlags; 123 124 bits<12> offset; 125 bits<1> glc; 126 bits<1> dlc; 127 bits<7> format; 128 bits<8> vaddr; 129 bits<8> vdata; 130 bits<7> srsrc; 131 bits<1> slc; 132 bits<1> tfe; 133 bits<8> soffset; 134 135 bits<4> dfmt = format{3-0}; 136 bits<3> nfmt = format{6-4}; 137} 138 139class getMTBUFInsDA<list<RegisterClass> vdataList, 140 list<RegisterClass> vaddrList=[]> { 141 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 142 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdataClass:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 152 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 153 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 155 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc,$format $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc,$format $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc,$format $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc,$format $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 187 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 188 189 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 190 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 191 192 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 193 194 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 int elems, 201 list<dag> pattern=[], 202 // Workaround bug bz30254 203 int addrKindCopy = addrKind> 204 : MTBUF_Pseudo<opName, 205 (outs vdataClass:$vdata), 206 getMTBUFIns<addrKindCopy>.ret, 207 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 208 pattern>, 209 MTBUF_SetupAddr<addrKindCopy> { 210 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 211 let mayLoad = 1; 212 let mayStore = 0; 213 let elements = elems; 214} 215 216multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 217 int elems, ValueType load_vt = i32, 218 SDPatternOperator ld = null_frag> { 219 220 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 221 [(set load_vt:$vdata, 222 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 223 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 224 MTBUFAddr64Table<0, NAME>; 225 226 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 227 [(set load_vt:$vdata, 228 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 229 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 230 MTBUFAddr64Table<1, NAME>; 231 232 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 233 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 234 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 235 236 let DisableWQM = 1 in { 237 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 238 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 239 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 240 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 241 } 242} 243 244class MTBUF_Store_Pseudo <string opName, 245 int addrKind, 246 RegisterClass vdataClass, 247 int elems, 248 list<dag> pattern=[], 249 // Workaround bug bz30254 250 int addrKindCopy = addrKind, 251 RegisterClass vdataClassCopy = vdataClass> 252 : MTBUF_Pseudo<opName, 253 (outs), 254 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 255 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 256 pattern>, 257 MTBUF_SetupAddr<addrKindCopy> { 258 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 259 let mayLoad = 0; 260 let mayStore = 1; 261 let elements = elems; 262} 263 264multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 265 int elems, ValueType store_vt = i32, 266 SDPatternOperator st = null_frag> { 267 268 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 269 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 270 i16:$offset, i8:$format, i1:$glc, 271 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 272 MTBUFAddr64Table<0, NAME>; 273 274 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 275 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 276 i16:$offset, i8:$format, i1:$glc, 277 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 278 MTBUFAddr64Table<1, NAME>; 279 280 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 281 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 282 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 283 284 let DisableWQM = 1 in { 285 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 286 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 287 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 288 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 289 } 290} 291 292 293//===----------------------------------------------------------------------===// 294// MUBUF classes 295//===----------------------------------------------------------------------===// 296 297class MUBUFGetBaseOpcode<string Op> { 298 string ret = !subst("DWORDX2", "DWORD", 299 !subst("DWORDX3", "DWORD", 300 !subst("DWORDX4", "DWORD", Op))); 301} 302 303class MUBUF_Pseudo <string opName, dag outs, dag ins, 304 string asmOps, list<dag> pattern=[]> : 305 InstSI<outs, ins, "", pattern>, 306 SIMCInstr<opName, SIEncodingFamily.NONE> { 307 308 let isPseudo = 1; 309 let isCodeGenOnly = 1; 310 let Size = 8; 311 let UseNamedOperandTable = 1; 312 313 string Mnemonic = opName; 314 string AsmOperands = asmOps; 315 316 Instruction Opcode = !cast<Instruction>(NAME); 317 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 318 319 let VM_CNT = 1; 320 let EXP_CNT = 1; 321 let MUBUF = 1; 322 let Uses = [EXEC]; 323 let hasSideEffects = 0; 324 let SchedRW = [WriteVMEM]; 325 326 let AsmMatchConverter = "cvtMubuf"; 327 328 bits<1> offen = 0; 329 bits<1> idxen = 0; 330 bits<1> addr64 = 0; 331 bits<1> lds = 0; 332 bits<1> has_vdata = 1; 333 bits<1> has_vaddr = 1; 334 bits<1> has_glc = 1; 335 bits<1> has_dlc = 1; 336 bits<1> glc_value = 0; // the value for glc if no such operand 337 bits<1> dlc_value = 0; // the value for dlc if no such operand 338 bits<1> has_srsrc = 1; 339 bits<1> has_soffset = 1; 340 bits<1> has_offset = 1; 341 bits<1> has_slc = 1; 342 bits<1> has_tfe = 1; 343 bits<4> elements = 0; 344} 345 346class MUBUF_Real <MUBUF_Pseudo ps> : 347 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 348 349 let isPseudo = 0; 350 let isCodeGenOnly = 0; 351 352 // copy relevant pseudo op flags 353 let SubtargetPredicate = ps.SubtargetPredicate; 354 let AsmMatchConverter = ps.AsmMatchConverter; 355 let Constraints = ps.Constraints; 356 let DisableEncoding = ps.DisableEncoding; 357 let TSFlags = ps.TSFlags; 358 359 bits<12> offset; 360 bits<1> glc; 361 bits<1> dlc; 362 bits<8> vaddr; 363 bits<8> vdata; 364 bits<7> srsrc; 365 bits<1> slc; 366 bits<1> tfe; 367 bits<8> soffset; 368} 369 370 371// For cache invalidation instructions. 372class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 373 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 374 375 let AsmMatchConverter = ""; 376 377 let hasSideEffects = 1; 378 let mayLoad = 0; 379 let mayStore = 0; 380 381 // Set everything to 0. 382 let offen = 0; 383 let idxen = 0; 384 let addr64 = 0; 385 let has_vdata = 0; 386 let has_vaddr = 0; 387 let has_glc = 0; 388 let has_dlc = 0; 389 let glc_value = 0; 390 let dlc_value = 0; 391 let has_srsrc = 0; 392 let has_soffset = 0; 393 let has_offset = 0; 394 let has_slc = 0; 395 let has_tfe = 0; 396} 397 398class getMUBUFInsDA<list<RegisterClass> vdataList, 399 list<RegisterClass> vaddrList=[], 400 bit isLds = 0> { 401 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 402 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 403 dag InsNoData = !if(!empty(vaddrList), 404 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 405 offset:$offset, GLC:$glc, SLC:$slc), 406 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 407 offset:$offset, GLC:$glc, SLC:$slc) 408 ); 409 dag InsData = !if(!empty(vaddrList), 410 (ins vdataClass:$vdata, SReg_128:$srsrc, 411 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 412 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 413 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 414 ); 415 dag ret = !con( 416 !if(!empty(vdataList), InsNoData, InsData), 417 !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz)) 418 ); 419} 420 421class getMUBUFElements<ValueType vt> { 422 // eq does not support ValueType for some reason. 423 string vtAsStr = !cast<string>(vt); 424 425 int ret = 426 !if(!eq(vtAsStr, "f16"), 1, 427 !if(!eq(vtAsStr, "v2f16"), 2, 428 !if(!eq(vtAsStr, "v3f16"), 3, 429 !if(!eq(vtAsStr, "v4f16"), 4, 430 !if(!eq(vt.Size, 32), 1, 431 !if(!eq(vt.Size, 64), 2, 432 !if(!eq(vt.Size, 96), 3, 433 !if(!eq(vt.Size, 128), 4, 0) 434 ) 435 ) 436 ) 437 ) 438 ) 439 ) 440 ); 441} 442 443class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 444 dag ret = 445 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 446 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 447 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 448 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 449 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 450 (ins)))))); 451} 452 453class getMUBUFAsmOps<int addrKind> { 454 string Pfx = 455 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 456 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 457 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 458 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 459 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 460 ""))))); 461 string ret = Pfx # "$offset"; 462} 463 464class MUBUF_SetupAddr<int addrKind> { 465 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 466 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 467 468 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 469 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 470 471 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 472 473 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 474} 475 476class MUBUF_Load_Pseudo <string opName, 477 int addrKind, 478 ValueType vdata_vt, 479 bit HasTiedDest = 0, 480 bit isLds = 0, 481 list<dag> pattern=[], 482 // Workaround bug bz30254 483 int addrKindCopy = addrKind> 484 : MUBUF_Pseudo<opName, 485 (outs getVregSrcForVT<vdata_vt>.ret:$vdata), 486 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 487 !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))), 488 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 489 !if(isLds, " lds", "$tfe") # "$dlc" # "$swz", 490 pattern>, 491 MUBUF_SetupAddr<addrKindCopy> { 492 let PseudoInstr = opName # !if(isLds, "_lds", "") # 493 "_" # getAddrName<addrKindCopy>.ret; 494 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 495 496 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 497 let mayLoad = 1; 498 let mayStore = 0; 499 let maybeAtomic = 1; 500 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 501 let has_tfe = !if(isLds, 0, 1); 502 let lds = isLds; 503 let elements = getMUBUFElements<vdata_vt>.ret; 504} 505 506class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 507 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 508 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 509>; 510 511class MUBUF_Addr64_Load_Pat <Instruction inst, 512 ValueType load_vt = i32, 513 SDPatternOperator ld = null_frag> : Pat < 514 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 515 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 516>; 517 518multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 519 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 520 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 521} 522 523 524// FIXME: tfe can't be an operand because it requires a separate 525// opcode because it needs an N+1 register class dest register. 526multiclass MUBUF_Pseudo_Loads<string opName, 527 ValueType load_vt = i32, 528 SDPatternOperator ld = null_frag, 529 bit TiedDest = 0, 530 bit isLds = 0> { 531 532 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>, 533 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 534 535 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>, 536 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 537 538 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 539 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 540 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 541 542 let DisableWQM = 1 in { 543 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>; 544 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 545 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 546 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 547 } 548} 549 550multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 551 SDPatternOperator ld_nolds = null_frag, 552 SDPatternOperator ld_lds = null_frag> { 553 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 554 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 555} 556 557class MUBUF_Store_Pseudo <string opName, 558 int addrKind, 559 ValueType store_vt, 560 list<dag> pattern=[], 561 // Workaround bug bz30254 562 int addrKindCopy = addrKind> 563 : MUBUF_Pseudo<opName, 564 (outs), 565 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 566 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 567 pattern>, 568 MUBUF_SetupAddr<addrKindCopy> { 569 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 570 let mayLoad = 0; 571 let mayStore = 1; 572 let maybeAtomic = 1; 573 let elements = getMUBUFElements<store_vt>.ret; 574} 575 576multiclass MUBUF_Pseudo_Stores<string opName, 577 ValueType store_vt = i32, 578 SDPatternOperator st = null_frag> { 579 580 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, 581 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 582 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 583 MUBUFAddr64Table<0, NAME>; 584 585 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, 586 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 587 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 588 MUBUFAddr64Table<1, NAME>; 589 590 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 591 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 592 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 593 594 let DisableWQM = 1 in { 595 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>; 596 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 597 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 598 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 599 } 600} 601 602class MUBUF_Pseudo_Store_Lds<string opName> 603 : MUBUF_Pseudo<opName, 604 (outs), 605 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz), 606 " $srsrc, $soffset$offset lds$glc$slc$swz"> { 607 let mayLoad = 0; 608 let mayStore = 1; 609 let maybeAtomic = 1; 610 611 let has_vdata = 0; 612 let has_vaddr = 0; 613 let has_tfe = 0; 614 let lds = 1; 615 616 let Uses = [EXEC, M0]; 617 let AsmMatchConverter = "cvtMubufLds"; 618} 619 620class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 621 list<RegisterClass> vaddrList=[]> { 622 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 623 dag ret = !if(vdata_in, 624 !if(!empty(vaddrList), 625 (ins vdataClass:$vdata_in, 626 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 627 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 628 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 629 ), 630 !if(!empty(vaddrList), 631 (ins vdataClass:$vdata, 632 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 633 (ins vdataClass:$vdata, vaddrClass:$vaddr, 634 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 635 )); 636} 637 638class getMUBUFAtomicIns<int addrKind, 639 RegisterClass vdataClass, 640 bit vdata_in, 641 // Workaround bug bz30254 642 RegisterClass vdataClassCopy=vdataClass> { 643 dag ret = 644 !if(!eq(addrKind, BUFAddrKind.Offset), 645 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 646 !if(!eq(addrKind, BUFAddrKind.OffEn), 647 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 648 !if(!eq(addrKind, BUFAddrKind.IdxEn), 649 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 650 !if(!eq(addrKind, BUFAddrKind.BothEn), 651 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 652 !if(!eq(addrKind, BUFAddrKind.Addr64), 653 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 654 (ins)))))); 655} 656 657class MUBUF_Atomic_Pseudo<string opName, 658 int addrKind, 659 dag outs, 660 dag ins, 661 string asmOps, 662 list<dag> pattern=[], 663 // Workaround bug bz30254 664 int addrKindCopy = addrKind> 665 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 666 MUBUF_SetupAddr<addrKindCopy> { 667 let mayStore = 1; 668 let mayLoad = 1; 669 let hasPostISelHook = 1; 670 let hasSideEffects = 1; 671 let DisableWQM = 1; 672 let has_glc = 0; 673 let has_dlc = 0; 674 let has_tfe = 0; 675 let maybeAtomic = 1; 676} 677 678class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 679 RegisterClass vdataClass, 680 list<dag> pattern=[], 681 // Workaround bug bz30254 682 int addrKindCopy = addrKind, 683 RegisterClass vdataClassCopy = vdataClass> 684 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 685 (outs), 686 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 687 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 688 pattern>, 689 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 690 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 691 let glc_value = 0; 692 let dlc_value = 0; 693 let AsmMatchConverter = "cvtMubufAtomic"; 694} 695 696class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 697 RegisterClass vdataClass, 698 list<dag> pattern=[], 699 // Workaround bug bz30254 700 int addrKindCopy = addrKind, 701 RegisterClass vdataClassCopy = vdataClass> 702 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 703 (outs vdataClassCopy:$vdata), 704 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 705 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 706 pattern>, 707 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 708 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 709 let glc_value = 1; 710 let dlc_value = 0; 711 let Constraints = "$vdata = $vdata_in"; 712 let DisableEncoding = "$vdata_in"; 713 let AsmMatchConverter = "cvtMubufAtomicReturn"; 714} 715 716multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 717 RegisterClass vdataClass, 718 ValueType vdataType, 719 SDPatternOperator atomic, 720 bit isFP = isFloatType<vdataType>.ret> { 721 let FPAtomic = isFP in 722 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 723 MUBUFAddr64Table <0, NAME>; 724 725 let FPAtomic = isFP in 726 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 727 MUBUFAddr64Table <1, NAME>; 728 729 let FPAtomic = isFP in 730 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 731 732 let FPAtomic = isFP in 733 734 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 735 736 let FPAtomic = isFP in 737 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 738} 739 740multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 741 RegisterClass vdataClass, 742 ValueType vdataType, 743 SDPatternOperator atomic, 744 bit isFP = isFloatType<vdataType>.ret> { 745 let FPAtomic = isFP in 746 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 747 [(set vdataType:$vdata, 748 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 749 vdataType:$vdata_in))]>, 750 MUBUFAddr64Table <0, NAME # "_RTN">; 751 752 let FPAtomic = isFP in 753 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 754 [(set vdataType:$vdata, 755 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 756 vdataType:$vdata_in))]>, 757 MUBUFAddr64Table <1, NAME # "_RTN">; 758 759 let FPAtomic = isFP in 760 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 761 762 let FPAtomic = isFP in 763 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 764 765 let FPAtomic = isFP in 766 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 767} 768 769multiclass MUBUF_Pseudo_Atomics <string opName, 770 RegisterClass vdataClass, 771 ValueType vdataType, 772 SDPatternOperator atomic> : 773 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 774 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 775 776 777//===----------------------------------------------------------------------===// 778// MUBUF Instructions 779//===----------------------------------------------------------------------===// 780 781defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 782 "buffer_load_format_x", f32 783>; 784defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 785 "buffer_load_format_xy", v2f32 786>; 787defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 788 "buffer_load_format_xyz", v3f32 789>; 790defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 791 "buffer_load_format_xyzw", v4f32 792>; 793defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 794 "buffer_store_format_x", f32 795>; 796defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 797 "buffer_store_format_xy", v2f32 798>; 799defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 800 "buffer_store_format_xyz", v3f32 801>; 802defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 803 "buffer_store_format_xyzw", v4f32 804>; 805 806let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 807 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 808 "buffer_load_format_d16_x", i32 809 >; 810 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 811 "buffer_load_format_d16_xy", v2i32 812 >; 813 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 814 "buffer_load_format_d16_xyz", v3i32 815 >; 816 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 817 "buffer_load_format_d16_xyzw", v4i32 818 >; 819 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 820 "buffer_store_format_d16_x", i32 821 >; 822 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 823 "buffer_store_format_d16_xy", v2i32 824 >; 825 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 826 "buffer_store_format_d16_xyz", v3i32 827 >; 828 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 829 "buffer_store_format_d16_xyzw", v4i32 830 >; 831} // End HasUnpackedD16VMem. 832 833let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 834 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 835 "buffer_load_format_d16_x", f16 836 >; 837 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 838 "buffer_load_format_d16_xy", v2f16 839 >; 840 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_xyz", v3f16 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xyzw", v4f16 845 >; 846 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 847 "buffer_store_format_d16_x", f16 848 >; 849 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 850 "buffer_store_format_d16_xy", v2f16 851 >; 852 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_xyz", v3f16 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xyzw", v4f16 857 >; 858} // End HasPackedD16VMem. 859 860defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 861 "buffer_load_ubyte", i32 862>; 863defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 864 "buffer_load_sbyte", i32 865>; 866defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 867 "buffer_load_ushort", i32 868>; 869defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 870 "buffer_load_sshort", i32 871>; 872defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 873 "buffer_load_dword", i32 874>; 875defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 876 "buffer_load_dwordx2", v2i32 877>; 878defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 879 "buffer_load_dwordx3", v3i32 880>; 881defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 882 "buffer_load_dwordx4", v4i32 883>; 884 885defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 886defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 887defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 888defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 889defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 890defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 891defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 892defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 893defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 894defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 895 896// This is not described in AMD documentation, 897// but 'lds' versions of these opcodes are available 898// in at least GFX8+ chips. See Bug 37653. 899let SubtargetPredicate = isGFX8GFX9 in { 900defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 901 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 902>; 903defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 904 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 905>; 906defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 907 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 908>; 909} 910 911defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 912 "buffer_store_byte", i32, truncstorei8_global 913>; 914defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 915 "buffer_store_short", i32, truncstorei16_global 916>; 917defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 918 "buffer_store_dword", i32, store_global 919>; 920defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 921 "buffer_store_dwordx2", v2i32, store_global 922>; 923defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 924 "buffer_store_dwordx3", v3i32, store_global 925>; 926defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 927 "buffer_store_dwordx4", v4i32, store_global 928>; 929defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 930 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 931>; 932defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 933 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 934>; 935defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 936 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 937>; 938defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 939 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 940>; 941defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 942 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 943>; 944defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 945 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 946>; 947defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 948 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 949>; 950defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 951 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 952>; 953defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 954 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 955>; 956defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 957 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 958>; 959defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 960 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 961>; 962defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 963 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 964>; 965defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 966 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 967>; 968defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 969 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 970>; 971defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 972 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 973>; 974defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 975 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 976>; 977defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 978 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 979>; 980defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 981 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 982>; 983defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 984 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 985>; 986defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 987 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 988>; 989defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 990 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 991>; 992defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 993 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 994>; 995defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 996 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 997>; 998defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 999 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 1000>; 1001defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1002 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1003>; 1004defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1005 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1006>; 1007 1008let SubtargetPredicate = HasGFX10_BEncoding in 1009defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1010 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1011>; 1012 1013let SubtargetPredicate = isGFX8GFX9 in { 1014def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1015} 1016 1017let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1018/* 1019defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1020defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1021*/ 1022 1023def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1024 int_amdgcn_buffer_wbinvl1_sc>; 1025} 1026 1027let SubtargetPredicate = isGFX6GFX7GFX10 in { 1028 1029defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1030 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1031>; 1032defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1033 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1034>; 1035defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1036 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1037>; 1038defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1039 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1040>; 1041defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1042 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1043>; 1044defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1045 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1046>; 1047 1048} 1049 1050let SubtargetPredicate = HasD16LoadStore in { 1051 1052defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1053 "buffer_load_ubyte_d16", i32, null_frag, 1 1054>; 1055 1056defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1057 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1058>; 1059 1060defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1061 "buffer_load_sbyte_d16", i32, null_frag, 1 1062>; 1063 1064defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1065 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1066>; 1067 1068defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1069 "buffer_load_short_d16", i32, null_frag, 1 1070>; 1071 1072defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1073 "buffer_load_short_d16_hi", i32, null_frag, 1 1074>; 1075 1076defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1077 "buffer_store_byte_d16_hi", i32 1078>; 1079 1080defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1081 "buffer_store_short_d16_hi", i32 1082>; 1083 1084defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1085 "buffer_load_format_d16_hi_x", i32 1086>; 1087defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1088 "buffer_store_format_d16_hi_x", i32 1089>; 1090 1091} // End HasD16LoadStore 1092 1093def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1094 int_amdgcn_buffer_wbinvl1>; 1095 1096let SubtargetPredicate = HasAtomicFaddInsts in { 1097 1098defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1099 "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret 1100>; 1101defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1102 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_fadd_global_noret 1103>; 1104 1105} // End SubtargetPredicate = HasAtomicFaddInsts 1106 1107//===----------------------------------------------------------------------===// 1108// MTBUF Instructions 1109//===----------------------------------------------------------------------===// 1110 1111defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1112defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1113defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1114defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1115defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1116defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1117defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1118defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1119 1120let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1121 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1122 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1123 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1124 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1125 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1126 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1127 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1128 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1129} // End HasUnpackedD16VMem. 1130 1131let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1132 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1133 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1134 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1135 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1136 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1137 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1138 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1139 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1140} // End HasPackedD16VMem. 1141 1142let SubtargetPredicate = isGFX7Plus in { 1143 1144//===----------------------------------------------------------------------===// 1145// Instruction definitions for CI and newer. 1146//===----------------------------------------------------------------------===// 1147 1148def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1149 int_amdgcn_buffer_wbinvl1_vol>; 1150 1151} // End let SubtargetPredicate = isGFX7Plus 1152 1153let SubtargetPredicate = isGFX10Plus in { 1154 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1155 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1156} // End SubtargetPredicate = isGFX10Plus 1157 1158//===----------------------------------------------------------------------===// 1159// MUBUF Patterns 1160//===----------------------------------------------------------------------===// 1161 1162//===----------------------------------------------------------------------===// 1163// buffer_load/store_format patterns 1164//===----------------------------------------------------------------------===// 1165 1166multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1167 string opcode> { 1168 def : GCNPat< 1169 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1170 timm:$auxiliary, 0)), 1171 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1172 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1173 (extract_swz $auxiliary)) 1174 >; 1175 1176 def : GCNPat< 1177 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1178 timm:$auxiliary, 0)), 1179 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1180 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1181 (extract_swz $auxiliary)) 1182 >; 1183 1184 def : GCNPat< 1185 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1186 timm:$auxiliary, timm)), 1187 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1188 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1189 (extract_swz $auxiliary)) 1190 >; 1191 1192 def : GCNPat< 1193 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1194 timm:$auxiliary, timm)), 1195 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1196 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1197 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1198 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1199 (extract_swz $auxiliary)) 1200 >; 1201} 1202 1203defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1204defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1205defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1206defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1207defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1208defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1209defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1210defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1211 1212let SubtargetPredicate = HasUnpackedD16VMem in { 1213 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1214 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1215 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1216 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1217 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1218} // End HasUnpackedD16VMem. 1219 1220let SubtargetPredicate = HasPackedD16VMem in { 1221 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1222 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1223 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1224 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1225 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1226 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1227 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1228} // End HasPackedD16VMem. 1229 1230defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1231defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1232defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1233defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1234defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1235defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1236defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1237defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1238defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1239defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1240defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1241defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1242defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1243defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1244defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1245defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1246 1247multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1248 string opcode> { 1249 def : GCNPat< 1250 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1251 timm:$auxiliary, 0), 1252 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1253 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1254 (extract_swz $auxiliary)) 1255 >; 1256 1257 def : GCNPat< 1258 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1259 timm:$auxiliary, 0), 1260 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1261 (as_i16timm $offset), (extract_glc $auxiliary), 1262 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1263 (extract_swz $auxiliary)) 1264 >; 1265 1266 def : GCNPat< 1267 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1268 timm:$auxiliary, timm), 1269 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1270 (as_i16timm $offset), (extract_glc $auxiliary), 1271 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1272 (extract_swz $auxiliary)) 1273 >; 1274 1275 def : GCNPat< 1276 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1277 timm:$auxiliary, timm), 1278 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1279 getVregSrcForVT<vt>.ret:$vdata, 1280 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1281 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary), 1282 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1283 (extract_swz $auxiliary)) 1284 >; 1285} 1286 1287defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1288defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1289defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1290defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1291defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1292defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1293defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1294defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1295 1296let SubtargetPredicate = HasUnpackedD16VMem in { 1297 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1298 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1299 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1300 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1301 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1302} // End HasUnpackedD16VMem. 1303 1304let SubtargetPredicate = HasPackedD16VMem in { 1305 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1306 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1307 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1308 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1309 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1310 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1311 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1312} // End HasPackedD16VMem. 1313 1314defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1315defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1316defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1317defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1318defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1319defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1320defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1321defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1322defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1323defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1324defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1325defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1326defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1327defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1328 1329//===----------------------------------------------------------------------===// 1330// buffer_atomic patterns 1331//===----------------------------------------------------------------------===// 1332 1333multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1334 string opcode> { 1335 def : GCNPat< 1336 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1337 timm:$offset, timm:$cachepolicy, 0)), 1338 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1339 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1340 (as_i16timm $offset), (extract_slc $cachepolicy)) 1341 >; 1342 1343 def : GCNPat< 1344 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1345 timm:$offset, timm:$cachepolicy, timm)), 1346 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1347 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1348 (as_i16timm $offset), (extract_slc $cachepolicy)) 1349 >; 1350 1351 def : GCNPat< 1352 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1353 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1354 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1355 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1356 (as_i16timm $offset), (extract_slc $cachepolicy)) 1357 >; 1358 1359 def : GCNPat< 1360 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1361 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1362 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1363 getVregSrcForVT<vt>.ret:$vdata_in, 1364 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1365 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1366 (extract_slc $cachepolicy)) 1367 >; 1368} 1369 1370defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1371defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">; 1372defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1373defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1374defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1375defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1376defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1377defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1378defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1379defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1380defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1381defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1382defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1383defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1384defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1385defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1386defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1387defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1388defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1389defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1390defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1391defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1392defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1393defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1394defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1395defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1396 1397multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1398 string opcode> { 1399 def : GCNPat< 1400 (name vt:$vdata_in, v4i32:$rsrc, 0, 1401 0, i32:$soffset, timm:$offset, 1402 timm:$cachepolicy, 0), 1403 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1404 (as_i16timm $offset), (extract_slc $cachepolicy)) 1405 >; 1406 1407 def : GCNPat< 1408 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1409 0, i32:$soffset, timm:$offset, 1410 timm:$cachepolicy, timm), 1411 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1412 (as_i16timm $offset), (extract_slc $cachepolicy)) 1413 >; 1414 1415 def : GCNPat< 1416 (name vt:$vdata_in, v4i32:$rsrc, 0, 1417 i32:$voffset, i32:$soffset, timm:$offset, 1418 timm:$cachepolicy, 0), 1419 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1420 (as_i16timm $offset), (extract_slc $cachepolicy)) 1421 >; 1422 1423 def : GCNPat< 1424 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1425 i32:$voffset, i32:$soffset, timm:$offset, 1426 timm:$cachepolicy, timm), 1427 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1428 getVregSrcForVT<vt>.ret:$vdata_in, 1429 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1430 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)) 1431 >; 1432} 1433 1434let SubtargetPredicate = HasAtomicFaddInsts in { 1435defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1436defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1437} 1438 1439def : GCNPat< 1440 (SIbuffer_atomic_cmpswap 1441 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1442 timm:$offset, timm:$cachepolicy, 0), 1443 (EXTRACT_SUBREG 1444 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1445 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1446 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1447 (extract_slc $cachepolicy)), sub0) 1448>; 1449 1450def : GCNPat< 1451 (SIbuffer_atomic_cmpswap 1452 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1453 0, i32:$soffset, timm:$offset, 1454 timm:$cachepolicy, timm), 1455 (EXTRACT_SUBREG 1456 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1457 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1458 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1459 sub0) 1460>; 1461 1462def : GCNPat< 1463 (SIbuffer_atomic_cmpswap 1464 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1465 i32:$voffset, i32:$soffset, timm:$offset, 1466 timm:$cachepolicy, 0), 1467 (EXTRACT_SUBREG 1468 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1469 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1470 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1471 sub0) 1472>; 1473 1474def : GCNPat< 1475 (SIbuffer_atomic_cmpswap 1476 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1477 i32:$voffset, i32:$soffset, timm:$offset, 1478 timm:$cachepolicy, timm), 1479 (EXTRACT_SUBREG 1480 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1481 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1482 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1483 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1484 sub0) 1485>; 1486 1487class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1488 PatFrag constant_ld> : GCNPat < 1489 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1490 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1491 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1492 >; 1493 1494multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1495 ValueType vt, PatFrag atomic_ld> { 1496 def : GCNPat < 1497 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1498 i16:$offset, i1:$slc))), 1499 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1500 >; 1501 1502 def : GCNPat < 1503 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1504 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1505 >; 1506} 1507 1508let SubtargetPredicate = isGFX6GFX7 in { 1509def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1510def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1511def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1512def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1513def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1514def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1515 1516defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1517defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1518} // End SubtargetPredicate = isGFX6GFX7 1519 1520multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1521 PatFrag ld> { 1522 1523 def : GCNPat < 1524 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1525 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1526 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1527 >; 1528} 1529 1530let OtherPredicates = [Has16BitInsts] in { 1531 1532defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1533defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1534defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1535defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1536defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1537defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1538 1539defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1540 1541} // End OtherPredicates = [Has16BitInsts] 1542 1543multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1544 MUBUF_Pseudo InstrOffset, 1545 ValueType vt, PatFrag ld> { 1546 def : GCNPat < 1547 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1548 i32:$soffset, u16imm:$offset))), 1549 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1550 >; 1551 1552 def : GCNPat < 1553 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1554 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1555 >; 1556} 1557 1558// XXX - Is it possible to have a complex pattern in a PatFrag? 1559multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1560 MUBUF_Pseudo InstrOffset, 1561 ValueType vt, PatFrag ld_frag> { 1562 def : GCNPat < 1563 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1564 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1565 >; 1566 1567 def : GCNPat < 1568 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1569 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1570 >; 1571} 1572 1573defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1574defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1575defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1576defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1577defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1578defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1579defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1580defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1581defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1582defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1583 1584foreach vt = Reg32Types.types in { 1585defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1586} 1587defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1588defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1589defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1590 1591let OtherPredicates = [D16PreservesUnusedBits] in { 1592defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1593defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1594defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1595defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1596defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1597defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1598 1599defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1600defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1601defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1602defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1603defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1604defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1605} 1606 1607multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1608 ValueType vt, PatFrag atomic_st> { 1609 // Store follows atomic op convention so address is first 1610 def : GCNPat < 1611 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1612 i16:$offset, i1:$slc), vt:$val), 1613 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1614 >; 1615 1616 def : GCNPat < 1617 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1618 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1619 >; 1620} 1621let SubtargetPredicate = isGFX6GFX7 in { 1622defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1623defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1624} // End Predicates = isGFX6GFX7 1625 1626 1627multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1628 PatFrag st> { 1629 1630 def : GCNPat < 1631 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1632 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1633 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1634 >; 1635} 1636 1637defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1638defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1639 1640multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1641 MUBUF_Pseudo InstrOffset, 1642 ValueType vt, PatFrag st, 1643 RegisterClass rc = VGPR_32> { 1644 def : GCNPat < 1645 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1646 i32:$soffset, u16imm:$offset)), 1647 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1648 >; 1649 1650 def : GCNPat < 1651 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1652 u16imm:$offset)), 1653 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1654 >; 1655} 1656 1657defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1658defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1659defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1660defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1661 1662foreach vt = Reg32Types.types in { 1663defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1664} 1665 1666defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1667defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1668defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1669 1670 1671let OtherPredicates = [D16PreservesUnusedBits] in { 1672 // Hiding the extract high pattern in the PatFrag seems to not 1673 // automatically increase the complexity. 1674let AddedComplexity = 1 in { 1675defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1676defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1677} 1678} 1679 1680//===----------------------------------------------------------------------===// 1681// MTBUF Patterns 1682//===----------------------------------------------------------------------===// 1683 1684//===----------------------------------------------------------------------===// 1685// tbuffer_load/store_format patterns 1686//===----------------------------------------------------------------------===// 1687 1688multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1689 string opcode> { 1690 def : GCNPat< 1691 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1692 timm:$format, timm:$auxiliary, 0)), 1693 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1694 (as_i8timm $format), 1695 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1696 (extract_swz $auxiliary)) 1697 >; 1698 1699 def : GCNPat< 1700 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1701 timm:$format, timm:$auxiliary, timm)), 1702 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1703 (as_i8timm $format), 1704 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1705 (extract_swz $auxiliary)) 1706 >; 1707 1708 def : GCNPat< 1709 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1710 timm:$format, timm:$auxiliary, 0)), 1711 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1712 (as_i8timm $format), 1713 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1714 (extract_swz $auxiliary)) 1715 >; 1716 1717 def : GCNPat< 1718 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1719 timm:$format, timm:$auxiliary, timm)), 1720 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1721 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1722 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1723 (as_i8timm $format), 1724 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1725 (extract_swz $auxiliary)) 1726 >; 1727} 1728 1729defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1730defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1731defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1732defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1733defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1734defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1735defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1736defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1737 1738let SubtargetPredicate = HasUnpackedD16VMem in { 1739 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1740 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1741 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1742 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1743} // End HasUnpackedD16VMem. 1744 1745let SubtargetPredicate = HasPackedD16VMem in { 1746 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1747 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1748 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1749 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1750} // End HasPackedD16VMem. 1751 1752multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1753 string opcode> { 1754 def : GCNPat< 1755 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1756 timm:$format, timm:$auxiliary, 0), 1757 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1758 (as_i16timm $offset), (as_i8timm $format), 1759 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1760 (extract_swz $auxiliary)) 1761 >; 1762 1763 def : GCNPat< 1764 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1765 timm:$format, timm:$auxiliary, timm), 1766 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1767 (as_i16timm $offset), (as_i8timm $format), 1768 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1769 (extract_swz $auxiliary)) 1770 >; 1771 1772 def : GCNPat< 1773 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1774 timm:$format, timm:$auxiliary, 0), 1775 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1776 (as_i16timm $offset), (as_i8timm $format), 1777 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1778 (extract_swz $auxiliary)) 1779 >; 1780 1781 def : GCNPat< 1782 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1783 timm:$offset, timm:$format, timm:$auxiliary, timm), 1784 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1785 getVregSrcForVT<vt>.ret:$vdata, 1786 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1787 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1788 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1789 (extract_swz $auxiliary)) 1790 >; 1791} 1792 1793defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1794defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1795defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1796defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1797defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1798defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1799defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1800defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1801 1802let SubtargetPredicate = HasUnpackedD16VMem in { 1803 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1804 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1805 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1806 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1807} // End HasUnpackedD16VMem. 1808 1809let SubtargetPredicate = HasPackedD16VMem in { 1810 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1811 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1812 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1813 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1814} // End HasPackedD16VMem. 1815 1816//===----------------------------------------------------------------------===// 1817// Target-specific instruction encodings. 1818//===----------------------------------------------------------------------===// 1819 1820//===----------------------------------------------------------------------===// 1821// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1822//===----------------------------------------------------------------------===// 1823 1824class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1825 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1826 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1827 let Inst{12} = ps.offen; 1828 let Inst{13} = ps.idxen; 1829 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1830 let Inst{16} = !if(ps.lds, 1, 0); 1831 let Inst{24-18} = op; 1832 let Inst{31-26} = 0x38; 1833 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1834 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1835 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1836 let Inst{54} = !if(ps.has_slc, slc, ?); 1837 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1838 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1839} 1840 1841class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1842 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1843 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1844 let Inst{25} = op{7}; 1845} 1846 1847class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1848 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1849 let Inst{15} = ps.addr64; 1850} 1851 1852//===----------------------------------------------------------------------===// 1853// MUBUF - GFX10. 1854//===----------------------------------------------------------------------===// 1855 1856let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1857 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1858 string asmName> { 1859 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1860 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1861 let AsmString = asmName # ps.AsmOperands; 1862 } 1863 } 1864 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1865 def _BOTHEN_gfx10 : 1866 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1867 def _IDXEN_gfx10 : 1868 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1869 def _OFFEN_gfx10 : 1870 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1871 def _OFFSET_gfx10 : 1872 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1873 } 1874 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1875 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1876 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1877 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1878 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1879 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1880 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1881 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1882 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1883 1884 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1885 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1886 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1887 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1888 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1889 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1890 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1891 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1892 } 1893 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1894 def _BOTHEN_RTN_gfx10 : 1895 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1896 def _IDXEN_RTN_gfx10 : 1897 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1898 def _OFFEN_RTN_gfx10 : 1899 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1900 def _OFFSET_RTN_gfx10 : 1901 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1902 } 1903 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1904 MUBUF_Real_AllAddr_gfx10<op>, MUBUF_Real_Atomics_RTN_gfx10<op>; 1905} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1906 1907defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1908defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1909defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1910defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1911defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1912defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1913defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1914defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1915// FIXME-GFX10: Add following instructions: 1916//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1917//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1918defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1919defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1920defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1921defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1922defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1923defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1924defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1925defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1926 1927def BUFFER_GL0_INV_gfx10 : 1928 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1929def BUFFER_GL1_INV_gfx10 : 1930 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1931 1932//===----------------------------------------------------------------------===// 1933// MUBUF - GFX6, GFX7, GFX10. 1934//===----------------------------------------------------------------------===// 1935 1936let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1937 multiclass MUBUF_Real_gfx6<bits<8> op> { 1938 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1939 } 1940} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1941 1942let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1943 multiclass MUBUF_Real_gfx7<bits<8> op> { 1944 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1945 } 1946} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1947 1948let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1949 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1950 def _ADDR64_gfx6_gfx7 : 1951 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1952 def _BOTHEN_gfx6_gfx7 : 1953 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1954 def _IDXEN_gfx6_gfx7 : 1955 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1956 def _OFFEN_gfx6_gfx7 : 1957 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1958 def _OFFSET_gfx6_gfx7 : 1959 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1960 } 1961 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1962 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1963 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 1964 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1965 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 1966 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1967 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 1968 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1969 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 1970 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1971 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 1972 1973 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1974 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 1975 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1976 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 1977 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1978 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 1979 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1980 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 1981 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1982 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 1983 } 1984 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 1985 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 1986 def _ADDR64_RTN_gfx6_gfx7 : 1987 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1988 def _BOTHEN_RTN_gfx6_gfx7 : 1989 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1990 def _IDXEN_RTN_gfx6_gfx7 : 1991 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1992 def _OFFEN_RTN_gfx6_gfx7 : 1993 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1994 def _OFFSET_RTN_gfx6_gfx7 : 1995 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1996 } 1997} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 1998 1999multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2000 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2001 2002multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2003 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2004 2005multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2006 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2007 2008// FIXME-GFX6: Following instructions are available only on GFX6. 2009//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2010//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2011 2012defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2013defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2014defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2015defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2016defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2017defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2018defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2019defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2020defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2021defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2022defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2023defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2024defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2025defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2026defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2027defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2028defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2029defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2030defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2031defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2032defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2033defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2034 2035defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2036defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2037defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2038defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2039defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2040defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2041defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2042defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2043defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2044defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2045defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2046defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2047defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2048defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2049defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2050defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2051defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2052defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2053defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2054defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2055defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2056defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2057defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2058defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2059defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2060defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2061defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2062defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2063defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2064// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2065defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2066defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2067defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2068 2069defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2070 2071defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2072defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2073def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2074 2075//===----------------------------------------------------------------------===// 2076// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2077//===----------------------------------------------------------------------===// 2078 2079class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2080 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2081 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2082 let Inst{12} = ps.offen; 2083 let Inst{13} = ps.idxen; 2084 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2085 let Inst{18-16} = op; 2086 let Inst{31-26} = 0x3a; //encoding 2087 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2088 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2089 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2090 let Inst{54} = !if(ps.has_slc, slc, ?); 2091 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2092 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2093} 2094 2095//===----------------------------------------------------------------------===// 2096// MTBUF - GFX10. 2097//===----------------------------------------------------------------------===// 2098 2099class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2100 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2101 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2102 let Inst{25-19} = format; 2103 let Inst{53} = op{3}; 2104} 2105 2106let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2107 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2108 def _BOTHEN_gfx10 : 2109 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2110 def _IDXEN_gfx10 : 2111 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2112 def _OFFEN_gfx10 : 2113 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2114 def _OFFSET_gfx10 : 2115 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2116 } 2117} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2118 2119defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2120defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2121defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2122defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2123defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2124defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2125defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2126defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2127 2128//===----------------------------------------------------------------------===// 2129// MTBUF - GFX6, GFX7, GFX10. 2130//===----------------------------------------------------------------------===// 2131 2132class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2133 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2134 let Inst{15} = ps.addr64; 2135 let Inst{22-19} = dfmt; 2136 let Inst{25-23} = nfmt; 2137} 2138 2139let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2140 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2141 def _ADDR64_gfx6_gfx7 : 2142 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2143 def _BOTHEN_gfx6_gfx7 : 2144 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2145 def _IDXEN_gfx6_gfx7 : 2146 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2147 def _OFFEN_gfx6_gfx7 : 2148 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2149 def _OFFSET_gfx6_gfx7 : 2150 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2151 } 2152} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2153 2154multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2155 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2156 2157defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2158defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2159defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2160defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2161defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2162defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2163defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2164defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2165 2166//===----------------------------------------------------------------------===// 2167// GFX8, GFX9 (VI). 2168//===----------------------------------------------------------------------===// 2169 2170class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2171 MUBUF_Real<ps>, 2172 Enc64, 2173 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2174 let AssemblerPredicate = isGFX8GFX9; 2175 let DecoderNamespace = "GFX8"; 2176 2177 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2178 let Inst{12} = ps.offen; 2179 let Inst{13} = ps.idxen; 2180 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2181 let Inst{16} = !if(ps.lds, 1, 0); 2182 let Inst{17} = !if(ps.has_slc, slc, ?); 2183 let Inst{24-18} = op; 2184 let Inst{31-26} = 0x38; //encoding 2185 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2186 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2187 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2188 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2189 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2190} 2191 2192multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2193 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2194 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2195 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2196 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2197} 2198 2199multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2200 2201 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2202 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2203 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2204 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2205 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2206 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2207 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2208 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2209 2210 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2211 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2212 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2213 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2214 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2215 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2216 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2217 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2218} 2219 2220class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2221 MUBUF_Real<ps>, 2222 Enc64, 2223 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2224 let AssemblerPredicate=HasUnpackedD16VMem; 2225 let DecoderNamespace="GFX80_UNPACKED"; 2226 2227 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2228 let Inst{12} = ps.offen; 2229 let Inst{13} = ps.idxen; 2230 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2231 let Inst{16} = !if(ps.lds, 1, 0); 2232 let Inst{17} = !if(ps.has_slc, slc, ?); 2233 let Inst{24-18} = op; 2234 let Inst{31-26} = 0x38; //encoding 2235 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2236 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2237 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2238 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2239 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2240} 2241 2242multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2243 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2244 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2245 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2246 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2247} 2248 2249multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2250 MUBUF_Real_AllAddr_vi<op> { 2251 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2252 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2253 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2254 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2255} 2256 2257defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2258defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2259defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2260defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2261defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2262defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2263defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2264defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2265let SubtargetPredicate = HasUnpackedD16VMem in { 2266 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2267 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2268 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2269 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2270 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2271 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2272 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2273 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2274} // End HasUnpackedD16VMem. 2275let SubtargetPredicate = HasPackedD16VMem in { 2276 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2277 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2278 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2279 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2280 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2281 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2282 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2283 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2284} // End HasPackedD16VMem. 2285defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2286defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2287defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2288defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2289defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2290defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2291defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2292defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2293defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2294defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2295defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2296defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2297defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2298defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2299defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2300defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2301 2302defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2303defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2304defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2305defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2306defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2307defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2308 2309defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2310defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2311 2312defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2313defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2314defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2315defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2316defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2317defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2318defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2319defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2320defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2321defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2322defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2323defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2324defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2325 2326defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2327defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2328defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2329defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2330defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2331defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2332defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2333defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2334defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2335defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2336defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2337defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2338defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2339 2340def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2341 2342def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2343def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2344 2345let SubtargetPredicate = HasAtomicFaddInsts in { 2346 2347defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2348defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2349 2350} // End SubtargetPredicate = HasAtomicFaddInsts 2351 2352class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2353 MTBUF_Real<ps>, 2354 Enc64, 2355 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2356 let AssemblerPredicate = isGFX8GFX9; 2357 let DecoderNamespace = "GFX8"; 2358 2359 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2360 let Inst{12} = ps.offen; 2361 let Inst{13} = ps.idxen; 2362 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2363 let Inst{18-15} = op; 2364 let Inst{22-19} = dfmt; 2365 let Inst{25-23} = nfmt; 2366 let Inst{31-26} = 0x3a; //encoding 2367 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2368 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2369 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2370 let Inst{54} = !if(ps.has_slc, slc, ?); 2371 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2372 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2373} 2374 2375multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2376 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2377 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2378 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2379 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2380} 2381 2382class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2383 MTBUF_Real<ps>, 2384 Enc64, 2385 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2386 let AssemblerPredicate=HasUnpackedD16VMem; 2387 let DecoderNamespace="GFX80_UNPACKED"; 2388 2389 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2390 let Inst{12} = ps.offen; 2391 let Inst{13} = ps.idxen; 2392 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2393 let Inst{18-15} = op; 2394 let Inst{22-19} = dfmt; 2395 let Inst{25-23} = nfmt; 2396 let Inst{31-26} = 0x3a; //encoding 2397 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2398 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2399 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2400 let Inst{54} = !if(ps.has_slc, slc, ?); 2401 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2402 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2403} 2404 2405multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2406 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2407 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2408 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2409 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2410} 2411 2412defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2413defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2414defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2415defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2416defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2417defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2418defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2419defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2420let SubtargetPredicate = HasUnpackedD16VMem in { 2421 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2422 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2423 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2424 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2425 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2426 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2427 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2428 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2429} // End HasUnpackedD16VMem. 2430let SubtargetPredicate = HasPackedD16VMem in { 2431 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2432 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2433 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2434 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2435 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2436 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2437 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2438 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2439} // End HasUnpackedD16VMem. 2440 2441def MUBUFInfoTable : GenericTable { 2442 let FilterClass = "MUBUF_Pseudo"; 2443 let CppTypeName = "MUBUFInfo"; 2444 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2445 2446 let PrimaryKey = ["Opcode"]; 2447 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2448} 2449 2450def getMUBUFInfoFromOpcode : SearchIndex { 2451 let Table = MUBUFInfoTable; 2452 let Key = ["Opcode"]; 2453} 2454 2455def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2456 let Table = MUBUFInfoTable; 2457 let Key = ["BaseOpcode", "elements"]; 2458} 2459 2460def MTBUFInfoTable : GenericTable { 2461 let FilterClass = "MTBUF_Pseudo"; 2462 let CppTypeName = "MTBUFInfo"; 2463 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2464 2465 let PrimaryKey = ["Opcode"]; 2466 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2467} 2468 2469def getMTBUFInfoFromOpcode : SearchIndex { 2470 let Table = MTBUFInfoTable; 2471 let Key = ["Opcode"]; 2472} 2473 2474def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2475 let Table = MTBUFInfoTable; 2476 let Key = ["BaseOpcode", "elements"]; 2477} 2478