1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 15def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 16 17def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 18def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 19def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 20def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">; 21def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">; 22 23class MubufLoad <SDPatternOperator op> : PatFrag < 24 (ops node:$ptr), (op node:$ptr), [{ 25 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 26 return AS == AMDGPUASI.GLOBAL_ADDRESS || 27 AS == AMDGPUASI.CONSTANT_ADDRESS; 28}]>; 29 30def mubuf_load : MubufLoad <load>; 31def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 32def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 33def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 34def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 35def mubuf_load_atomic : MubufLoad <atomic_load>; 36 37def BUFAddrKind { 38 int Offset = 0; 39 int OffEn = 1; 40 int IdxEn = 2; 41 int BothEn = 3; 42 int Addr64 = 4; 43} 44 45class getAddrName<int addrKind> { 46 string ret = 47 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 48 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 49 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 50 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 51 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 52 ""))))); 53} 54 55class MUBUFAddr64Table <bit is_addr64, string suffix = ""> { 56 bit IsAddr64 = is_addr64; 57 string OpName = NAME # suffix; 58} 59 60class MTBUFAddr64Table <bit is_addr64, string suffix = ""> { 61 bit IsAddr64 = is_addr64; 62 string OpName = NAME # suffix; 63} 64 65//===----------------------------------------------------------------------===// 66// MTBUF classes 67//===----------------------------------------------------------------------===// 68 69class MTBUF_Pseudo <string opName, dag outs, dag ins, 70 string asmOps, list<dag> pattern=[]> : 71 InstSI<outs, ins, "", pattern>, 72 SIMCInstr<opName, SIEncodingFamily.NONE> { 73 74 let isPseudo = 1; 75 let isCodeGenOnly = 1; 76 let Size = 8; 77 let UseNamedOperandTable = 1; 78 79 string Mnemonic = opName; 80 string AsmOperands = asmOps; 81 82 let VM_CNT = 1; 83 let EXP_CNT = 1; 84 let MTBUF = 1; 85 let Uses = [EXEC]; 86 let hasSideEffects = 0; 87 let SchedRW = [WriteVMEM]; 88 89 let AsmMatchConverter = "cvtMtbuf"; 90 91 bits<1> offen = 0; 92 bits<1> idxen = 0; 93 bits<1> addr64 = 0; 94 bits<1> has_vdata = 1; 95 bits<1> has_vaddr = 1; 96 bits<1> has_glc = 1; 97 bits<1> glc_value = 0; // the value for glc if no such operand 98 bits<4> dfmt_value = 1; // the value for dfmt if no such operand 99 bits<3> nfmt_value = 0; // the value for nfmt if no such operand 100 bits<1> has_srsrc = 1; 101 bits<1> has_soffset = 1; 102 bits<1> has_offset = 1; 103 bits<1> has_slc = 1; 104 bits<1> has_tfe = 1; 105 bits<1> has_dfmt = 1; 106 bits<1> has_nfmt = 1; 107} 108 109class MTBUF_Real <MTBUF_Pseudo ps> : 110 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 111 112 let isPseudo = 0; 113 let isCodeGenOnly = 0; 114 115 // copy relevant pseudo op flags 116 let SubtargetPredicate = ps.SubtargetPredicate; 117 let AsmMatchConverter = ps.AsmMatchConverter; 118 let Constraints = ps.Constraints; 119 let DisableEncoding = ps.DisableEncoding; 120 let TSFlags = ps.TSFlags; 121 122 bits<12> offset; 123 bits<1> glc; 124 bits<4> dfmt; 125 bits<3> nfmt; 126 bits<8> vaddr; 127 bits<8> vdata; 128 bits<7> srsrc; 129 bits<1> slc; 130 bits<1> tfe; 131 bits<8> soffset; 132} 133 134class getMTBUFInsDA<list<RegisterClass> vdataList, 135 list<RegisterClass> vaddrList=[]> { 136 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 137 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 138 dag InsNoData = !if(!empty(vaddrList), 139 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 140 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe), 141 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 142 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe) 143 ); 144 dag InsData = !if(!empty(vaddrList), 145 (ins vdataClass:$vdata, SReg_128:$srsrc, 146 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 147 slc:$slc, tfe:$tfe), 148 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 149 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 150 slc:$slc, tfe:$tfe) 151 ); 152 dag ret = !if(!empty(vdataList), InsNoData, InsData); 153} 154 155class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 156 dag ret = 157 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 158 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 159 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 160 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 161 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 162 (ins)))))); 163} 164 165class getMTBUFAsmOps<int addrKind> { 166 string Pfx = 167 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset", 168 !if(!eq(addrKind, BUFAddrKind.OffEn), 169 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen", 170 !if(!eq(addrKind, BUFAddrKind.IdxEn), 171 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen", 172 !if(!eq(addrKind, BUFAddrKind.BothEn), 173 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen", 174 !if(!eq(addrKind, BUFAddrKind.Addr64), 175 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64", 176 ""))))); 177 string ret = Pfx # "$offset"; 178} 179 180class MTBUF_SetupAddr<int addrKind> { 181 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 183 184 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 185 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 186 187 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 188 189 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 190} 191 192class MTBUF_Load_Pseudo <string opName, 193 int addrKind, 194 RegisterClass vdataClass, 195 list<dag> pattern=[], 196 // Workaround bug bz30254 197 int addrKindCopy = addrKind> 198 : MTBUF_Pseudo<opName, 199 (outs vdataClass:$vdata), 200 getMTBUFIns<addrKindCopy>.ret, 201 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 202 pattern>, 203 MTBUF_SetupAddr<addrKindCopy> { 204 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 205 let mayLoad = 1; 206 let mayStore = 0; 207} 208 209multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 210 ValueType load_vt = i32, 211 SDPatternOperator ld = null_frag> { 212 213 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 214 [(set load_vt:$vdata, 215 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt, 216 i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 217 MTBUFAddr64Table<0>; 218 219 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 220 [(set load_vt:$vdata, 221 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 222 i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 223 MTBUFAddr64Table<1>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 list<dag> pattern=[], 241 // Workaround bug bz30254 242 int addrKindCopy = addrKind, 243 RegisterClass vdataClassCopy = vdataClass> 244 : MTBUF_Pseudo<opName, 245 (outs), 246 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 247 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 248 pattern>, 249 MTBUF_SetupAddr<addrKindCopy> { 250 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 251 let mayLoad = 0; 252 let mayStore = 1; 253} 254 255multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 256 ValueType store_vt = i32, 257 SDPatternOperator st = null_frag> { 258 259 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 260 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 261 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 262 i1:$slc, i1:$tfe))]>, 263 MTBUFAddr64Table<0>; 264 265 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 266 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 267 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 268 i1:$slc, i1:$tfe))]>, 269 MTBUFAddr64Table<1>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUF_Pseudo <string opName, dag outs, dag ins, 289 string asmOps, list<dag> pattern=[]> : 290 InstSI<outs, ins, "", pattern>, 291 SIMCInstr<opName, SIEncodingFamily.NONE> { 292 293 let isPseudo = 1; 294 let isCodeGenOnly = 1; 295 let Size = 8; 296 let UseNamedOperandTable = 1; 297 298 string Mnemonic = opName; 299 string AsmOperands = asmOps; 300 301 let VM_CNT = 1; 302 let EXP_CNT = 1; 303 let MUBUF = 1; 304 let Uses = [EXEC]; 305 let hasSideEffects = 0; 306 let SchedRW = [WriteVMEM]; 307 308 let AsmMatchConverter = "cvtMubuf"; 309 310 bits<1> offen = 0; 311 bits<1> idxen = 0; 312 bits<1> addr64 = 0; 313 bits<1> has_vdata = 1; 314 bits<1> has_vaddr = 1; 315 bits<1> has_glc = 1; 316 bits<1> glc_value = 0; // the value for glc if no such operand 317 bits<1> has_srsrc = 1; 318 bits<1> has_soffset = 1; 319 bits<1> has_offset = 1; 320 bits<1> has_slc = 1; 321 bits<1> has_tfe = 1; 322} 323 324class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 325 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 326 327 let isPseudo = 0; 328 let isCodeGenOnly = 0; 329 330 // copy relevant pseudo op flags 331 let SubtargetPredicate = ps.SubtargetPredicate; 332 let AsmMatchConverter = ps.AsmMatchConverter; 333 let Constraints = ps.Constraints; 334 let DisableEncoding = ps.DisableEncoding; 335 let TSFlags = ps.TSFlags; 336 337 bits<12> offset; 338 bits<1> glc; 339 bits<1> lds = 0; 340 bits<8> vaddr; 341 bits<8> vdata; 342 bits<7> srsrc; 343 bits<1> slc; 344 bits<1> tfe; 345 bits<8> soffset; 346} 347 348 349// For cache invalidation instructions. 350class MUBUF_Invalidate <string opName, SDPatternOperator node> : 351 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 352 353 let AsmMatchConverter = ""; 354 355 let hasSideEffects = 1; 356 let mayStore = 1; 357 358 // Set everything to 0. 359 let offen = 0; 360 let idxen = 0; 361 let addr64 = 0; 362 let has_vdata = 0; 363 let has_vaddr = 0; 364 let has_glc = 0; 365 let glc_value = 0; 366 let has_srsrc = 0; 367 let has_soffset = 0; 368 let has_offset = 0; 369 let has_slc = 0; 370 let has_tfe = 0; 371} 372 373class getMUBUFInsDA<list<RegisterClass> vdataList, 374 list<RegisterClass> vaddrList=[]> { 375 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 376 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 377 dag InsNoData = !if(!empty(vaddrList), 378 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 379 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 380 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 381 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 382 ); 383 dag InsData = !if(!empty(vaddrList), 384 (ins vdataClass:$vdata, SReg_128:$srsrc, 385 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 386 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 387 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 388 ); 389 dag ret = !if(!empty(vdataList), InsNoData, InsData); 390} 391 392class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 393 dag ret = 394 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret, 395 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 396 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 397 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 398 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 399 (ins)))))); 400} 401 402class getMUBUFAsmOps<int addrKind> { 403 string Pfx = 404 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 405 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 406 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 407 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 408 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 409 ""))))); 410 string ret = Pfx # "$offset"; 411} 412 413class MUBUF_SetupAddr<int addrKind> { 414 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 415 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 416 417 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 418 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 419 420 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 421 422 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 423} 424 425class MUBUF_Load_Pseudo <string opName, 426 int addrKind, 427 RegisterClass vdataClass, 428 bit HasTiedDest = 0, 429 list<dag> pattern=[], 430 // Workaround bug bz30254 431 int addrKindCopy = addrKind> 432 : MUBUF_Pseudo<opName, 433 (outs vdataClass:$vdata), 434 !con(getMUBUFIns<addrKindCopy>.ret, !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 435 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 436 pattern>, 437 MUBUF_SetupAddr<addrKindCopy> { 438 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 439 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 440 let mayLoad = 1; 441 let mayStore = 0; 442 let maybeAtomic = 1; 443} 444 445// FIXME: tfe can't be an operand because it requires a separate 446// opcode because it needs an N+1 register class dest register. 447multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 448 ValueType load_vt = i32, 449 SDPatternOperator ld = null_frag, 450 bit TiedDest = 0> { 451 452 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 453 TiedDest, 454 [(set load_vt:$vdata, 455 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 456 MUBUFAddr64Table<0>; 457 458 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 459 TiedDest, 460 [(set load_vt:$vdata, 461 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 462 MUBUFAddr64Table<1>; 463 464 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>; 465 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>; 466 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>; 467 468 let DisableWQM = 1 in { 469 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest>; 470 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>; 471 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>; 472 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>; 473 } 474} 475 476class MUBUF_Store_Pseudo <string opName, 477 int addrKind, 478 RegisterClass vdataClass, 479 list<dag> pattern=[], 480 // Workaround bug bz30254 481 int addrKindCopy = addrKind, 482 RegisterClass vdataClassCopy = vdataClass> 483 : MUBUF_Pseudo<opName, 484 (outs), 485 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 486 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 487 pattern>, 488 MUBUF_SetupAddr<addrKindCopy> { 489 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 490 let mayLoad = 0; 491 let mayStore = 1; 492 let maybeAtomic = 1; 493} 494 495multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 496 ValueType store_vt = i32, 497 SDPatternOperator st = null_frag> { 498 499 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 500 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 501 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 502 MUBUFAddr64Table<0>; 503 504 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 505 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 506 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 507 MUBUFAddr64Table<1>; 508 509 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 510 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 511 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 512 513 let DisableWQM = 1 in { 514 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 515 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 516 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 517 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 518 } 519} 520 521 522class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 523 list<RegisterClass> vaddrList=[]> { 524 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 525 dag ret = !if(vdata_in, 526 !if(!empty(vaddrList), 527 (ins vdataClass:$vdata_in, 528 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 529 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 530 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 531 ), 532 !if(!empty(vaddrList), 533 (ins vdataClass:$vdata, 534 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 535 (ins vdataClass:$vdata, vaddrClass:$vaddr, 536 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 537 )); 538} 539 540class getMUBUFAtomicIns<int addrKind, 541 RegisterClass vdataClass, 542 bit vdata_in, 543 // Workaround bug bz30254 544 RegisterClass vdataClassCopy=vdataClass> { 545 dag ret = 546 !if(!eq(addrKind, BUFAddrKind.Offset), 547 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 548 !if(!eq(addrKind, BUFAddrKind.OffEn), 549 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 550 !if(!eq(addrKind, BUFAddrKind.IdxEn), 551 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 552 !if(!eq(addrKind, BUFAddrKind.BothEn), 553 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 554 !if(!eq(addrKind, BUFAddrKind.Addr64), 555 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 556 (ins)))))); 557} 558 559class MUBUF_Atomic_Pseudo<string opName, 560 int addrKind, 561 dag outs, 562 dag ins, 563 string asmOps, 564 list<dag> pattern=[], 565 // Workaround bug bz30254 566 int addrKindCopy = addrKind> 567 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 568 MUBUF_SetupAddr<addrKindCopy> { 569 let mayStore = 1; 570 let mayLoad = 1; 571 let hasPostISelHook = 1; 572 let hasSideEffects = 1; 573 let DisableWQM = 1; 574 let has_glc = 0; 575 let has_tfe = 0; 576 let maybeAtomic = 1; 577} 578 579class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 580 RegisterClass vdataClass, 581 list<dag> pattern=[], 582 // Workaround bug bz30254 583 int addrKindCopy = addrKind, 584 RegisterClass vdataClassCopy = vdataClass> 585 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 586 (outs), 587 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 588 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 589 pattern>, 590 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 591 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 592 let glc_value = 0; 593 let AsmMatchConverter = "cvtMubufAtomic"; 594} 595 596class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 597 RegisterClass vdataClass, 598 list<dag> pattern=[], 599 // Workaround bug bz30254 600 int addrKindCopy = addrKind, 601 RegisterClass vdataClassCopy = vdataClass> 602 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 603 (outs vdataClassCopy:$vdata), 604 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 605 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 606 pattern>, 607 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 608 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 609 let glc_value = 1; 610 let Constraints = "$vdata = $vdata_in"; 611 let DisableEncoding = "$vdata_in"; 612 let AsmMatchConverter = "cvtMubufAtomicReturn"; 613} 614 615multiclass MUBUF_Pseudo_Atomics <string opName, 616 RegisterClass vdataClass, 617 ValueType vdataType, 618 SDPatternOperator atomic> { 619 620 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 621 MUBUFAddr64Table <0>; 622 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 623 MUBUFAddr64Table <1>; 624 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 625 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 626 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 627 628 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 629 [(set vdataType:$vdata, 630 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 631 vdataType:$vdata_in))]>, 632 MUBUFAddr64Table <0, "_RTN">; 633 634 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 635 [(set vdataType:$vdata, 636 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 637 vdataType:$vdata_in))]>, 638 MUBUFAddr64Table <1, "_RTN">; 639 640 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 641 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 642 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 643} 644 645 646//===----------------------------------------------------------------------===// 647// MUBUF Instructions 648//===----------------------------------------------------------------------===// 649 650defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads < 651 "buffer_load_format_x", VGPR_32 652>; 653defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 654 "buffer_load_format_xy", VReg_64 655>; 656defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 657 "buffer_load_format_xyz", VReg_96 658>; 659defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 660 "buffer_load_format_xyzw", VReg_128 661>; 662defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 663 "buffer_store_format_x", VGPR_32 664>; 665defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 666 "buffer_store_format_xy", VReg_64 667>; 668defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 669 "buffer_store_format_xyz", VReg_96 670>; 671defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 672 "buffer_store_format_xyzw", VReg_128 673>; 674defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads < 675 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 676>; 677defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads < 678 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 679>; 680defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads < 681 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 682>; 683defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads < 684 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 685>; 686defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads < 687 "buffer_load_dword", VGPR_32, i32, mubuf_load 688>; 689defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 690 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 691>; 692defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 693 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 694>; 695defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 696 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 697>; 698defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 699 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 700>; 701defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 702 "buffer_store_short", VGPR_32, i32, truncstorei16_global 703>; 704defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 705 "buffer_store_dword", VGPR_32, i32, store_global 706>; 707defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 708 "buffer_store_dwordx2", VReg_64, v2i32, store_global 709>; 710defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 711 "buffer_store_dwordx3", VReg_96, untyped, store_global 712>; 713defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 714 "buffer_store_dwordx4", VReg_128, v4i32, store_global 715>; 716defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 717 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 718>; 719defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 720 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 721>; 722defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 723 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 724>; 725defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 726 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 727>; 728defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 729 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 730>; 731defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 732 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 733>; 734defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 735 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 736>; 737defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 738 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 739>; 740defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 741 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 742>; 743defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 744 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 745>; 746defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 747 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 748>; 749defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 750 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 751>; 752defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 753 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 754>; 755defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 756 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 757>; 758defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 759 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 760>; 761defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 762 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 763>; 764defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 765 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 766>; 767defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 768 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 769>; 770defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 771 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 772>; 773defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 774 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 775>; 776defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 777 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 778>; 779defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 780 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 781>; 782defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 783 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 784>; 785defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 786 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 787>; 788defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 789 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 790>; 791defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 792 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 793>; 794 795let SubtargetPredicate = isSI in { // isn't on CI & VI 796/* 797defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 798defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 799defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 800defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 801defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 802defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 803defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 804defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 805*/ 806 807def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 808 int_amdgcn_buffer_wbinvl1_sc>; 809} 810 811let SubtargetPredicate = HasD16LoadStore in { 812 813defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 814 "buffer_load_ubyte_d16", VGPR_32, i32 815>; 816 817defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 818 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 819>; 820 821defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 822 "buffer_load_sbyte_d16", VGPR_32, i32 823>; 824 825defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 826 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 827>; 828 829defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 830 "buffer_load_short_d16", VGPR_32, i32 831>; 832 833defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 834 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 835>; 836 837defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 838 "buffer_store_byte_d16_hi", VGPR_32, i32 839>; 840 841defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 842 "buffer_store_short_d16_hi", VGPR_32, i32 843>; 844 845} // End HasD16LoadStore 846 847def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 848 int_amdgcn_buffer_wbinvl1>; 849 850//===----------------------------------------------------------------------===// 851// MTBUF Instructions 852//===----------------------------------------------------------------------===// 853 854defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 855defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 856defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>; 857defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 858defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 859defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 860defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>; 861defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 862 863let SubtargetPredicate = isCIVI in { 864 865//===----------------------------------------------------------------------===// 866// Instruction definitions for CI and newer. 867//===----------------------------------------------------------------------===// 868// Remaining instructions: 869// BUFFER_LOAD_DWORDX3 870// BUFFER_STORE_DWORDX3 871 872def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 873 int_amdgcn_buffer_wbinvl1_vol>; 874 875} // End let SubtargetPredicate = isCIVI 876 877//===----------------------------------------------------------------------===// 878// MUBUF Patterns 879//===----------------------------------------------------------------------===// 880 881// Offset in an 32-bit VGPR 882def : GCNPat < 883 (SIload_constant v4i32:$sbase, i32:$voff), 884 (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0) 885>; 886 887 888//===----------------------------------------------------------------------===// 889// buffer_load/store_format patterns 890//===----------------------------------------------------------------------===// 891 892multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 893 string opcode> { 894 def : GCNPat< 895 (vt (name v4i32:$rsrc, 0, 896 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 897 imm:$glc, imm:$slc)), 898 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 899 (as_i1imm $glc), (as_i1imm $slc), 0) 900 >; 901 902 def : GCNPat< 903 (vt (name v4i32:$rsrc, i32:$vindex, 904 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 905 imm:$glc, imm:$slc)), 906 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 907 (as_i1imm $glc), (as_i1imm $slc), 0) 908 >; 909 910 def : GCNPat< 911 (vt (name v4i32:$rsrc, 0, 912 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 913 imm:$glc, imm:$slc)), 914 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 915 (as_i1imm $glc), (as_i1imm $slc), 0) 916 >; 917 918 def : GCNPat< 919 (vt (name v4i32:$rsrc, i32:$vindex, 920 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 921 imm:$glc, imm:$slc)), 922 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 923 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 924 $rsrc, $soffset, (as_i16imm $offset), 925 (as_i1imm $glc), (as_i1imm $slc), 0) 926 >; 927} 928 929defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 930defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 931defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 932defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 933defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 934defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 935 936multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 937 string opcode> { 938 def : GCNPat< 939 (name vt:$vdata, v4i32:$rsrc, 0, 940 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 941 imm:$glc, imm:$slc), 942 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 943 (as_i1imm $glc), (as_i1imm $slc), 0) 944 >; 945 946 def : GCNPat< 947 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 948 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 949 imm:$glc, imm:$slc), 950 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 951 (as_i16imm $offset), (as_i1imm $glc), 952 (as_i1imm $slc), 0) 953 >; 954 955 def : GCNPat< 956 (name vt:$vdata, v4i32:$rsrc, 0, 957 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 958 imm:$glc, imm:$slc), 959 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 960 (as_i16imm $offset), (as_i1imm $glc), 961 (as_i1imm $slc), 0) 962 >; 963 964 def : GCNPat< 965 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 966 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 967 imm:$glc, imm:$slc), 968 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 969 $vdata, 970 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 971 $rsrc, $soffset, (as_i16imm $offset), 972 (as_i1imm $glc), (as_i1imm $slc), 0) 973 >; 974} 975 976defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 977defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 978defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 979defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">; 980defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 981defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 982 983//===----------------------------------------------------------------------===// 984// buffer_atomic patterns 985//===----------------------------------------------------------------------===// 986 987multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 988 def : GCNPat< 989 (name i32:$vdata_in, v4i32:$rsrc, 0, 990 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 991 imm:$slc), 992 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 993 (as_i16imm $offset), (as_i1imm $slc)) 994 >; 995 996 def : GCNPat< 997 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 998 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 999 imm:$slc), 1000 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1001 (as_i16imm $offset), (as_i1imm $slc)) 1002 >; 1003 1004 def : GCNPat< 1005 (name i32:$vdata_in, v4i32:$rsrc, 0, 1006 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1007 imm:$slc), 1008 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1009 (as_i16imm $offset), (as_i1imm $slc)) 1010 >; 1011 1012 def : GCNPat< 1013 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1014 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1015 imm:$slc), 1016 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1017 $vdata_in, 1018 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1019 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)) 1020 >; 1021} 1022 1023defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 1024defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">; 1025defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 1026defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 1027defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 1028defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 1029defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 1030defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">; 1031defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">; 1032defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 1033 1034def : GCNPat< 1035 (int_amdgcn_buffer_atomic_cmpswap 1036 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1037 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1038 imm:$slc), 1039 (EXTRACT_SUBREG 1040 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1041 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1042 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1043 sub0) 1044>; 1045 1046def : GCNPat< 1047 (int_amdgcn_buffer_atomic_cmpswap 1048 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1049 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1050 imm:$slc), 1051 (EXTRACT_SUBREG 1052 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1053 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1054 $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1055 sub0) 1056>; 1057 1058def : GCNPat< 1059 (int_amdgcn_buffer_atomic_cmpswap 1060 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1061 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1062 imm:$slc), 1063 (EXTRACT_SUBREG 1064 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1065 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1066 $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1067 sub0) 1068>; 1069 1070def : GCNPat< 1071 (int_amdgcn_buffer_atomic_cmpswap 1072 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1073 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1074 imm:$slc), 1075 (EXTRACT_SUBREG 1076 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1077 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1078 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1079 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1080 sub0) 1081>; 1082 1083 1084class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1085 PatFrag constant_ld> : GCNPat < 1086 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1087 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1088 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1089 >; 1090 1091multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1092 ValueType vt, PatFrag atomic_ld> { 1093 def : GCNPat < 1094 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1095 i16:$offset, i1:$slc))), 1096 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1097 >; 1098 1099 def : GCNPat < 1100 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1101 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1102 >; 1103} 1104 1105let SubtargetPredicate = isSICI in { 1106def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1107def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1108def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1109def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1110 1111defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1112defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1113} // End SubtargetPredicate = isSICI 1114 1115multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1116 PatFrag ld> { 1117 1118 def : GCNPat < 1119 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1120 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1121 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1122 >; 1123} 1124 1125let OtherPredicates = [Has16BitInsts] in { 1126 1127defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1128defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1129defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1130defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1131 1132defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1133 1134} // End OtherPredicates = [Has16BitInsts] 1135 1136multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1137 MUBUF_Pseudo InstrOffset, 1138 ValueType vt, PatFrag ld> { 1139 def : GCNPat < 1140 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1141 i32:$soffset, u16imm:$offset))), 1142 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1143 >; 1144 1145 def : GCNPat < 1146 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1147 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1148 >; 1149} 1150 1151// XXX - Is it possible to have a complex pattern in a PatFrag? 1152multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen, 1153 MUBUF_Pseudo InstrOffset, 1154 ValueType vt, PatFrag ld> { 1155 def : GCNPat < 1156 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1157 i32:$soffset, u16imm:$offset)))), 1158 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1159 >; 1160 1161 def : GCNPat < 1162 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1163 i32:$soffset, u16imm:$offset)))))), 1164 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1165 >; 1166 1167 1168 def : GCNPat < 1169 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))), 1170 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1171 >; 1172 1173 def : GCNPat < 1174 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))), 1175 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1176 >; 1177} 1178 1179defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1180defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1181defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1182defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1183defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1184defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1185defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1186defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1187defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1188defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1189 1190let OtherPredicates = [HasD16LoadStore] in { 1191defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>; 1192defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>; 1193defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>; 1194} 1195 1196// BUFFER_LOAD_DWORD*, addr64=0 1197multiclass MUBUF_Load_Dword <ValueType vt, 1198 MUBUF_Pseudo offset, 1199 MUBUF_Pseudo offen, 1200 MUBUF_Pseudo idxen, 1201 MUBUF_Pseudo bothen> { 1202 1203 def : GCNPat < 1204 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset, 1205 imm:$offset, 0, 0, imm:$glc, imm:$slc, 1206 imm:$tfe)), 1207 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1208 (as_i1imm $slc), (as_i1imm $tfe)) 1209 >; 1210 1211 def : GCNPat < 1212 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1213 imm:$offset, 1, 0, imm:$glc, imm:$slc, 1214 imm:$tfe)), 1215 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1216 (as_i1imm $tfe)) 1217 >; 1218 1219 def : GCNPat < 1220 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1221 imm:$offset, 0, 1, imm:$glc, imm:$slc, 1222 imm:$tfe)), 1223 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1224 (as_i1imm $slc), (as_i1imm $tfe)) 1225 >; 1226 1227 def : GCNPat < 1228 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset, 1229 imm:$offset, 1, 1, imm:$glc, imm:$slc, 1230 imm:$tfe)), 1231 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1232 (as_i1imm $tfe)) 1233 >; 1234} 1235 1236defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN, 1237 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>; 1238defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN, 1239 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>; 1240defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN, 1241 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>; 1242 1243multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1244 ValueType vt, PatFrag atomic_st> { 1245 // Store follows atomic op convention so address is forst 1246 def : GCNPat < 1247 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1248 i16:$offset, i1:$slc), vt:$val), 1249 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1250 >; 1251 1252 def : GCNPat < 1253 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1254 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1255 >; 1256} 1257let SubtargetPredicate = isSICI in { 1258defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1259defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1260} // End Predicates = isSICI 1261 1262 1263multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1264 PatFrag st> { 1265 1266 def : GCNPat < 1267 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1268 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1269 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1270 >; 1271} 1272 1273defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1274defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1275 1276multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1277 MUBUF_Pseudo InstrOffset, 1278 ValueType vt, PatFrag st> { 1279 def : GCNPat < 1280 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1281 i32:$soffset, u16imm:$offset)), 1282 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1283 >; 1284 1285 def : GCNPat < 1286 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1287 u16imm:$offset)), 1288 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1289 >; 1290} 1291 1292defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1293defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1294defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1295defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1296defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1297defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1298defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1299 1300 1301let OtherPredicates = [HasD16LoadStore] in { 1302 // Hiding the extract high pattern in the PatFrag seems to not 1303 // automatically increase the complexity. 1304let AddedComplexity = 1 in { 1305defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1306defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1307} 1308} 1309 1310//===----------------------------------------------------------------------===// 1311// MTBUF Patterns 1312//===----------------------------------------------------------------------===// 1313 1314//===----------------------------------------------------------------------===// 1315// tbuffer_load/store_format patterns 1316//===----------------------------------------------------------------------===// 1317 1318multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1319 string opcode> { 1320 def : GCNPat< 1321 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1322 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1323 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1324 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1325 >; 1326 1327 def : GCNPat< 1328 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1329 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1330 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1331 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1332 >; 1333 1334 def : GCNPat< 1335 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1336 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1337 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1338 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1339 >; 1340 1341 def : GCNPat< 1342 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1343 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1344 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1345 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1346 $rsrc, $soffset, (as_i16imm $offset), 1347 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1348 >; 1349} 1350 1351defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1352defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1353defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1354defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1355defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1356defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1357 1358multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1359 string opcode> { 1360 def : GCNPat< 1361 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1362 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1363 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1364 (as_i16imm $offset), (as_i8imm $dfmt), 1365 (as_i8imm $nfmt), (as_i1imm $glc), 1366 (as_i1imm $slc), 0) 1367 >; 1368 1369 def : GCNPat< 1370 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1371 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1372 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1373 (as_i16imm $offset), (as_i8imm $dfmt), 1374 (as_i8imm $nfmt), (as_i1imm $glc), 1375 (as_i1imm $slc), 0) 1376 >; 1377 1378 def : GCNPat< 1379 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1380 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1381 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1382 (as_i16imm $offset), (as_i8imm $dfmt), 1383 (as_i8imm $nfmt), (as_i1imm $glc), 1384 (as_i1imm $slc), 0) 1385 >; 1386 1387 def : GCNPat< 1388 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1389 imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1390 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1391 $vdata, 1392 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1393 $rsrc, $soffset, (as_i16imm $offset), 1394 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1395 >; 1396} 1397 1398defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1399defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1400defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">; 1401defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1402defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1403defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1404defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">; 1405defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1406 1407//===----------------------------------------------------------------------===// 1408// Target instructions, move to the appropriate target TD file 1409//===----------------------------------------------------------------------===// 1410 1411//===----------------------------------------------------------------------===// 1412// SI 1413//===----------------------------------------------------------------------===// 1414 1415class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1416 MUBUF_Real<op, ps>, 1417 Enc64, 1418 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1419 let AssemblerPredicate=isSICI; 1420 let DecoderNamespace="SICI"; 1421 1422 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1423 let Inst{12} = ps.offen; 1424 let Inst{13} = ps.idxen; 1425 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1426 let Inst{15} = ps.addr64; 1427 let Inst{16} = lds; 1428 let Inst{24-18} = op; 1429 let Inst{31-26} = 0x38; //encoding 1430 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1431 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1432 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1433 let Inst{54} = !if(ps.has_slc, slc, ?); 1434 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1435 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1436} 1437 1438multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1439 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1440 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1441 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1442 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1443 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1444} 1445 1446multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1447 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1448 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1449 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1450 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1451 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1452} 1453 1454defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_si <0x00>; 1455defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1456defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1457defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1458defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1459defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1460defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1461defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1462defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_si <0x08>; 1463defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_si <0x09>; 1464defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_si <0x0a>; 1465defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_si <0x0b>; 1466defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_si <0x0c>; 1467defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1468defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1469defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1470defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1471defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1472defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1473defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1474defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1475defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1476 1477defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1478defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1479defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1480defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1481//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1482defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1483defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1484defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1485defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1486defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1487defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1488defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1489defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1490defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1491 1492//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1493//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1494//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1495defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1496defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1497defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1498defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1499//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1500defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1501defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1502defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1503defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1504defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1505defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1506defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1507defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1508defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1509// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1510//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1511//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1512//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1513 1514def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1515def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1516 1517class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1518 MTBUF_Real<ps>, 1519 Enc64, 1520 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1521 let AssemblerPredicate=isSICI; 1522 let DecoderNamespace="SICI"; 1523 1524 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1525 let Inst{12} = ps.offen; 1526 let Inst{13} = ps.idxen; 1527 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1528 let Inst{15} = ps.addr64; 1529 let Inst{18-16} = op; 1530 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 1531 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 1532 let Inst{31-26} = 0x3a; //encoding 1533 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1534 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1535 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1536 let Inst{54} = !if(ps.has_slc, slc, ?); 1537 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1538 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1539} 1540 1541multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1542 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1543 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1544 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1545 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1546 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1547} 1548 1549defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1550defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1551//defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1552defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1553defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1554defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1555defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1556defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1557 1558//===----------------------------------------------------------------------===// 1559// CI 1560//===----------------------------------------------------------------------===// 1561 1562class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1563 MUBUF_Real_si<op, ps> { 1564 let AssemblerPredicate=isCIOnly; 1565 let DecoderNamespace="CI"; 1566} 1567 1568def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1569 1570 1571//===----------------------------------------------------------------------===// 1572// VI 1573//===----------------------------------------------------------------------===// 1574 1575class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1576 MUBUF_Real<op, ps>, 1577 Enc64, 1578 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1579 let AssemblerPredicate=isVI; 1580 let DecoderNamespace="VI"; 1581 1582 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1583 let Inst{12} = ps.offen; 1584 let Inst{13} = ps.idxen; 1585 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1586 let Inst{16} = lds; 1587 let Inst{17} = !if(ps.has_slc, slc, ?); 1588 let Inst{24-18} = op; 1589 let Inst{31-26} = 0x38; //encoding 1590 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1591 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1592 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1593 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1594 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1595} 1596 1597multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1598 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1599 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1600 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1601 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1602} 1603 1604multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1605 MUBUF_Real_AllAddr_vi<op> { 1606 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1607 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1608 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1609 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1610} 1611 1612defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_vi <0x00>; 1613defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1614defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1615defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1616defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1617defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1618defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1619defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1620defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_vi <0x10>; 1621defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_vi <0x11>; 1622defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_vi <0x12>; 1623defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_vi <0x13>; 1624defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_vi <0x14>; 1625defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 1626defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 1627defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 1628defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1629defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1630defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1631defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1632defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1633defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1634defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1635defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1636 1637defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1638defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1639defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1640defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1641defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1642defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1643 1644defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1645defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1646defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1647defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1648defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1649defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1650defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1651defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1652defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1653defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1654defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1655defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1656defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1657 1658defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1659defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1660defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1661defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1662defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1663defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1664defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1665defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1666defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1667defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1668defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1669defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1670defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1671 1672def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 1673def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 1674 1675class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 1676 MTBUF_Real<ps>, 1677 Enc64, 1678 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1679 let AssemblerPredicate=isVI; 1680 let DecoderNamespace="VI"; 1681 1682 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1683 let Inst{12} = ps.offen; 1684 let Inst{13} = ps.idxen; 1685 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1686 let Inst{18-15} = op; 1687 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 1688 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 1689 let Inst{31-26} = 0x3a; //encoding 1690 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1691 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1692 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1693 let Inst{54} = !if(ps.has_slc, slc, ?); 1694 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1695 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1696} 1697 1698multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 1699 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1700 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1701 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1702 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1703} 1704 1705defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0>; 1706defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <1>; 1707//defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <2>; 1708defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <3>; 1709defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <4>; 1710defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <5>; 1711defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <6>; 1712defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <7>; 1713