1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 15def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 16 17def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 18def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 19def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 20def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">; 21def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">; 22 23class MubufLoad <SDPatternOperator op> : PatFrag < 24 (ops node:$ptr), (op node:$ptr), [{ 25 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 26 return AS == AMDGPUASI.GLOBAL_ADDRESS || 27 AS == AMDGPUASI.CONSTANT_ADDRESS; 28}]>; 29 30def mubuf_load : MubufLoad <load>; 31def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 32def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 33def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 34def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 35def mubuf_load_atomic : MubufLoad <atomic_load>; 36 37def BUFAddrKind { 38 int Offset = 0; 39 int OffEn = 1; 40 int IdxEn = 2; 41 int BothEn = 3; 42 int Addr64 = 4; 43} 44 45class getAddrName<int addrKind> { 46 string ret = 47 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 48 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 49 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 50 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 51 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 52 ""))))); 53} 54 55class MUBUFAddr64Table <bit is_addr64, string suffix = ""> { 56 bit IsAddr64 = is_addr64; 57 string OpName = NAME # suffix; 58} 59 60class MTBUFAddr64Table <bit is_addr64, string suffix = ""> { 61 bit IsAddr64 = is_addr64; 62 string OpName = NAME # suffix; 63} 64 65//===----------------------------------------------------------------------===// 66// MTBUF classes 67//===----------------------------------------------------------------------===// 68 69class MTBUF_Pseudo <string opName, dag outs, dag ins, 70 string asmOps, list<dag> pattern=[]> : 71 InstSI<outs, ins, "", pattern>, 72 SIMCInstr<opName, SIEncodingFamily.NONE> { 73 74 let isPseudo = 1; 75 let isCodeGenOnly = 1; 76 let Size = 8; 77 let UseNamedOperandTable = 1; 78 79 string Mnemonic = opName; 80 string AsmOperands = asmOps; 81 82 let VM_CNT = 1; 83 let EXP_CNT = 1; 84 let MTBUF = 1; 85 let Uses = [EXEC]; 86 let hasSideEffects = 0; 87 let SchedRW = [WriteVMEM]; 88 89 let AsmMatchConverter = "cvtMtbuf"; 90 91 bits<1> offen = 0; 92 bits<1> idxen = 0; 93 bits<1> addr64 = 0; 94 bits<1> has_vdata = 1; 95 bits<1> has_vaddr = 1; 96 bits<1> has_glc = 1; 97 bits<1> glc_value = 0; // the value for glc if no such operand 98 bits<4> dfmt_value = 1; // the value for dfmt if no such operand 99 bits<3> nfmt_value = 0; // the value for nfmt if no such operand 100 bits<1> has_srsrc = 1; 101 bits<1> has_soffset = 1; 102 bits<1> has_offset = 1; 103 bits<1> has_slc = 1; 104 bits<1> has_tfe = 1; 105 bits<1> has_dfmt = 1; 106 bits<1> has_nfmt = 1; 107} 108 109class MTBUF_Real <MTBUF_Pseudo ps> : 110 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 111 112 let isPseudo = 0; 113 let isCodeGenOnly = 0; 114 115 // copy relevant pseudo op flags 116 let SubtargetPredicate = ps.SubtargetPredicate; 117 let AsmMatchConverter = ps.AsmMatchConverter; 118 let Constraints = ps.Constraints; 119 let DisableEncoding = ps.DisableEncoding; 120 let TSFlags = ps.TSFlags; 121 122 bits<12> offset; 123 bits<1> glc; 124 bits<4> dfmt; 125 bits<3> nfmt; 126 bits<8> vaddr; 127 bits<8> vdata; 128 bits<7> srsrc; 129 bits<1> slc; 130 bits<1> tfe; 131 bits<8> soffset; 132} 133 134class getMTBUFInsDA<list<RegisterClass> vdataList, 135 list<RegisterClass> vaddrList=[]> { 136 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 137 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 138 dag InsNoData = !if(!empty(vaddrList), 139 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 140 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe), 141 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 142 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe) 143 ); 144 dag InsData = !if(!empty(vaddrList), 145 (ins vdataClass:$vdata, SReg_128:$srsrc, 146 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 147 slc:$slc, tfe:$tfe), 148 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 149 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 150 slc:$slc, tfe:$tfe) 151 ); 152 dag ret = !if(!empty(vdataList), InsNoData, InsData); 153} 154 155class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 156 dag ret = 157 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 158 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 159 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 160 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 161 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 162 (ins)))))); 163} 164 165class getMTBUFAsmOps<int addrKind> { 166 string Pfx = 167 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset", 168 !if(!eq(addrKind, BUFAddrKind.OffEn), 169 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen", 170 !if(!eq(addrKind, BUFAddrKind.IdxEn), 171 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen", 172 !if(!eq(addrKind, BUFAddrKind.BothEn), 173 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen", 174 !if(!eq(addrKind, BUFAddrKind.Addr64), 175 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64", 176 ""))))); 177 string ret = Pfx # "$offset"; 178} 179 180class MTBUF_SetupAddr<int addrKind> { 181 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 183 184 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 185 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 186 187 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 188 189 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 190} 191 192class MTBUF_Load_Pseudo <string opName, 193 int addrKind, 194 RegisterClass vdataClass, 195 list<dag> pattern=[], 196 // Workaround bug bz30254 197 int addrKindCopy = addrKind> 198 : MTBUF_Pseudo<opName, 199 (outs vdataClass:$vdata), 200 getMTBUFIns<addrKindCopy>.ret, 201 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 202 pattern>, 203 MTBUF_SetupAddr<addrKindCopy> { 204 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 205 let mayLoad = 1; 206 let mayStore = 0; 207} 208 209multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 210 ValueType load_vt = i32, 211 SDPatternOperator ld = null_frag> { 212 213 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 214 [(set load_vt:$vdata, 215 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt, 216 i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 217 MTBUFAddr64Table<0>; 218 219 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 220 [(set load_vt:$vdata, 221 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 222 i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 223 MTBUFAddr64Table<1>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 list<dag> pattern=[], 241 // Workaround bug bz30254 242 int addrKindCopy = addrKind, 243 RegisterClass vdataClassCopy = vdataClass> 244 : MTBUF_Pseudo<opName, 245 (outs), 246 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 247 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 248 pattern>, 249 MTBUF_SetupAddr<addrKindCopy> { 250 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 251 let mayLoad = 0; 252 let mayStore = 1; 253} 254 255multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 256 ValueType store_vt = i32, 257 SDPatternOperator st = null_frag> { 258 259 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 260 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 261 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 262 i1:$slc, i1:$tfe))]>, 263 MTBUFAddr64Table<0>; 264 265 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 266 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 267 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 268 i1:$slc, i1:$tfe))]>, 269 MTBUFAddr64Table<1>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUF_Pseudo <string opName, dag outs, dag ins, 289 string asmOps, list<dag> pattern=[]> : 290 InstSI<outs, ins, "", pattern>, 291 SIMCInstr<opName, SIEncodingFamily.NONE> { 292 293 let isPseudo = 1; 294 let isCodeGenOnly = 1; 295 let Size = 8; 296 let UseNamedOperandTable = 1; 297 298 string Mnemonic = opName; 299 string AsmOperands = asmOps; 300 301 let VM_CNT = 1; 302 let EXP_CNT = 1; 303 let MUBUF = 1; 304 let Uses = [EXEC]; 305 let hasSideEffects = 0; 306 let SchedRW = [WriteVMEM]; 307 308 let AsmMatchConverter = "cvtMubuf"; 309 310 bits<1> offen = 0; 311 bits<1> idxen = 0; 312 bits<1> addr64 = 0; 313 bits<1> has_vdata = 1; 314 bits<1> has_vaddr = 1; 315 bits<1> has_glc = 1; 316 bits<1> glc_value = 0; // the value for glc if no such operand 317 bits<1> has_srsrc = 1; 318 bits<1> has_soffset = 1; 319 bits<1> has_offset = 1; 320 bits<1> has_slc = 1; 321 bits<1> has_tfe = 1; 322} 323 324class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 325 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 326 327 let isPseudo = 0; 328 let isCodeGenOnly = 0; 329 330 // copy relevant pseudo op flags 331 let SubtargetPredicate = ps.SubtargetPredicate; 332 let AsmMatchConverter = ps.AsmMatchConverter; 333 let Constraints = ps.Constraints; 334 let DisableEncoding = ps.DisableEncoding; 335 let TSFlags = ps.TSFlags; 336 337 bits<12> offset; 338 bits<1> glc; 339 bits<1> lds = 0; 340 bits<8> vaddr; 341 bits<8> vdata; 342 bits<7> srsrc; 343 bits<1> slc; 344 bits<1> tfe; 345 bits<8> soffset; 346} 347 348 349// For cache invalidation instructions. 350class MUBUF_Invalidate <string opName, SDPatternOperator node> : 351 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 352 353 let AsmMatchConverter = ""; 354 355 let hasSideEffects = 1; 356 let mayStore = 1; 357 358 // Set everything to 0. 359 let offen = 0; 360 let idxen = 0; 361 let addr64 = 0; 362 let has_vdata = 0; 363 let has_vaddr = 0; 364 let has_glc = 0; 365 let glc_value = 0; 366 let has_srsrc = 0; 367 let has_soffset = 0; 368 let has_offset = 0; 369 let has_slc = 0; 370 let has_tfe = 0; 371} 372 373class getMUBUFInsDA<list<RegisterClass> vdataList, 374 list<RegisterClass> vaddrList=[]> { 375 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 376 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 377 dag InsNoData = !if(!empty(vaddrList), 378 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 379 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 380 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 381 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 382 ); 383 dag InsData = !if(!empty(vaddrList), 384 (ins vdataClass:$vdata, SReg_128:$srsrc, 385 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 386 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 387 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 388 ); 389 dag ret = !if(!empty(vdataList), InsNoData, InsData); 390} 391 392class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 393 dag ret = 394 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret, 395 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 396 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 397 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 398 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 399 (ins)))))); 400} 401 402class getMUBUFAsmOps<int addrKind> { 403 string Pfx = 404 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 405 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 406 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 407 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 408 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 409 ""))))); 410 string ret = Pfx # "$offset"; 411} 412 413class MUBUF_SetupAddr<int addrKind> { 414 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 415 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 416 417 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 418 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 419 420 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 421 422 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 423} 424 425class MUBUF_Load_Pseudo <string opName, 426 int addrKind, 427 RegisterClass vdataClass, 428 bit HasTiedDest = 0, 429 list<dag> pattern=[], 430 // Workaround bug bz30254 431 int addrKindCopy = addrKind> 432 : MUBUF_Pseudo<opName, 433 (outs vdataClass:$vdata), 434 !con(getMUBUFIns<addrKindCopy>.ret, !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 435 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 436 pattern>, 437 MUBUF_SetupAddr<addrKindCopy> { 438 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 439 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 440 let mayLoad = 1; 441 let mayStore = 0; 442 let maybeAtomic = 1; 443} 444 445// FIXME: tfe can't be an operand because it requires a separate 446// opcode because it needs an N+1 register class dest register. 447multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 448 ValueType load_vt = i32, 449 SDPatternOperator ld = null_frag, 450 bit TiedDest = 0> { 451 452 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 453 TiedDest, 454 [(set load_vt:$vdata, 455 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 456 MUBUFAddr64Table<0>; 457 458 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 459 TiedDest, 460 [(set load_vt:$vdata, 461 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 462 MUBUFAddr64Table<1>; 463 464 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>; 465 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>; 466 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>; 467 468 let DisableWQM = 1 in { 469 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest>; 470 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>; 471 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>; 472 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>; 473 } 474} 475 476class MUBUF_Store_Pseudo <string opName, 477 int addrKind, 478 RegisterClass vdataClass, 479 list<dag> pattern=[], 480 // Workaround bug bz30254 481 int addrKindCopy = addrKind, 482 RegisterClass vdataClassCopy = vdataClass> 483 : MUBUF_Pseudo<opName, 484 (outs), 485 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 486 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 487 pattern>, 488 MUBUF_SetupAddr<addrKindCopy> { 489 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 490 let mayLoad = 0; 491 let mayStore = 1; 492 let maybeAtomic = 1; 493} 494 495multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 496 ValueType store_vt = i32, 497 SDPatternOperator st = null_frag> { 498 499 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 500 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 501 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 502 MUBUFAddr64Table<0>; 503 504 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 505 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 506 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 507 MUBUFAddr64Table<1>; 508 509 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 510 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 511 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 512 513 let DisableWQM = 1 in { 514 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 515 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 516 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 517 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 518 } 519} 520 521 522class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 523 list<RegisterClass> vaddrList=[]> { 524 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 525 dag ret = !if(vdata_in, 526 !if(!empty(vaddrList), 527 (ins vdataClass:$vdata_in, 528 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 529 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 530 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 531 ), 532 !if(!empty(vaddrList), 533 (ins vdataClass:$vdata, 534 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 535 (ins vdataClass:$vdata, vaddrClass:$vaddr, 536 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 537 )); 538} 539 540class getMUBUFAtomicIns<int addrKind, 541 RegisterClass vdataClass, 542 bit vdata_in, 543 // Workaround bug bz30254 544 RegisterClass vdataClassCopy=vdataClass> { 545 dag ret = 546 !if(!eq(addrKind, BUFAddrKind.Offset), 547 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 548 !if(!eq(addrKind, BUFAddrKind.OffEn), 549 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 550 !if(!eq(addrKind, BUFAddrKind.IdxEn), 551 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 552 !if(!eq(addrKind, BUFAddrKind.BothEn), 553 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 554 !if(!eq(addrKind, BUFAddrKind.Addr64), 555 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 556 (ins)))))); 557} 558 559class MUBUF_Atomic_Pseudo<string opName, 560 int addrKind, 561 dag outs, 562 dag ins, 563 string asmOps, 564 list<dag> pattern=[], 565 // Workaround bug bz30254 566 int addrKindCopy = addrKind> 567 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 568 MUBUF_SetupAddr<addrKindCopy> { 569 let mayStore = 1; 570 let mayLoad = 1; 571 let hasPostISelHook = 1; 572 let hasSideEffects = 1; 573 let DisableWQM = 1; 574 let has_glc = 0; 575 let has_tfe = 0; 576 let maybeAtomic = 1; 577} 578 579class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 580 RegisterClass vdataClass, 581 list<dag> pattern=[], 582 // Workaround bug bz30254 583 int addrKindCopy = addrKind, 584 RegisterClass vdataClassCopy = vdataClass> 585 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 586 (outs), 587 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 588 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 589 pattern>, 590 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 591 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 592 let glc_value = 0; 593 let AsmMatchConverter = "cvtMubufAtomic"; 594} 595 596class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 597 RegisterClass vdataClass, 598 list<dag> pattern=[], 599 // Workaround bug bz30254 600 int addrKindCopy = addrKind, 601 RegisterClass vdataClassCopy = vdataClass> 602 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 603 (outs vdataClassCopy:$vdata), 604 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 605 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 606 pattern>, 607 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 608 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 609 let glc_value = 1; 610 let Constraints = "$vdata = $vdata_in"; 611 let DisableEncoding = "$vdata_in"; 612 let AsmMatchConverter = "cvtMubufAtomicReturn"; 613} 614 615multiclass MUBUF_Pseudo_Atomics <string opName, 616 RegisterClass vdataClass, 617 ValueType vdataType, 618 SDPatternOperator atomic> { 619 620 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 621 MUBUFAddr64Table <0>; 622 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 623 MUBUFAddr64Table <1>; 624 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 625 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 626 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 627 628 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 629 [(set vdataType:$vdata, 630 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 631 vdataType:$vdata_in))]>, 632 MUBUFAddr64Table <0, "_RTN">; 633 634 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 635 [(set vdataType:$vdata, 636 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 637 vdataType:$vdata_in))]>, 638 MUBUFAddr64Table <1, "_RTN">; 639 640 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 641 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 642 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 643} 644 645 646//===----------------------------------------------------------------------===// 647// MUBUF Instructions 648//===----------------------------------------------------------------------===// 649 650defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads < 651 "buffer_load_format_x", VGPR_32 652>; 653defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 654 "buffer_load_format_xy", VReg_64 655>; 656defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 657 "buffer_load_format_xyz", VReg_96 658>; 659defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 660 "buffer_load_format_xyzw", VReg_128 661>; 662defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 663 "buffer_store_format_x", VGPR_32 664>; 665defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 666 "buffer_store_format_xy", VReg_64 667>; 668defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 669 "buffer_store_format_xyz", VReg_96 670>; 671defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 672 "buffer_store_format_xyzw", VReg_128 673>; 674defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads < 675 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 676>; 677defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads < 678 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 679>; 680defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads < 681 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 682>; 683defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads < 684 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 685>; 686defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads < 687 "buffer_load_dword", VGPR_32, i32, mubuf_load 688>; 689defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 690 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 691>; 692defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 693 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 694>; 695defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 696 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 697>; 698defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 699 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 700>; 701defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 702 "buffer_store_short", VGPR_32, i32, truncstorei16_global 703>; 704defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 705 "buffer_store_dword", VGPR_32, i32, store_global 706>; 707defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 708 "buffer_store_dwordx2", VReg_64, v2i32, store_global 709>; 710defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 711 "buffer_store_dwordx3", VReg_96, untyped, store_global 712>; 713defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 714 "buffer_store_dwordx4", VReg_128, v4i32, store_global 715>; 716defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 717 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 718>; 719defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 720 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 721>; 722defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 723 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 724>; 725defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 726 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 727>; 728defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 729 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 730>; 731defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 732 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 733>; 734defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 735 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 736>; 737defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 738 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 739>; 740defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 741 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 742>; 743defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 744 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 745>; 746defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 747 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 748>; 749defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 750 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 751>; 752defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 753 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 754>; 755defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 756 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 757>; 758defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 759 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 760>; 761defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 762 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 763>; 764defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 765 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 766>; 767defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 768 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 769>; 770defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 771 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 772>; 773defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 774 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 775>; 776defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 777 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 778>; 779defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 780 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 781>; 782defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 783 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 784>; 785defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 786 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 787>; 788defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 789 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 790>; 791defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 792 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 793>; 794 795let SubtargetPredicate = isSI in { // isn't on CI & VI 796/* 797defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 798defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 799defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 800defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 801defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 802defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 803defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 804defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 805*/ 806 807def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 808 int_amdgcn_buffer_wbinvl1_sc>; 809} 810 811let SubtargetPredicate = HasD16LoadStore in { 812 813defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 814 "buffer_load_ubyte_d16", VGPR_32, i32 815>; 816 817defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 818 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 819>; 820 821defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 822 "buffer_load_sbyte_d16", VGPR_32, i32 823>; 824 825defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 826 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 827>; 828 829defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 830 "buffer_load_short_d16", VGPR_32, i32 831>; 832 833defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 834 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 835>; 836 837defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 838 "buffer_store_byte_d16_hi", VGPR_32, i32 839>; 840 841defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 842 "buffer_store_short_d16_hi", VGPR_32, i32 843>; 844 845} // End HasD16LoadStore 846 847def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 848 int_amdgcn_buffer_wbinvl1>; 849 850//===----------------------------------------------------------------------===// 851// MTBUF Instructions 852//===----------------------------------------------------------------------===// 853 854defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 855defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 856defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>; 857defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 858defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 859defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 860defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>; 861defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 862 863let SubtargetPredicate = isCIVI in { 864 865//===----------------------------------------------------------------------===// 866// Instruction definitions for CI and newer. 867//===----------------------------------------------------------------------===// 868// Remaining instructions: 869// BUFFER_LOAD_DWORDX3 870// BUFFER_STORE_DWORDX3 871 872def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 873 int_amdgcn_buffer_wbinvl1_vol>; 874 875} // End let SubtargetPredicate = isCIVI 876 877//===----------------------------------------------------------------------===// 878// MUBUF Patterns 879//===----------------------------------------------------------------------===// 880 881//===----------------------------------------------------------------------===// 882// buffer_load/store_format patterns 883//===----------------------------------------------------------------------===// 884 885multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 886 string opcode> { 887 def : GCNPat< 888 (vt (name v4i32:$rsrc, 0, 889 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 890 imm:$glc, imm:$slc)), 891 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 892 (as_i1imm $glc), (as_i1imm $slc), 0) 893 >; 894 895 def : GCNPat< 896 (vt (name v4i32:$rsrc, i32:$vindex, 897 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 898 imm:$glc, imm:$slc)), 899 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 900 (as_i1imm $glc), (as_i1imm $slc), 0) 901 >; 902 903 def : GCNPat< 904 (vt (name v4i32:$rsrc, 0, 905 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 906 imm:$glc, imm:$slc)), 907 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 908 (as_i1imm $glc), (as_i1imm $slc), 0) 909 >; 910 911 def : GCNPat< 912 (vt (name v4i32:$rsrc, i32:$vindex, 913 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 914 imm:$glc, imm:$slc)), 915 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 916 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 917 $rsrc, $soffset, (as_i16imm $offset), 918 (as_i1imm $glc), (as_i1imm $slc), 0) 919 >; 920} 921 922defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 923defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 924defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 925defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 926defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 927defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 928 929multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 930 string opcode> { 931 def : GCNPat< 932 (name vt:$vdata, v4i32:$rsrc, 0, 933 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 934 imm:$glc, imm:$slc), 935 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 936 (as_i1imm $glc), (as_i1imm $slc), 0) 937 >; 938 939 def : GCNPat< 940 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 941 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 942 imm:$glc, imm:$slc), 943 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 944 (as_i16imm $offset), (as_i1imm $glc), 945 (as_i1imm $slc), 0) 946 >; 947 948 def : GCNPat< 949 (name vt:$vdata, v4i32:$rsrc, 0, 950 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 951 imm:$glc, imm:$slc), 952 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 953 (as_i16imm $offset), (as_i1imm $glc), 954 (as_i1imm $slc), 0) 955 >; 956 957 def : GCNPat< 958 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 959 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 960 imm:$glc, imm:$slc), 961 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 962 $vdata, 963 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 964 $rsrc, $soffset, (as_i16imm $offset), 965 (as_i1imm $glc), (as_i1imm $slc), 0) 966 >; 967} 968 969defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 970defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 971defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 972defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">; 973defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 974defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 975 976//===----------------------------------------------------------------------===// 977// buffer_atomic patterns 978//===----------------------------------------------------------------------===// 979 980multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 981 def : GCNPat< 982 (name i32:$vdata_in, v4i32:$rsrc, 0, 983 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 984 imm:$slc), 985 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 986 (as_i16imm $offset), (as_i1imm $slc)) 987 >; 988 989 def : GCNPat< 990 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 991 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 992 imm:$slc), 993 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 994 (as_i16imm $offset), (as_i1imm $slc)) 995 >; 996 997 def : GCNPat< 998 (name i32:$vdata_in, v4i32:$rsrc, 0, 999 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1000 imm:$slc), 1001 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1002 (as_i16imm $offset), (as_i1imm $slc)) 1003 >; 1004 1005 def : GCNPat< 1006 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1007 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1008 imm:$slc), 1009 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1010 $vdata_in, 1011 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1012 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)) 1013 >; 1014} 1015 1016defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 1017defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">; 1018defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 1019defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 1020defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 1021defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 1022defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 1023defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">; 1024defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">; 1025defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 1026 1027def : GCNPat< 1028 (int_amdgcn_buffer_atomic_cmpswap 1029 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1030 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1031 imm:$slc), 1032 (EXTRACT_SUBREG 1033 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1034 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1035 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1036 sub0) 1037>; 1038 1039def : GCNPat< 1040 (int_amdgcn_buffer_atomic_cmpswap 1041 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1042 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1043 imm:$slc), 1044 (EXTRACT_SUBREG 1045 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1046 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1047 $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1048 sub0) 1049>; 1050 1051def : GCNPat< 1052 (int_amdgcn_buffer_atomic_cmpswap 1053 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1054 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1055 imm:$slc), 1056 (EXTRACT_SUBREG 1057 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1058 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1059 $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1060 sub0) 1061>; 1062 1063def : GCNPat< 1064 (int_amdgcn_buffer_atomic_cmpswap 1065 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1066 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1067 imm:$slc), 1068 (EXTRACT_SUBREG 1069 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1070 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1071 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1072 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1073 sub0) 1074>; 1075 1076 1077class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1078 PatFrag constant_ld> : GCNPat < 1079 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1080 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1081 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1082 >; 1083 1084multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1085 ValueType vt, PatFrag atomic_ld> { 1086 def : GCNPat < 1087 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1088 i16:$offset, i1:$slc))), 1089 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1090 >; 1091 1092 def : GCNPat < 1093 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1094 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1095 >; 1096} 1097 1098let SubtargetPredicate = isSICI in { 1099def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1100def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1101def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1102def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1103 1104defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1105defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1106} // End SubtargetPredicate = isSICI 1107 1108multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1109 PatFrag ld> { 1110 1111 def : GCNPat < 1112 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1113 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1114 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1115 >; 1116} 1117 1118let OtherPredicates = [Has16BitInsts] in { 1119 1120defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1121defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1122defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1123defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1124 1125defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1126 1127} // End OtherPredicates = [Has16BitInsts] 1128 1129multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1130 MUBUF_Pseudo InstrOffset, 1131 ValueType vt, PatFrag ld> { 1132 def : GCNPat < 1133 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1134 i32:$soffset, u16imm:$offset))), 1135 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1136 >; 1137 1138 def : GCNPat < 1139 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1140 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1141 >; 1142} 1143 1144// XXX - Is it possible to have a complex pattern in a PatFrag? 1145multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen, 1146 MUBUF_Pseudo InstrOffset, 1147 ValueType vt, PatFrag ld> { 1148 def : GCNPat < 1149 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1150 i32:$soffset, u16imm:$offset)))), 1151 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1152 >; 1153 1154 def : GCNPat < 1155 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1156 i32:$soffset, u16imm:$offset)))))), 1157 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1158 >; 1159 1160 1161 def : GCNPat < 1162 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))), 1163 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1164 >; 1165 1166 def : GCNPat < 1167 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))), 1168 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1169 >; 1170} 1171 1172defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1173defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1174defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1175defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1176defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1177defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1178defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1179defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1180defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1181defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1182 1183let OtherPredicates = [HasD16LoadStore] in { 1184defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>; 1185defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>; 1186defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>; 1187} 1188 1189// BUFFER_LOAD_DWORD*, addr64=0 1190multiclass MUBUF_Load_Dword <ValueType vt, 1191 MUBUF_Pseudo offset, 1192 MUBUF_Pseudo offen, 1193 MUBUF_Pseudo idxen, 1194 MUBUF_Pseudo bothen> { 1195 1196 def : GCNPat < 1197 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset, 1198 imm:$offset, 0, 0, imm:$glc, imm:$slc, 1199 imm:$tfe)), 1200 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1201 (as_i1imm $slc), (as_i1imm $tfe)) 1202 >; 1203 1204 def : GCNPat < 1205 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1206 imm:$offset, 1, 0, imm:$glc, imm:$slc, 1207 imm:$tfe)), 1208 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1209 (as_i1imm $tfe)) 1210 >; 1211 1212 def : GCNPat < 1213 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1214 imm:$offset, 0, 1, imm:$glc, imm:$slc, 1215 imm:$tfe)), 1216 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1217 (as_i1imm $slc), (as_i1imm $tfe)) 1218 >; 1219 1220 def : GCNPat < 1221 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset, 1222 imm:$offset, 1, 1, imm:$glc, imm:$slc, 1223 imm:$tfe)), 1224 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1225 (as_i1imm $tfe)) 1226 >; 1227} 1228 1229defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN, 1230 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>; 1231defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN, 1232 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>; 1233defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN, 1234 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>; 1235 1236multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1237 ValueType vt, PatFrag atomic_st> { 1238 // Store follows atomic op convention so address is forst 1239 def : GCNPat < 1240 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1241 i16:$offset, i1:$slc), vt:$val), 1242 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1243 >; 1244 1245 def : GCNPat < 1246 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1247 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1248 >; 1249} 1250let SubtargetPredicate = isSICI in { 1251defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1252defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1253} // End Predicates = isSICI 1254 1255 1256multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1257 PatFrag st> { 1258 1259 def : GCNPat < 1260 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1261 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1262 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1263 >; 1264} 1265 1266defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1267defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1268 1269multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1270 MUBUF_Pseudo InstrOffset, 1271 ValueType vt, PatFrag st> { 1272 def : GCNPat < 1273 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1274 i32:$soffset, u16imm:$offset)), 1275 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1276 >; 1277 1278 def : GCNPat < 1279 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1280 u16imm:$offset)), 1281 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1282 >; 1283} 1284 1285defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1286defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1287defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1288defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1289defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1290defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1291defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1292 1293 1294let OtherPredicates = [HasD16LoadStore] in { 1295 // Hiding the extract high pattern in the PatFrag seems to not 1296 // automatically increase the complexity. 1297let AddedComplexity = 1 in { 1298defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1299defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1300} 1301} 1302 1303//===----------------------------------------------------------------------===// 1304// MTBUF Patterns 1305//===----------------------------------------------------------------------===// 1306 1307//===----------------------------------------------------------------------===// 1308// tbuffer_load/store_format patterns 1309//===----------------------------------------------------------------------===// 1310 1311multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1312 string opcode> { 1313 def : GCNPat< 1314 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1315 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1316 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1317 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1318 >; 1319 1320 def : GCNPat< 1321 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1322 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1323 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1324 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1325 >; 1326 1327 def : GCNPat< 1328 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1329 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1330 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1331 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1332 >; 1333 1334 def : GCNPat< 1335 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1336 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1337 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1338 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1339 $rsrc, $soffset, (as_i16imm $offset), 1340 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1341 >; 1342} 1343 1344defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1345defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1346defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1347defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1348defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1349defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1350 1351multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1352 string opcode> { 1353 def : GCNPat< 1354 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1355 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1356 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1357 (as_i16imm $offset), (as_i8imm $dfmt), 1358 (as_i8imm $nfmt), (as_i1imm $glc), 1359 (as_i1imm $slc), 0) 1360 >; 1361 1362 def : GCNPat< 1363 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1364 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1365 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1366 (as_i16imm $offset), (as_i8imm $dfmt), 1367 (as_i8imm $nfmt), (as_i1imm $glc), 1368 (as_i1imm $slc), 0) 1369 >; 1370 1371 def : GCNPat< 1372 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1373 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1374 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1375 (as_i16imm $offset), (as_i8imm $dfmt), 1376 (as_i8imm $nfmt), (as_i1imm $glc), 1377 (as_i1imm $slc), 0) 1378 >; 1379 1380 def : GCNPat< 1381 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1382 imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1383 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1384 $vdata, 1385 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1386 $rsrc, $soffset, (as_i16imm $offset), 1387 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1388 >; 1389} 1390 1391defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1392defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1393defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">; 1394defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1395defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1396defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1397defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">; 1398defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1399 1400//===----------------------------------------------------------------------===// 1401// Target instructions, move to the appropriate target TD file 1402//===----------------------------------------------------------------------===// 1403 1404//===----------------------------------------------------------------------===// 1405// SI 1406//===----------------------------------------------------------------------===// 1407 1408class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1409 MUBUF_Real<op, ps>, 1410 Enc64, 1411 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1412 let AssemblerPredicate=isSICI; 1413 let DecoderNamespace="SICI"; 1414 1415 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1416 let Inst{12} = ps.offen; 1417 let Inst{13} = ps.idxen; 1418 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1419 let Inst{15} = ps.addr64; 1420 let Inst{16} = lds; 1421 let Inst{24-18} = op; 1422 let Inst{31-26} = 0x38; //encoding 1423 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1424 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1425 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1426 let Inst{54} = !if(ps.has_slc, slc, ?); 1427 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1428 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1429} 1430 1431multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1432 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1433 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1434 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1435 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1436 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1437} 1438 1439multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1440 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1441 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1442 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1443 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1444 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1445} 1446 1447defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_si <0x00>; 1448defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1449defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1450defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1451defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1452defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1453defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1454defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1455defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_si <0x08>; 1456defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_si <0x09>; 1457defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_si <0x0a>; 1458defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_si <0x0b>; 1459defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_si <0x0c>; 1460defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1461defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1462defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1463defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1464defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1465defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1466defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1467defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1468defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1469 1470defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1471defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1472defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1473defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1474//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1475defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1476defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1477defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1478defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1479defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1480defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1481defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1482defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1483defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1484 1485//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1486//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1487//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1488defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1489defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1490defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1491defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1492//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1493defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1494defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1495defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1496defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1497defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1498defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1499defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1500defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1501defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1502// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1503//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1504//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1505//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1506 1507def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1508def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1509 1510class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1511 MTBUF_Real<ps>, 1512 Enc64, 1513 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1514 let AssemblerPredicate=isSICI; 1515 let DecoderNamespace="SICI"; 1516 1517 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1518 let Inst{12} = ps.offen; 1519 let Inst{13} = ps.idxen; 1520 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1521 let Inst{15} = ps.addr64; 1522 let Inst{18-16} = op; 1523 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 1524 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 1525 let Inst{31-26} = 0x3a; //encoding 1526 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1527 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1528 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1529 let Inst{54} = !if(ps.has_slc, slc, ?); 1530 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1531 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1532} 1533 1534multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1535 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1536 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1537 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1538 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1539 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1540} 1541 1542defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1543defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1544//defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1545defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1546defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1547defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1548defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1549defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1550 1551//===----------------------------------------------------------------------===// 1552// CI 1553//===----------------------------------------------------------------------===// 1554 1555class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1556 MUBUF_Real_si<op, ps> { 1557 let AssemblerPredicate=isCIOnly; 1558 let DecoderNamespace="CI"; 1559} 1560 1561def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1562 1563 1564//===----------------------------------------------------------------------===// 1565// VI 1566//===----------------------------------------------------------------------===// 1567 1568class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1569 MUBUF_Real<op, ps>, 1570 Enc64, 1571 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1572 let AssemblerPredicate=isVI; 1573 let DecoderNamespace="VI"; 1574 1575 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1576 let Inst{12} = ps.offen; 1577 let Inst{13} = ps.idxen; 1578 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1579 let Inst{16} = lds; 1580 let Inst{17} = !if(ps.has_slc, slc, ?); 1581 let Inst{24-18} = op; 1582 let Inst{31-26} = 0x38; //encoding 1583 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1584 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1585 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1586 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1587 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1588} 1589 1590multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1591 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1592 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1593 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1594 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1595} 1596 1597multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1598 MUBUF_Real_AllAddr_vi<op> { 1599 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1600 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1601 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1602 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1603} 1604 1605defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_vi <0x00>; 1606defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1607defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1608defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1609defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1610defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1611defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1612defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1613defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_vi <0x10>; 1614defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_vi <0x11>; 1615defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_vi <0x12>; 1616defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_vi <0x13>; 1617defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_vi <0x14>; 1618defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 1619defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 1620defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 1621defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1622defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1623defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1624defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1625defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1626defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1627defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1628defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1629 1630defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1631defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1632defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1633defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1634defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1635defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1636 1637defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1638defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1639defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1640defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1641defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1642defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1643defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1644defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1645defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1646defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1647defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1648defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1649defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1650 1651defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1652defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1653defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1654defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1655defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1656defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1657defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1658defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1659defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1660defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1661defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1662defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1663defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1664 1665def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 1666def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 1667 1668class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 1669 MTBUF_Real<ps>, 1670 Enc64, 1671 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1672 let AssemblerPredicate=isVI; 1673 let DecoderNamespace="VI"; 1674 1675 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1676 let Inst{12} = ps.offen; 1677 let Inst{13} = ps.idxen; 1678 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1679 let Inst{18-15} = op; 1680 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 1681 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 1682 let Inst{31-26} = 0x3a; //encoding 1683 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1684 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1685 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1686 let Inst{54} = !if(ps.has_slc, slc, ?); 1687 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1688 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1689} 1690 1691multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 1692 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1693 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1694 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1695 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1696} 1697 1698defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0>; 1699defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <1>; 1700//defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <2>; 1701defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <3>; 1702defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <4>; 1703defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <5>; 1704defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <6>; 1705defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <7>; 1706