1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20class MubufLoad <SDPatternOperator op> : PatFrag < 21 (ops node:$ptr), (op node:$ptr), [{ 22 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 23 return AS == AMDGPUAS::GLOBAL_ADDRESS || 24 AS == AMDGPUAS::CONSTANT_ADDRESS; 25}]>; 26 27def mubuf_load : MubufLoad <load>; 28def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 29def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 30def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 31def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 32def mubuf_load_atomic : MubufLoad <atomic_load>; 33 34def BUFAddrKind { 35 int Offset = 0; 36 int OffEn = 1; 37 int IdxEn = 2; 38 int BothEn = 3; 39 int Addr64 = 4; 40} 41 42class getAddrName<int addrKind> { 43 string ret = 44 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 45 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 46 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 47 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 48 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 49 ""))))); 50} 51 52class MUBUFAddr64Table <bit is_addr64, string Name> { 53 bit IsAddr64 = is_addr64; 54 string OpName = Name; 55} 56 57class MUBUFLdsTable <bit is_lds, string Name> { 58 bit IsLds = is_lds; 59 string OpName = Name; 60} 61 62class MTBUFAddr64Table <bit is_addr64, string Name> { 63 bit IsAddr64 = is_addr64; 64 string OpName = Name; 65} 66 67//===----------------------------------------------------------------------===// 68// MTBUF classes 69//===----------------------------------------------------------------------===// 70 71class MTBUF_Pseudo <string opName, dag outs, dag ins, 72 string asmOps, list<dag> pattern=[]> : 73 InstSI<outs, ins, "", pattern>, 74 SIMCInstr<opName, SIEncodingFamily.NONE> { 75 76 let isPseudo = 1; 77 let isCodeGenOnly = 1; 78 let Size = 8; 79 let UseNamedOperandTable = 1; 80 81 string Mnemonic = opName; 82 string AsmOperands = asmOps; 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> glc_value = 0; // the value for glc if no such operand 100 bits<1> has_srsrc = 1; 101 bits<1> has_soffset = 1; 102 bits<1> has_offset = 1; 103 bits<1> has_slc = 1; 104 bits<1> has_tfe = 1; 105} 106 107class MTBUF_Real <MTBUF_Pseudo ps> : 108 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 109 110 let isPseudo = 0; 111 let isCodeGenOnly = 0; 112 113 // copy relevant pseudo op flags 114 let SubtargetPredicate = ps.SubtargetPredicate; 115 let AsmMatchConverter = ps.AsmMatchConverter; 116 let Constraints = ps.Constraints; 117 let DisableEncoding = ps.DisableEncoding; 118 let TSFlags = ps.TSFlags; 119 120 bits<12> offset; 121 bits<1> glc; 122 bits<7> format; 123 bits<8> vaddr; 124 bits<8> vdata; 125 bits<7> srsrc; 126 bits<1> slc; 127 bits<1> tfe; 128 bits<8> soffset; 129 130 bits<4> dfmt = format{3-0}; 131 bits<3> nfmt = format{6-4}; 132} 133 134class getMTBUFInsDA<list<RegisterClass> vdataList, 135 list<RegisterClass> vaddrList=[]> { 136 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 137 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 138 dag InsNoData = !if(!empty(vaddrList), 139 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 140 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe), 141 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 142 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe) 143 ); 144 dag InsData = !if(!empty(vaddrList), 145 (ins vdataClass:$vdata, SReg_128:$srsrc, 146 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 147 SLC:$slc, TFE:$tfe), 148 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 149 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 150 SLC:$slc, TFE:$tfe) 151 ); 152 dag ret = !if(!empty(vdataList), InsNoData, InsData); 153} 154 155class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 156 dag ret = 157 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 158 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 159 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 160 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 161 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 162 (ins)))))); 163} 164 165class getMTBUFAsmOps<int addrKind> { 166 string Pfx = 167 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 168 !if(!eq(addrKind, BUFAddrKind.OffEn), 169 "$vaddr, $srsrc, $format, $soffset offen", 170 !if(!eq(addrKind, BUFAddrKind.IdxEn), 171 "$vaddr, $srsrc, $format, $soffset idxen", 172 !if(!eq(addrKind, BUFAddrKind.BothEn), 173 "$vaddr, $srsrc, $format, $soffset idxen offen", 174 !if(!eq(addrKind, BUFAddrKind.Addr64), 175 "$vaddr, $srsrc, $format, $soffset addr64", 176 ""))))); 177 string ret = Pfx # "$offset"; 178} 179 180class MTBUF_SetupAddr<int addrKind> { 181 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 182 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 183 184 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 185 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 186 187 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 188 189 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 190} 191 192class MTBUF_Load_Pseudo <string opName, 193 int addrKind, 194 RegisterClass vdataClass, 195 list<dag> pattern=[], 196 // Workaround bug bz30254 197 int addrKindCopy = addrKind> 198 : MTBUF_Pseudo<opName, 199 (outs vdataClass:$vdata), 200 getMTBUFIns<addrKindCopy>.ret, 201 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 202 pattern>, 203 MTBUF_SetupAddr<addrKindCopy> { 204 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 205 let mayLoad = 1; 206 let mayStore = 0; 207} 208 209multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 210 ValueType load_vt = i32, 211 SDPatternOperator ld = null_frag> { 212 213 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 214 [(set load_vt:$vdata, 215 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 216 i1:$glc, i1:$slc, i1:$tfe)))]>, 217 MTBUFAddr64Table<0, NAME>; 218 219 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 220 [(set load_vt:$vdata, 221 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 222 i8:$format, i1:$glc, i1:$slc, i1:$tfe)))]>, 223 MTBUFAddr64Table<1, NAME>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 list<dag> pattern=[], 241 // Workaround bug bz30254 242 int addrKindCopy = addrKind, 243 RegisterClass vdataClassCopy = vdataClass> 244 : MTBUF_Pseudo<opName, 245 (outs), 246 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 247 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 248 pattern>, 249 MTBUF_SetupAddr<addrKindCopy> { 250 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 251 let mayLoad = 0; 252 let mayStore = 1; 253} 254 255multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 256 ValueType store_vt = i32, 257 SDPatternOperator st = null_frag> { 258 259 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 260 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 261 i16:$offset, i8:$format, i1:$glc, 262 i1:$slc, i1:$tfe))]>, 263 MTBUFAddr64Table<0, NAME>; 264 265 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 266 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 267 i16:$offset, i8:$format, i1:$glc, 268 i1:$slc, i1:$tfe))]>, 269 MTBUFAddr64Table<1, NAME>; 270 271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 274 275 let DisableWQM = 1 in { 276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 280 } 281} 282 283 284//===----------------------------------------------------------------------===// 285// MUBUF classes 286//===----------------------------------------------------------------------===// 287 288class MUBUFGetBaseOpcode<string Op> { 289 string ret = !subst("DWORDX2", "DWORD", 290 !subst("DWORDX3", "DWORD", 291 !subst("DWORDX4", "DWORD", Op))); 292} 293 294class MUBUF_Pseudo <string opName, dag outs, dag ins, 295 string asmOps, list<dag> pattern=[]> : 296 InstSI<outs, ins, "", pattern>, 297 SIMCInstr<opName, SIEncodingFamily.NONE> { 298 299 let isPseudo = 1; 300 let isCodeGenOnly = 1; 301 let Size = 8; 302 let UseNamedOperandTable = 1; 303 304 string Mnemonic = opName; 305 string AsmOperands = asmOps; 306 307 Instruction Opcode = !cast<Instruction>(NAME); 308 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 309 310 let VM_CNT = 1; 311 let EXP_CNT = 1; 312 let MUBUF = 1; 313 let Uses = [EXEC]; 314 let hasSideEffects = 0; 315 let SchedRW = [WriteVMEM]; 316 317 let AsmMatchConverter = "cvtMubuf"; 318 319 bits<1> offen = 0; 320 bits<1> idxen = 0; 321 bits<1> addr64 = 0; 322 bits<1> lds = 0; 323 bits<1> has_vdata = 1; 324 bits<1> has_vaddr = 1; 325 bits<1> has_glc = 1; 326 bits<1> glc_value = 0; // the value for glc if no such operand 327 bits<1> has_srsrc = 1; 328 bits<1> has_soffset = 1; 329 bits<1> has_offset = 1; 330 bits<1> has_slc = 1; 331 bits<1> has_tfe = 1; 332 bits<4> dwords = 0; 333} 334 335class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 336 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 337 338 let isPseudo = 0; 339 let isCodeGenOnly = 0; 340 341 // copy relevant pseudo op flags 342 let SubtargetPredicate = ps.SubtargetPredicate; 343 let AsmMatchConverter = ps.AsmMatchConverter; 344 let Constraints = ps.Constraints; 345 let DisableEncoding = ps.DisableEncoding; 346 let TSFlags = ps.TSFlags; 347 348 bits<12> offset; 349 bits<1> glc; 350 bits<8> vaddr; 351 bits<8> vdata; 352 bits<7> srsrc; 353 bits<1> slc; 354 bits<1> tfe; 355 bits<8> soffset; 356} 357 358 359// For cache invalidation instructions. 360class MUBUF_Invalidate <string opName, SDPatternOperator node> : 361 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 362 363 let AsmMatchConverter = ""; 364 365 let hasSideEffects = 1; 366 let mayStore = 1; 367 368 // Set everything to 0. 369 let offen = 0; 370 let idxen = 0; 371 let addr64 = 0; 372 let has_vdata = 0; 373 let has_vaddr = 0; 374 let has_glc = 0; 375 let glc_value = 0; 376 let has_srsrc = 0; 377 let has_soffset = 0; 378 let has_offset = 0; 379 let has_slc = 0; 380 let has_tfe = 0; 381} 382 383class getMUBUFInsDA<list<RegisterClass> vdataList, 384 list<RegisterClass> vaddrList=[], 385 bit isLds = 0> { 386 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 387 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 388 dag InsNoData = !if(!empty(vaddrList), 389 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 390 offset:$offset, GLC:$glc, SLC:$slc), 391 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 392 offset:$offset, GLC:$glc, SLC:$slc) 393 ); 394 dag InsData = !if(!empty(vaddrList), 395 (ins vdataClass:$vdata, SReg_128:$srsrc, 396 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 397 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 398 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 399 ); 400 dag ret = !con( 401 !if(!empty(vdataList), InsNoData, InsData), 402 !if(isLds, (ins), (ins TFE:$tfe)) 403 ); 404} 405 406class getMUBUFDwords<RegisterClass regClass> { 407 string regClassAsInt = !cast<string>(regClass); 408 int ret = 409 !if(!eq(regClassAsInt, !cast<string>(VGPR_32)), 1, 410 !if(!eq(regClassAsInt, !cast<string>(VReg_64)), 2, 411 !if(!eq(regClassAsInt, !cast<string>(VReg_96)), 3, 412 !if(!eq(regClassAsInt, !cast<string>(VReg_128)), 4, 413 0)))); 414} 415 416class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 417 dag ret = 418 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 419 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 420 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 421 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 422 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 423 (ins)))))); 424} 425 426class getMUBUFAsmOps<int addrKind> { 427 string Pfx = 428 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 429 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 430 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 431 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 432 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 433 ""))))); 434 string ret = Pfx # "$offset"; 435} 436 437class MUBUF_SetupAddr<int addrKind> { 438 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 439 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 440 441 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 442 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 443 444 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 445 446 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 447} 448 449class MUBUF_Load_Pseudo <string opName, 450 int addrKind, 451 RegisterClass vdataClass, 452 bit HasTiedDest = 0, 453 bit isLds = 0, 454 list<dag> pattern=[], 455 // Workaround bug bz30254 456 int addrKindCopy = addrKind> 457 : MUBUF_Pseudo<opName, 458 (outs vdataClass:$vdata), 459 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 460 !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 461 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 462 !if(isLds, " lds", "$tfe"), 463 pattern>, 464 MUBUF_SetupAddr<addrKindCopy> { 465 let PseudoInstr = opName # !if(isLds, "_lds", "") # 466 "_" # getAddrName<addrKindCopy>.ret; 467 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 468 469 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 470 let mayLoad = 1; 471 let mayStore = 0; 472 let maybeAtomic = 1; 473 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 474 let has_tfe = !if(isLds, 0, 1); 475 let lds = isLds; 476 let dwords = getMUBUFDwords<vdataClass>.ret; 477} 478 479// FIXME: tfe can't be an operand because it requires a separate 480// opcode because it needs an N+1 register class dest register. 481multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 482 ValueType load_vt = i32, 483 SDPatternOperator ld = null_frag, 484 bit TiedDest = 0, 485 bit isLds = 0> { 486 487 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 488 TiedDest, isLds, 489 !if(isLds, 490 [], 491 [(set load_vt:$vdata, 492 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 493 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 494 495 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 496 TiedDest, isLds, 497 !if(isLds, 498 [], 499 [(set load_vt:$vdata, 500 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 501 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 502 503 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 504 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 505 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 506 507 let DisableWQM = 1 in { 508 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>; 509 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 510 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 511 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 512 } 513} 514 515multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass, 516 ValueType load_vt = i32, 517 SDPatternOperator ld_nolds = null_frag, 518 SDPatternOperator ld_lds = null_frag> { 519 defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>; 520 defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>; 521} 522 523class MUBUF_Store_Pseudo <string opName, 524 int addrKind, 525 RegisterClass vdataClass, 526 list<dag> pattern=[], 527 // Workaround bug bz30254 528 int addrKindCopy = addrKind, 529 RegisterClass vdataClassCopy = vdataClass> 530 : MUBUF_Pseudo<opName, 531 (outs), 532 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 533 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 534 pattern>, 535 MUBUF_SetupAddr<addrKindCopy> { 536 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 537 let mayLoad = 0; 538 let mayStore = 1; 539 let maybeAtomic = 1; 540 let dwords = getMUBUFDwords<vdataClass>.ret; 541} 542 543multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 544 ValueType store_vt = i32, 545 SDPatternOperator st = null_frag> { 546 547 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 548 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 549 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 550 MUBUFAddr64Table<0, NAME>; 551 552 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 553 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 554 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 555 MUBUFAddr64Table<1, NAME>; 556 557 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 558 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 559 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 560 561 let DisableWQM = 1 in { 562 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 563 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 564 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 565 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 566 } 567} 568 569class MUBUF_Pseudo_Store_Lds<string opName> 570 : MUBUF_Pseudo<opName, 571 (outs), 572 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 573 " $srsrc, $soffset$offset lds$glc$slc"> { 574 let mayLoad = 0; 575 let mayStore = 1; 576 let maybeAtomic = 1; 577 578 let has_vdata = 0; 579 let has_vaddr = 0; 580 let has_tfe = 0; 581 let lds = 1; 582 583 let Uses = [EXEC, M0]; 584 let AsmMatchConverter = "cvtMubufLds"; 585} 586 587class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 588 list<RegisterClass> vaddrList=[]> { 589 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 590 dag ret = !if(vdata_in, 591 !if(!empty(vaddrList), 592 (ins vdataClass:$vdata_in, 593 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 594 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 595 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 596 ), 597 !if(!empty(vaddrList), 598 (ins vdataClass:$vdata, 599 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 600 (ins vdataClass:$vdata, vaddrClass:$vaddr, 601 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 602 )); 603} 604 605class getMUBUFAtomicIns<int addrKind, 606 RegisterClass vdataClass, 607 bit vdata_in, 608 // Workaround bug bz30254 609 RegisterClass vdataClassCopy=vdataClass> { 610 dag ret = 611 !if(!eq(addrKind, BUFAddrKind.Offset), 612 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 613 !if(!eq(addrKind, BUFAddrKind.OffEn), 614 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 615 !if(!eq(addrKind, BUFAddrKind.IdxEn), 616 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 617 !if(!eq(addrKind, BUFAddrKind.BothEn), 618 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 619 !if(!eq(addrKind, BUFAddrKind.Addr64), 620 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 621 (ins)))))); 622} 623 624class MUBUF_Atomic_Pseudo<string opName, 625 int addrKind, 626 dag outs, 627 dag ins, 628 string asmOps, 629 list<dag> pattern=[], 630 // Workaround bug bz30254 631 int addrKindCopy = addrKind> 632 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 633 MUBUF_SetupAddr<addrKindCopy> { 634 let mayStore = 1; 635 let mayLoad = 1; 636 let hasPostISelHook = 1; 637 let hasSideEffects = 1; 638 let DisableWQM = 1; 639 let has_glc = 0; 640 let has_tfe = 0; 641 let maybeAtomic = 1; 642} 643 644class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 645 RegisterClass vdataClass, 646 list<dag> pattern=[], 647 // Workaround bug bz30254 648 int addrKindCopy = addrKind, 649 RegisterClass vdataClassCopy = vdataClass> 650 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 651 (outs), 652 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 653 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 654 pattern>, 655 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 656 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 657 let glc_value = 0; 658 let AsmMatchConverter = "cvtMubufAtomic"; 659} 660 661class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 662 RegisterClass vdataClass, 663 list<dag> pattern=[], 664 // Workaround bug bz30254 665 int addrKindCopy = addrKind, 666 RegisterClass vdataClassCopy = vdataClass> 667 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 668 (outs vdataClassCopy:$vdata), 669 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 670 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 671 pattern>, 672 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 673 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 674 let glc_value = 1; 675 let Constraints = "$vdata = $vdata_in"; 676 let DisableEncoding = "$vdata_in"; 677 let AsmMatchConverter = "cvtMubufAtomicReturn"; 678} 679 680multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 681 RegisterClass vdataClass, 682 ValueType vdataType, 683 SDPatternOperator atomic> { 684 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 685 MUBUFAddr64Table <0, NAME>; 686 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 687 MUBUFAddr64Table <1, NAME>; 688 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 689 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 690 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 691} 692 693multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 694 RegisterClass vdataClass, 695 ValueType vdataType, 696 SDPatternOperator atomic> { 697 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 698 [(set vdataType:$vdata, 699 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 700 vdataType:$vdata_in))]>, 701 MUBUFAddr64Table <0, NAME # "_RTN">; 702 703 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 704 [(set vdataType:$vdata, 705 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 706 vdataType:$vdata_in))]>, 707 MUBUFAddr64Table <1, NAME # "_RTN">; 708 709 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 710 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 711 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 712} 713 714multiclass MUBUF_Pseudo_Atomics <string opName, 715 RegisterClass vdataClass, 716 ValueType vdataType, 717 SDPatternOperator atomic> : 718 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 719 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 720 721 722//===----------------------------------------------------------------------===// 723// MUBUF Instructions 724//===----------------------------------------------------------------------===// 725 726defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 727 "buffer_load_format_x", VGPR_32 728>; 729defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 730 "buffer_load_format_xy", VReg_64 731>; 732defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 733 "buffer_load_format_xyz", VReg_96 734>; 735defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 736 "buffer_load_format_xyzw", VReg_128 737>; 738defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 739 "buffer_store_format_x", VGPR_32 740>; 741defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 742 "buffer_store_format_xy", VReg_64 743>; 744defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 745 "buffer_store_format_xyz", VReg_96 746>; 747defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 748 "buffer_store_format_xyzw", VReg_128 749>; 750 751let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 752 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 753 "buffer_load_format_d16_x", VGPR_32 754 >; 755 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 756 "buffer_load_format_d16_xy", VReg_64 757 >; 758 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 759 "buffer_load_format_d16_xyz", VReg_96 760 >; 761 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 762 "buffer_load_format_d16_xyzw", VReg_128 763 >; 764 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 765 "buffer_store_format_d16_x", VGPR_32 766 >; 767 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 768 "buffer_store_format_d16_xy", VReg_64 769 >; 770 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 771 "buffer_store_format_d16_xyz", VReg_96 772 >; 773 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 774 "buffer_store_format_d16_xyzw", VReg_128 775 >; 776} // End HasUnpackedD16VMem. 777 778let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 779 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 780 "buffer_load_format_d16_x", VGPR_32 781 >; 782 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 783 "buffer_load_format_d16_xy", VGPR_32 784 >; 785 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 786 "buffer_load_format_d16_xyz", VReg_64 787 >; 788 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 789 "buffer_load_format_d16_xyzw", VReg_64 790 >; 791 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 792 "buffer_store_format_d16_x", VGPR_32 793 >; 794 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 795 "buffer_store_format_d16_xy", VGPR_32 796 >; 797 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 798 "buffer_store_format_d16_xyz", VReg_64 799 >; 800 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 801 "buffer_store_format_d16_xyzw", VReg_64 802 >; 803} // End HasPackedD16VMem. 804 805defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 806 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 807>; 808defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 809 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 810>; 811defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 812 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 813>; 814defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 815 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 816>; 817defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 818 "buffer_load_dword", VGPR_32, i32, mubuf_load 819>; 820defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 821 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 822>; 823defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 824 "buffer_load_dwordx3", VReg_96, v3i32, mubuf_load 825>; 826defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 827 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 828>; 829 830// This is not described in AMD documentation, 831// but 'lds' versions of these opcodes are available 832// in at least GFX8+ chips. See Bug 37653. 833let SubtargetPredicate = isVI in { 834defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 835 "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1 836>; 837defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 838 "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1 839>; 840defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 841 "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1 842>; 843} 844 845defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 846 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 847>; 848defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 849 "buffer_store_short", VGPR_32, i32, truncstorei16_global 850>; 851defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 852 "buffer_store_dword", VGPR_32, i32, store_global 853>; 854defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 855 "buffer_store_dwordx2", VReg_64, v2i32, store_global 856>; 857defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 858 "buffer_store_dwordx3", VReg_96, v3i32, store_global 859>; 860defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 861 "buffer_store_dwordx4", VReg_128, v4i32, store_global 862>; 863defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 864 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 865>; 866defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 867 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 868>; 869defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 870 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 871>; 872defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 873 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 874>; 875defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 876 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 877>; 878defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 879 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 880>; 881defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 882 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 883>; 884defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 885 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 886>; 887defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 888 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 889>; 890defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 891 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 892>; 893defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 894 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 895>; 896defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 897 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 898>; 899defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 900 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 901>; 902defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 903 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 904>; 905defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 906 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 907>; 908defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 909 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 910>; 911defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 912 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 913>; 914defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 915 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 916>; 917defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 918 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 919>; 920defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 921 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 922>; 923defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 924 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 925>; 926defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 927 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 928>; 929defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 930 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 931>; 932defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 933 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 934>; 935defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 936 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 937>; 938defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 939 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 940>; 941 942let SubtargetPredicate = isVI in { 943def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 944} 945 946let SubtargetPredicate = isSI in { // isn't on CI & VI 947/* 948defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 949defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 950defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 951defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 952defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 953defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 954defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 955defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 956*/ 957 958def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 959 int_amdgcn_buffer_wbinvl1_sc>; 960} 961 962let SubtargetPredicate = HasD16LoadStore in { 963 964defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 965 "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 966>; 967 968defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 969 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 970>; 971 972defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 973 "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 974>; 975 976defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 977 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 978>; 979 980defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 981 "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 982>; 983 984defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 985 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 986>; 987 988defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 989 "buffer_store_byte_d16_hi", VGPR_32, i32 990>; 991 992defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 993 "buffer_store_short_d16_hi", VGPR_32, i32 994>; 995 996defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 997 "buffer_load_format_d16_hi_x", VGPR_32 998>; 999defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1000 "buffer_store_format_d16_hi_x", VGPR_32 1001>; 1002 1003} // End HasD16LoadStore 1004 1005def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1006 int_amdgcn_buffer_wbinvl1>; 1007 1008//===----------------------------------------------------------------------===// 1009// MTBUF Instructions 1010//===----------------------------------------------------------------------===// 1011 1012defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 1013defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 1014defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96>; 1015defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 1016defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 1017defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 1018defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96>; 1019defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 1020 1021let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1022 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1023 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64>; 1024 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96>; 1025 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128>; 1026 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1027 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64>; 1028 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96>; 1029 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>; 1030} // End HasUnpackedD16VMem. 1031 1032let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1033 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1034 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32>; 1035 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64>; 1036 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64>; 1037 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1038 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32>; 1039 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64>; 1040 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>; 1041} // End HasPackedD16VMem. 1042 1043let SubtargetPredicate = isCIVI in { 1044 1045//===----------------------------------------------------------------------===// 1046// Instruction definitions for CI and newer. 1047//===----------------------------------------------------------------------===// 1048 1049def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1050 int_amdgcn_buffer_wbinvl1_vol>; 1051 1052} // End let SubtargetPredicate = isCIVI 1053 1054//===----------------------------------------------------------------------===// 1055// MUBUF Patterns 1056//===----------------------------------------------------------------------===// 1057 1058def extract_glc : SDNodeXForm<imm, [{ 1059 return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8); 1060}]>; 1061 1062def extract_slc : SDNodeXForm<imm, [{ 1063 return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8); 1064}]>; 1065 1066//===----------------------------------------------------------------------===// 1067// buffer_load/store_format patterns 1068//===----------------------------------------------------------------------===// 1069 1070multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1071 string opcode> { 1072 def : GCNPat< 1073 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1074 imm:$cachepolicy, 0)), 1075 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1076 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1077 >; 1078 1079 def : GCNPat< 1080 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1081 imm:$cachepolicy, 0)), 1082 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1083 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1084 >; 1085 1086 def : GCNPat< 1087 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1088 imm:$cachepolicy, imm)), 1089 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1090 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1091 >; 1092 1093 def : GCNPat< 1094 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1095 imm:$cachepolicy, imm)), 1096 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1097 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1098 $rsrc, $soffset, (as_i16imm $offset), 1099 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1100 >; 1101} 1102 1103defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1104defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1105defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1106defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1107defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1108defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1109defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1110defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1111 1112let SubtargetPredicate = HasUnpackedD16VMem in { 1113 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1114 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1115 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1116 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1117} // End HasUnpackedD16VMem. 1118 1119let SubtargetPredicate = HasPackedD16VMem in { 1120 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1121 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1122 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1123 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1124 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1125 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1126} // End HasPackedD16VMem. 1127 1128defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1129defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1130defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1131defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1132defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1133defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1134defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1135defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1136defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1137defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1138defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1139defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1140 1141multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1142 string opcode> { 1143 def : GCNPat< 1144 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1145 imm:$cachepolicy, 0), 1146 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1147 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1148 >; 1149 1150 def : GCNPat< 1151 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1152 imm:$cachepolicy, 0), 1153 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1154 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1155 >; 1156 1157 def : GCNPat< 1158 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1159 imm:$cachepolicy, imm), 1160 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1161 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1162 >; 1163 1164 def : GCNPat< 1165 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1166 imm:$cachepolicy, imm), 1167 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1168 $vdata, 1169 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1170 $rsrc, $soffset, (as_i16imm $offset), 1171 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1172 >; 1173} 1174 1175defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1176defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1177defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1178defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1179defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1180defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1181defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1182defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1183 1184let SubtargetPredicate = HasUnpackedD16VMem in { 1185 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1186 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1187 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1188 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1189} // End HasUnpackedD16VMem. 1190 1191let SubtargetPredicate = HasPackedD16VMem in { 1192 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1193 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1194 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1195 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1196 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1197 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1198} // End HasPackedD16VMem. 1199 1200defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1201defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1202defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1203defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1204defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1205defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1206defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1207defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1208defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1209defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1210 1211//===----------------------------------------------------------------------===// 1212// buffer_atomic patterns 1213//===----------------------------------------------------------------------===// 1214 1215multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1216 string opcode> { 1217 def : GCNPat< 1218 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1219 0, i32:$soffset, imm:$offset, 1220 imm:$cachepolicy, 0)), 1221 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1222 (as_i16imm $offset), (extract_slc $cachepolicy)) 1223 >; 1224 1225 def : GCNPat< 1226 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1227 0, i32:$soffset, imm:$offset, 1228 imm:$cachepolicy, imm)), 1229 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1230 (as_i16imm $offset), (extract_slc $cachepolicy)) 1231 >; 1232 1233 def : GCNPat< 1234 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1235 i32:$voffset, i32:$soffset, imm:$offset, 1236 imm:$cachepolicy, 0)), 1237 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1238 (as_i16imm $offset), (extract_slc $cachepolicy)) 1239 >; 1240 1241 def : GCNPat< 1242 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1243 i32:$voffset, i32:$soffset, imm:$offset, 1244 imm:$cachepolicy, imm)), 1245 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1246 $vdata_in, 1247 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1248 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1249 >; 1250} 1251 1252defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1253defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1254defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1255defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1256defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1257defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1258defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1259defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1260defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1261defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1262defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1263defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1264defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1265defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1266defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1267defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1268defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1269defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1270defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1271defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1272 1273def : GCNPat< 1274 (SIbuffer_atomic_cmpswap 1275 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1276 0, i32:$soffset, imm:$offset, 1277 imm:$cachepolicy, 0), 1278 (EXTRACT_SUBREG 1279 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1280 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1281 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1282 sub0) 1283>; 1284 1285def : GCNPat< 1286 (SIbuffer_atomic_cmpswap 1287 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1288 0, i32:$soffset, imm:$offset, 1289 imm:$cachepolicy, imm), 1290 (EXTRACT_SUBREG 1291 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1292 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1293 $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1294 sub0) 1295>; 1296 1297def : GCNPat< 1298 (SIbuffer_atomic_cmpswap 1299 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1300 i32:$voffset, i32:$soffset, imm:$offset, 1301 imm:$cachepolicy, 0), 1302 (EXTRACT_SUBREG 1303 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1304 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1305 $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1306 sub0) 1307>; 1308 1309def : GCNPat< 1310 (SIbuffer_atomic_cmpswap 1311 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1312 i32:$voffset, i32:$soffset, imm:$offset, 1313 imm:$cachepolicy, imm), 1314 (EXTRACT_SUBREG 1315 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1316 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1317 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1318 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1319 sub0) 1320>; 1321 1322class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1323 PatFrag constant_ld> : GCNPat < 1324 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1325 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1326 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1327 >; 1328 1329multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1330 ValueType vt, PatFrag atomic_ld> { 1331 def : GCNPat < 1332 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1333 i16:$offset, i1:$slc))), 1334 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1335 >; 1336 1337 def : GCNPat < 1338 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1339 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1340 >; 1341} 1342 1343let SubtargetPredicate = isSICI in { 1344def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1345def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1346def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1347def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1348 1349defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1350defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1351} // End SubtargetPredicate = isSICI 1352 1353multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1354 PatFrag ld> { 1355 1356 def : GCNPat < 1357 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1358 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1359 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1360 >; 1361} 1362 1363let OtherPredicates = [Has16BitInsts] in { 1364 1365defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1366defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1367defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1368defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1369 1370defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1371 1372} // End OtherPredicates = [Has16BitInsts] 1373 1374multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1375 MUBUF_Pseudo InstrOffset, 1376 ValueType vt, PatFrag ld> { 1377 def : GCNPat < 1378 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1379 i32:$soffset, u16imm:$offset))), 1380 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1381 >; 1382 1383 def : GCNPat < 1384 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1385 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1386 >; 1387} 1388 1389// XXX - Is it possible to have a complex pattern in a PatFrag? 1390multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1391 MUBUF_Pseudo InstrOffset, 1392 ValueType vt, PatFrag ld_frag> { 1393 def : GCNPat < 1394 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1395 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $in) 1396 >; 1397 1398 def : GCNPat < 1399 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1400 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $in) 1401 >; 1402} 1403 1404defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1405defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1406defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1407defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1408defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1409defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1410defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1411defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1412defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1413defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1414defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1415 1416let OtherPredicates = [D16PreservesUnusedBits] in { 1417defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1418defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1419defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1420defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1421defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1422defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1423 1424defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1425defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1426defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1427defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1428defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1429defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1430} 1431multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1432 ValueType vt, PatFrag atomic_st> { 1433 // Store follows atomic op convention so address is forst 1434 def : GCNPat < 1435 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1436 i16:$offset, i1:$slc), vt:$val), 1437 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1438 >; 1439 1440 def : GCNPat < 1441 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1442 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1443 >; 1444} 1445let SubtargetPredicate = isSICI in { 1446defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1447defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1448} // End Predicates = isSICI 1449 1450 1451multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1452 PatFrag st> { 1453 1454 def : GCNPat < 1455 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1456 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1457 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1458 >; 1459} 1460 1461defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1462defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1463 1464multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1465 MUBUF_Pseudo InstrOffset, 1466 ValueType vt, PatFrag st> { 1467 def : GCNPat < 1468 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1469 i32:$soffset, u16imm:$offset)), 1470 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1471 >; 1472 1473 def : GCNPat < 1474 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1475 u16imm:$offset)), 1476 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1477 >; 1478} 1479 1480defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1481defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1482defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1483defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1484defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1485defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1486defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private>; 1487defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1488 1489 1490let OtherPredicates = [D16PreservesUnusedBits] in { 1491 // Hiding the extract high pattern in the PatFrag seems to not 1492 // automatically increase the complexity. 1493let AddedComplexity = 1 in { 1494defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1495defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1496} 1497} 1498 1499//===----------------------------------------------------------------------===// 1500// MTBUF Patterns 1501//===----------------------------------------------------------------------===// 1502 1503//===----------------------------------------------------------------------===// 1504// tbuffer_load/store_format patterns 1505//===----------------------------------------------------------------------===// 1506 1507multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1508 string opcode> { 1509 def : GCNPat< 1510 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1511 imm:$format, imm:$cachepolicy, 0)), 1512 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1513 (as_i8imm $format), 1514 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1515 >; 1516 1517 def : GCNPat< 1518 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1519 imm:$format, imm:$cachepolicy, imm)), 1520 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1521 (as_i8imm $format), 1522 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1523 >; 1524 1525 def : GCNPat< 1526 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1527 imm:$format, imm:$cachepolicy, 0)), 1528 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1529 (as_i8imm $format), 1530 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1531 >; 1532 1533 def : GCNPat< 1534 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1535 imm:$format, imm:$cachepolicy, imm)), 1536 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1537 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1538 $rsrc, $soffset, (as_i16imm $offset), 1539 (as_i8imm $format), 1540 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1541 >; 1542} 1543 1544defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1545defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1546defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1547defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1548defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1549defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1550defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1551defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1552 1553let SubtargetPredicate = HasUnpackedD16VMem in { 1554 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1555 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1556 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1557} // End HasUnpackedD16VMem. 1558 1559let SubtargetPredicate = HasPackedD16VMem in { 1560 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1561 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1562 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1563} // End HasPackedD16VMem. 1564 1565multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1566 string opcode> { 1567 def : GCNPat< 1568 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1569 imm:$format, imm:$cachepolicy, 0), 1570 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1571 (as_i16imm $offset), (as_i8imm $format), 1572 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1573 >; 1574 1575 def : GCNPat< 1576 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1577 imm:$format, imm:$cachepolicy, imm), 1578 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1579 (as_i16imm $offset), (as_i8imm $format), 1580 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1581 >; 1582 1583 def : GCNPat< 1584 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1585 imm:$format, imm:$cachepolicy, 0), 1586 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1587 (as_i16imm $offset), (as_i8imm $format), 1588 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1589 >; 1590 1591 def : GCNPat< 1592 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1593 imm:$offset, imm:$format, imm:$cachepolicy, imm), 1594 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1595 $vdata, 1596 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1597 $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), 1598 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1599 >; 1600} 1601 1602defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1603defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1604defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1605defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1606defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1607defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1608defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1609defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1610 1611let SubtargetPredicate = HasUnpackedD16VMem in { 1612 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1613 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1614 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1615} // End HasUnpackedD16VMem. 1616 1617let SubtargetPredicate = HasPackedD16VMem in { 1618 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1619 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1620 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1621} // End HasPackedD16VMem. 1622 1623//===----------------------------------------------------------------------===// 1624// Target instructions, move to the appropriate target TD file 1625//===----------------------------------------------------------------------===// 1626 1627//===----------------------------------------------------------------------===// 1628// SI 1629//===----------------------------------------------------------------------===// 1630 1631class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1632 MUBUF_Real<op, ps>, 1633 Enc64, 1634 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1635 let AssemblerPredicate=isSICI; 1636 let DecoderNamespace="SICI"; 1637 1638 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1639 let Inst{12} = ps.offen; 1640 let Inst{13} = ps.idxen; 1641 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1642 let Inst{15} = ps.addr64; 1643 let Inst{16} = !if(ps.lds, 1, 0); 1644 let Inst{24-18} = op; 1645 let Inst{31-26} = 0x38; //encoding 1646 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1647 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1648 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1649 let Inst{54} = !if(ps.has_slc, slc, ?); 1650 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1651 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1652} 1653 1654multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1655 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1656 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1657 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1658 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1659 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1660} 1661 1662multiclass MUBUF_Real_AllAddr_Lds_si<bits<7> op> { 1663 1664 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1665 MUBUFLdsTable<0, NAME # "_OFFSET_si">; 1666 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1667 MUBUFLdsTable<0, NAME # "_ADDR64_si">; 1668 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1669 MUBUFLdsTable<0, NAME # "_OFFEN_si">; 1670 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1671 MUBUFLdsTable<0, NAME # "_IDXEN_si">; 1672 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1673 MUBUFLdsTable<0, NAME # "_BOTHEN_si">; 1674 1675 def _LDS_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1676 MUBUFLdsTable<1, NAME # "_OFFSET_si">; 1677 def _LDS_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1678 MUBUFLdsTable<1, NAME # "_ADDR64_si">; 1679 def _LDS_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1680 MUBUFLdsTable<1, NAME # "_OFFEN_si">; 1681 def _LDS_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1682 MUBUFLdsTable<1, NAME # "_IDXEN_si">; 1683 def _LDS_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1684 MUBUFLdsTable<1, NAME # "_BOTHEN_si">; 1685} 1686 1687multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1688 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1689 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1690 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1691 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1692 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1693} 1694 1695defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_si <0x00>; 1696defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1697defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1698defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1699defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1700defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1701defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1702defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1703defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_si <0x08>; 1704defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_si <0x09>; 1705defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_si <0x0a>; 1706defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_si <0x0b>; 1707defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_si <0x0c>; 1708defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1709defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1710defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1711defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1712defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1713defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1714defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1715defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1716defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1717 1718defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1719defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1720defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1721defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1722//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1723defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1724defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1725defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1726defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1727defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1728defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1729defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1730defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1731defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1732 1733//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1734//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1735//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1736defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1737defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1738defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1739defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1740//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1741defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1742defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1743defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1744defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1745defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1746defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1747defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1748defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1749defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1750// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1751//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1752//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1753//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1754 1755def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1756def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1757 1758class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1759 MTBUF_Real<ps>, 1760 Enc64, 1761 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1762 let AssemblerPredicate=isSICI; 1763 let DecoderNamespace="SICI"; 1764 1765 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1766 let Inst{12} = ps.offen; 1767 let Inst{13} = ps.idxen; 1768 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1769 let Inst{15} = ps.addr64; 1770 let Inst{18-16} = op; 1771 let Inst{22-19} = dfmt; 1772 let Inst{25-23} = nfmt; 1773 let Inst{31-26} = 0x3a; //encoding 1774 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1775 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1776 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1777 let Inst{54} = !if(ps.has_slc, slc, ?); 1778 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1779 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1780} 1781 1782multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1783 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1784 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1785 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1786 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1787 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1788} 1789 1790defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1791defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1792defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1793defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1794defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1795defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1796defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1797defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1798 1799//===----------------------------------------------------------------------===// 1800// CI 1801// MTBUF - GFX6, GFX7. 1802//===----------------------------------------------------------------------===// 1803 1804class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1805 MUBUF_Real_si<op, ps> { 1806 let AssemblerPredicate=isCIOnly; 1807 let DecoderNamespace="CI"; 1808} 1809 1810def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1811 1812 1813//===----------------------------------------------------------------------===// 1814// VI 1815//===----------------------------------------------------------------------===// 1816 1817class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1818 MUBUF_Real<op, ps>, 1819 Enc64, 1820 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1821 let AssemblerPredicate=isVI; 1822 let DecoderNamespace="VI"; 1823 1824 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1825 let Inst{12} = ps.offen; 1826 let Inst{13} = ps.idxen; 1827 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1828 let Inst{16} = !if(ps.lds, 1, 0); 1829 let Inst{17} = !if(ps.has_slc, slc, ?); 1830 let Inst{24-18} = op; 1831 let Inst{31-26} = 0x38; //encoding 1832 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1833 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1834 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1835 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1836 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1837} 1838 1839multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1840 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1841 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1842 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1843 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1844} 1845 1846multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 1847 1848 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1849 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 1850 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1851 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 1852 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1853 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 1854 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1855 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 1856 1857 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1858 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 1859 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1860 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 1861 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1862 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 1863 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1864 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 1865} 1866 1867class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 1868 MUBUF_Real<op, ps>, 1869 Enc64, 1870 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 1871 let AssemblerPredicate=HasUnpackedD16VMem; 1872 let DecoderNamespace="GFX80_UNPACKED"; 1873 1874 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1875 let Inst{12} = ps.offen; 1876 let Inst{13} = ps.idxen; 1877 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1878 let Inst{16} = !if(ps.lds, 1, 0); 1879 let Inst{17} = !if(ps.has_slc, slc, ?); 1880 let Inst{24-18} = op; 1881 let Inst{31-26} = 0x38; //encoding 1882 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1883 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1884 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1885 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1886 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1887} 1888 1889multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 1890 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1891 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1892 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1893 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1894} 1895 1896multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1897 MUBUF_Real_AllAddr_vi<op> { 1898 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1899 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1900 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1901 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1902} 1903 1904defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 1905defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1906defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1907defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1908defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1909defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1910defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1911defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1912let SubtargetPredicate = HasUnpackedD16VMem in { 1913 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 1914 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 1915 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 1916 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 1917 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 1918 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 1919 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 1920 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 1921} // End HasUnpackedD16VMem. 1922let SubtargetPredicate = HasPackedD16VMem in { 1923 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 1924 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 1925 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 1926 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 1927 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 1928 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 1929 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 1930 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 1931} // End HasPackedD16VMem. 1932defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 1933defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 1934defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 1935defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 1936defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 1937defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 1938defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 1939defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 1940defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1941defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1942defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1943defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1944defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1945defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1946defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1947defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1948 1949defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1950defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1951defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1952defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1953defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1954defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1955 1956defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 1957defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 1958 1959defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1960defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1961defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1962defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1963defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1964defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1965defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1966defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1967defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1968defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1969defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1970defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1971defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1972 1973defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1974defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1975defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1976defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1977defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1978defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1979defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1980defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1981defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1982defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1983defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1984defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1985defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1986 1987def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 1988 1989def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 1990def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 1991 1992class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 1993 MTBUF_Real<ps>, 1994 Enc64, 1995 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1996 let AssemblerPredicate=isVI; 1997 let DecoderNamespace="VI"; 1998 1999 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2000 let Inst{12} = ps.offen; 2001 let Inst{13} = ps.idxen; 2002 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2003 let Inst{18-15} = op; 2004 let Inst{22-19} = dfmt; 2005 let Inst{25-23} = nfmt; 2006 let Inst{31-26} = 0x3a; //encoding 2007 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2008 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2009 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2010 let Inst{54} = !if(ps.has_slc, slc, ?); 2011 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2012 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2013} 2014 2015multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2016 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2017 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2018 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2019 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2020} 2021 2022class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2023 MTBUF_Real<ps>, 2024 Enc64, 2025 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2026 let AssemblerPredicate=HasUnpackedD16VMem; 2027 let DecoderNamespace="GFX80_UNPACKED"; 2028 2029 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2030 let Inst{12} = ps.offen; 2031 let Inst{13} = ps.idxen; 2032 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2033 let Inst{18-15} = op; 2034 let Inst{22-19} = dfmt; 2035 let Inst{25-23} = nfmt; 2036 let Inst{31-26} = 0x3a; //encoding 2037 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2038 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2039 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2040 let Inst{54} = !if(ps.has_slc, slc, ?); 2041 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2042 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2043} 2044 2045multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2046 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2047 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2048 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2049 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2050} 2051 2052defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2053defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2054defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2055defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2056defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2057defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2058defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2059defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2060let SubtargetPredicate = HasUnpackedD16VMem in { 2061 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2062 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2063 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2064 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2065 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2066 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2067 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2068 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2069} // End HasUnpackedD16VMem. 2070let SubtargetPredicate = HasPackedD16VMem in { 2071 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2072 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2073 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2074 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2075 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2076 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2077 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2078 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2079} // End HasUnpackedD16VMem. 2080 2081def MUBUFInfoTable : GenericTable { 2082 let FilterClass = "MUBUF_Pseudo"; 2083 let CppTypeName = "MUBUFInfo"; 2084 let Fields = ["Opcode", "BaseOpcode", "dwords", "has_vaddr", "has_srsrc", "has_soffset"]; 2085 2086 let PrimaryKey = ["Opcode"]; 2087 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2088} 2089 2090def getMUBUFInfoFromOpcode : SearchIndex { 2091 let Table = MUBUFInfoTable; 2092 let Key = ["Opcode"]; 2093} 2094 2095def getMUBUFInfoFromBaseOpcodeAndDwords : SearchIndex { 2096 let Table = MUBUFInfoTable; 2097 let Key = ["BaseOpcode", "dwords"]; 2098} 2099