1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratch : ComplexPattern<i64, 4, "SelectMUBUFScratch">; 15def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 16def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 17def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 18def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">; 19def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">; 20 21class MubufLoad <SDPatternOperator op> : PatFrag < 22 (ops node:$ptr), (op node:$ptr), [{ 23 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 24 return AS == AMDGPUAS::GLOBAL_ADDRESS || 25 AS == AMDGPUAS::CONSTANT_ADDRESS; 26}]>; 27 28def mubuf_load : MubufLoad <load>; 29def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 30def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 31def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 32def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 33def mubuf_load_atomic : MubufLoad <atomic_load>; 34 35def BUFAddrKind { 36 int Offset = 0; 37 int OffEn = 1; 38 int IdxEn = 2; 39 int BothEn = 3; 40 int Addr64 = 4; 41} 42 43class getAddrName<int addrKind> { 44 string ret = 45 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 46 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 47 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 48 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 49 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 50 ""))))); 51} 52 53class MUBUFAddr64Table <bit is_addr64, string suffix = ""> { 54 bit IsAddr64 = is_addr64; 55 string OpName = NAME # suffix; 56} 57 58//===----------------------------------------------------------------------===// 59// MTBUF classes 60//===----------------------------------------------------------------------===// 61 62class MTBUF_Pseudo <string opName, dag outs, dag ins, 63 string asmOps, list<dag> pattern=[]> : 64 InstSI<outs, ins, "", pattern>, 65 SIMCInstr<opName, SIEncodingFamily.NONE> { 66 67 let isPseudo = 1; 68 let isCodeGenOnly = 1; 69 let Size = 8; 70 let UseNamedOperandTable = 1; 71 72 string Mnemonic = opName; 73 string AsmOperands = asmOps; 74 75 let VM_CNT = 1; 76 let EXP_CNT = 1; 77 let MTBUF = 1; 78 let Uses = [EXEC]; 79 80 let hasSideEffects = 0; 81 let SchedRW = [WriteVMEM]; 82} 83 84class MTBUF_Real <MTBUF_Pseudo ps> : 85 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>, 86 Enc64 { 87 88 let isPseudo = 0; 89 let isCodeGenOnly = 0; 90 91 // copy relevant pseudo op flags 92 let SubtargetPredicate = ps.SubtargetPredicate; 93 let AsmMatchConverter = ps.AsmMatchConverter; 94 let Constraints = ps.Constraints; 95 let DisableEncoding = ps.DisableEncoding; 96 let TSFlags = ps.TSFlags; 97 98 bits<8> vdata; 99 bits<12> offset; 100 bits<1> offen; 101 bits<1> idxen; 102 bits<1> glc; 103 bits<1> addr64; 104 bits<4> dfmt; 105 bits<3> nfmt; 106 bits<8> vaddr; 107 bits<7> srsrc; 108 bits<1> slc; 109 bits<1> tfe; 110 bits<8> soffset; 111 112 let Inst{11-0} = offset; 113 let Inst{12} = offen; 114 let Inst{13} = idxen; 115 let Inst{14} = glc; 116 let Inst{22-19} = dfmt; 117 let Inst{25-23} = nfmt; 118 let Inst{31-26} = 0x3a; //encoding 119 let Inst{39-32} = vaddr; 120 let Inst{47-40} = vdata; 121 let Inst{52-48} = srsrc{6-2}; 122 let Inst{54} = slc; 123 let Inst{55} = tfe; 124 let Inst{63-56} = soffset; 125} 126 127class MTBUF_Load_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo < 128 opName, (outs regClass:$dst), 129 (ins u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc, i1imm:$addr64, 130 i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr, SReg_128:$srsrc, 131 i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset), 132 " $dst, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"# 133 " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> { 134 let mayLoad = 1; 135 let mayStore = 0; 136} 137 138class MTBUF_Store_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo < 139 opName, (outs), 140 (ins regClass:$vdata, u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc, 141 i1imm:$addr64, i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr, 142 SReg_128:$srsrc, i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset), 143 " $vdata, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"# 144 " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> { 145 let mayLoad = 0; 146 let mayStore = 1; 147} 148 149//===----------------------------------------------------------------------===// 150// MUBUF classes 151//===----------------------------------------------------------------------===// 152 153class MUBUF_Pseudo <string opName, dag outs, dag ins, 154 string asmOps, list<dag> pattern=[]> : 155 InstSI<outs, ins, "", pattern>, 156 SIMCInstr<opName, SIEncodingFamily.NONE> { 157 158 let isPseudo = 1; 159 let isCodeGenOnly = 1; 160 let Size = 8; 161 let UseNamedOperandTable = 1; 162 163 string Mnemonic = opName; 164 string AsmOperands = asmOps; 165 166 let VM_CNT = 1; 167 let EXP_CNT = 1; 168 let MUBUF = 1; 169 let Uses = [EXEC]; 170 let hasSideEffects = 0; 171 let SchedRW = [WriteVMEM]; 172 173 let AsmMatchConverter = "cvtMubuf"; 174 175 bits<1> offen = 0; 176 bits<1> idxen = 0; 177 bits<1> addr64 = 0; 178 bits<1> has_vdata = 1; 179 bits<1> has_vaddr = 1; 180 bits<1> has_glc = 1; 181 bits<1> glc_value = 0; // the value for glc if no such operand 182 bits<1> has_srsrc = 1; 183 bits<1> has_soffset = 1; 184 bits<1> has_offset = 1; 185 bits<1> has_slc = 1; 186 bits<1> has_tfe = 1; 187} 188 189class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 190 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 191 192 let isPseudo = 0; 193 let isCodeGenOnly = 0; 194 195 // copy relevant pseudo op flags 196 let SubtargetPredicate = ps.SubtargetPredicate; 197 let AsmMatchConverter = ps.AsmMatchConverter; 198 let Constraints = ps.Constraints; 199 let DisableEncoding = ps.DisableEncoding; 200 let TSFlags = ps.TSFlags; 201 202 bits<12> offset; 203 bits<1> glc; 204 bits<1> lds = 0; 205 bits<8> vaddr; 206 bits<8> vdata; 207 bits<7> srsrc; 208 bits<1> slc; 209 bits<1> tfe; 210 bits<8> soffset; 211} 212 213 214// For cache invalidation instructions. 215class MUBUF_Invalidate <string opName, SDPatternOperator node> : 216 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 217 218 let AsmMatchConverter = ""; 219 220 let hasSideEffects = 1; 221 let mayStore = 1; 222 223 // Set everything to 0. 224 let offen = 0; 225 let idxen = 0; 226 let addr64 = 0; 227 let has_vdata = 0; 228 let has_vaddr = 0; 229 let has_glc = 0; 230 let glc_value = 0; 231 let has_srsrc = 0; 232 let has_soffset = 0; 233 let has_offset = 0; 234 let has_slc = 0; 235 let has_tfe = 0; 236} 237 238class getMUBUFInsDA<list<RegisterClass> vdataList, 239 list<RegisterClass> vaddrList=[]> { 240 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 241 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 242 dag InsNoData = !if(!empty(vaddrList), 243 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 244 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 245 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 246 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 247 ); 248 dag InsData = !if(!empty(vaddrList), 249 (ins vdataClass:$vdata, SReg_128:$srsrc, 250 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe), 251 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 252 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe) 253 ); 254 dag ret = !if(!empty(vdataList), InsNoData, InsData); 255} 256 257class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 258 dag ret = 259 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret, 260 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 261 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret, 262 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 263 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret, 264 (ins)))))); 265} 266 267class getMUBUFAsmOps<int addrKind> { 268 string Pfx = 269 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 270 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 271 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 272 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 273 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 274 ""))))); 275 string ret = Pfx # "$offset"; 276} 277 278class MUBUF_SetupAddr<int addrKind> { 279 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 280 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 281 282 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 283 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 284 285 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 286 287 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 288} 289 290class MUBUF_Load_Pseudo <string opName, 291 int addrKind, 292 RegisterClass vdataClass, 293 list<dag> pattern=[], 294 // Workaround bug bz30254 295 int addrKindCopy = addrKind> 296 : MUBUF_Pseudo<opName, 297 (outs vdataClass:$vdata), 298 getMUBUFIns<addrKindCopy>.ret, 299 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 300 pattern>, 301 MUBUF_SetupAddr<addrKindCopy> { 302 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 303 let mayLoad = 1; 304 let mayStore = 0; 305} 306 307// FIXME: tfe can't be an operand because it requires a separate 308// opcode because it needs an N+1 register class dest register. 309multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 310 ValueType load_vt = i32, 311 SDPatternOperator ld = null_frag> { 312 313 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 314 [(set load_vt:$vdata, 315 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 316 MUBUFAddr64Table<0>; 317 318 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 319 [(set load_vt:$vdata, 320 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>, 321 MUBUFAddr64Table<1>; 322 323 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 324 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 325 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 326 327 let DisableWQM = 1 in { 328 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 329 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 330 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 331 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 332 } 333} 334 335class MUBUF_Store_Pseudo <string opName, 336 int addrKind, 337 RegisterClass vdataClass, 338 list<dag> pattern=[], 339 // Workaround bug bz30254 340 int addrKindCopy = addrKind, 341 RegisterClass vdataClassCopy = vdataClass> 342 : MUBUF_Pseudo<opName, 343 (outs), 344 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 345 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 346 pattern>, 347 MUBUF_SetupAddr<addrKindCopy> { 348 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 349 let mayLoad = 0; 350 let mayStore = 1; 351} 352 353multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 354 ValueType store_vt = i32, 355 SDPatternOperator st = null_frag> { 356 357 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 358 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 359 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 360 MUBUFAddr64Table<0>; 361 362 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 363 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 364 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 365 MUBUFAddr64Table<1>; 366 367 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 368 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 369 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 370 371 let DisableWQM = 1 in { 372 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 373 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 374 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 375 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 376 } 377} 378 379 380class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 381 list<RegisterClass> vaddrList=[]> { 382 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 383 dag ret = !if(vdata_in, 384 !if(!empty(vaddrList), 385 (ins vdataClass:$vdata_in, 386 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 387 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 388 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 389 ), 390 !if(!empty(vaddrList), 391 (ins vdataClass:$vdata, 392 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc), 393 (ins vdataClass:$vdata, vaddrClass:$vaddr, 394 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc) 395 )); 396} 397 398class getMUBUFAtomicIns<int addrKind, 399 RegisterClass vdataClass, 400 bit vdata_in, 401 // Workaround bug bz30254 402 RegisterClass vdataClassCopy=vdataClass> { 403 dag ret = 404 !if(!eq(addrKind, BUFAddrKind.Offset), 405 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 406 !if(!eq(addrKind, BUFAddrKind.OffEn), 407 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 408 !if(!eq(addrKind, BUFAddrKind.IdxEn), 409 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 410 !if(!eq(addrKind, BUFAddrKind.BothEn), 411 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 412 !if(!eq(addrKind, BUFAddrKind.Addr64), 413 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 414 (ins)))))); 415} 416 417class MUBUF_Atomic_Pseudo<string opName, 418 int addrKind, 419 dag outs, 420 dag ins, 421 string asmOps, 422 list<dag> pattern=[], 423 // Workaround bug bz30254 424 int addrKindCopy = addrKind> 425 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 426 MUBUF_SetupAddr<addrKindCopy> { 427 let mayStore = 1; 428 let mayLoad = 1; 429 let hasPostISelHook = 1; 430 let hasSideEffects = 1; 431 let DisableWQM = 1; 432 let has_glc = 0; 433 let has_tfe = 0; 434} 435 436class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 437 RegisterClass vdataClass, 438 list<dag> pattern=[], 439 // Workaround bug bz30254 440 int addrKindCopy = addrKind, 441 RegisterClass vdataClassCopy = vdataClass> 442 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 443 (outs), 444 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 445 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 446 pattern>, 447 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 448 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 449 let glc_value = 0; 450 let AsmMatchConverter = "cvtMubufAtomic"; 451} 452 453class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 454 RegisterClass vdataClass, 455 list<dag> pattern=[], 456 // Workaround bug bz30254 457 int addrKindCopy = addrKind, 458 RegisterClass vdataClassCopy = vdataClass> 459 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 460 (outs vdataClassCopy:$vdata), 461 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 462 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 463 pattern>, 464 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 465 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 466 let glc_value = 1; 467 let Constraints = "$vdata = $vdata_in"; 468 let DisableEncoding = "$vdata_in"; 469 let AsmMatchConverter = "cvtMubufAtomicReturn"; 470} 471 472multiclass MUBUF_Pseudo_Atomics <string opName, 473 RegisterClass vdataClass, 474 ValueType vdataType, 475 SDPatternOperator atomic> { 476 477 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 478 MUBUFAddr64Table <0>; 479 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 480 MUBUFAddr64Table <1>; 481 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 482 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 483 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 484 485 def _RTN_OFFSET : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 486 [(set vdataType:$vdata, 487 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 488 vdataType:$vdata_in))]>, 489 MUBUFAddr64Table <0, "_RTN">; 490 491 def _RTN_ADDR64 : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 492 [(set vdataType:$vdata, 493 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 494 vdataType:$vdata_in))]>, 495 MUBUFAddr64Table <1, "_RTN">; 496 497 def _RTN_OFFEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 498 def _RTN_IDXEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 499 def _RTN_BOTHEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 500} 501 502 503//===----------------------------------------------------------------------===// 504// MUBUF Instructions 505//===----------------------------------------------------------------------===// 506 507let SubtargetPredicate = isGCN in { 508 509defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads < 510 "buffer_load_format_x", VGPR_32 511>; 512defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 513 "buffer_load_format_xy", VReg_64 514>; 515defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 516 "buffer_load_format_xyz", VReg_96 517>; 518defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 519 "buffer_load_format_xyzw", VReg_128 520>; 521defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 522 "buffer_store_format_x", VGPR_32 523>; 524defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 525 "buffer_store_format_xy", VReg_64 526>; 527defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 528 "buffer_store_format_xyz", VReg_96 529>; 530defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 531 "buffer_store_format_xyzw", VReg_128 532>; 533defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads < 534 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 535>; 536defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads < 537 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 538>; 539defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads < 540 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 541>; 542defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads < 543 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 544>; 545defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads < 546 "buffer_load_dword", VGPR_32, i32, mubuf_load 547>; 548defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 549 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 550>; 551defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 552 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 553>; 554defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 555 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 556>; 557defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 558 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 559>; 560defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 561 "buffer_store_short", VGPR_32, i32, truncstorei16_global 562>; 563defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 564 "buffer_store_dword", VGPR_32, i32, global_store 565>; 566defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 567 "buffer_store_dwordx2", VReg_64, v2i32, global_store 568>; 569defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 570 "buffer_store_dwordx3", VReg_96, untyped, global_store 571>; 572defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 573 "buffer_store_dwordx4", VReg_128, v4i32, global_store 574>; 575defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 576 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 577>; 578defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 579 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 580>; 581defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 582 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 583>; 584defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 585 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 586>; 587defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 588 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 589>; 590defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 591 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 592>; 593defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 594 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 595>; 596defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 597 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 598>; 599defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 600 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 601>; 602defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 603 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 604>; 605defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 606 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 607>; 608defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 609 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 610>; 611defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 612 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 613>; 614defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 615 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 616>; 617defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 618 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 619>; 620defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 621 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 622>; 623defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 624 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 625>; 626defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 627 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 628>; 629defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 630 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 631>; 632defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 633 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 634>; 635defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 636 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 637>; 638defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 639 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 640>; 641defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 642 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 643>; 644defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 645 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 646>; 647defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 648 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 649>; 650defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 651 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 652>; 653 654let SubtargetPredicate = isSI in { // isn't on CI & VI 655/* 656defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 657defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 658defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 659defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 660defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 661defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 662defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 663defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 664*/ 665 666def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 667 int_amdgcn_buffer_wbinvl1_sc>; 668} 669 670def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 671 int_amdgcn_buffer_wbinvl1>; 672 673//===----------------------------------------------------------------------===// 674// MTBUF Instructions 675//===----------------------------------------------------------------------===// 676 677//def TBUFFER_LOAD_FORMAT_X : MTBUF_ <0, "tbuffer_load_format_x", []>; 678//def TBUFFER_LOAD_FORMAT_XY : MTBUF_ <1, "tbuffer_load_format_xy", []>; 679//def TBUFFER_LOAD_FORMAT_XYZ : MTBUF_ <2, "tbuffer_load_format_xyz", []>; 680def TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Load_Pseudo <"tbuffer_load_format_xyzw", VReg_128>; 681def TBUFFER_STORE_FORMAT_X : MTBUF_Store_Pseudo <"tbuffer_store_format_x", VGPR_32>; 682def TBUFFER_STORE_FORMAT_XY : MTBUF_Store_Pseudo <"tbuffer_store_format_xy", VReg_64>; 683def TBUFFER_STORE_FORMAT_XYZ : MTBUF_Store_Pseudo <"tbuffer_store_format_xyz", VReg_128>; 684def TBUFFER_STORE_FORMAT_XYZW : MTBUF_Store_Pseudo <"tbuffer_store_format_xyzw", VReg_128>; 685 686} // End let SubtargetPredicate = isGCN 687 688let SubtargetPredicate = isCIVI in { 689 690//===----------------------------------------------------------------------===// 691// Instruction definitions for CI and newer. 692//===----------------------------------------------------------------------===// 693// Remaining instructions: 694// BUFFER_LOAD_DWORDX3 695// BUFFER_STORE_DWORDX3 696 697def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 698 int_amdgcn_buffer_wbinvl1_vol>; 699 700} // End let SubtargetPredicate = isCIVI 701 702//===----------------------------------------------------------------------===// 703// MUBUF Patterns 704//===----------------------------------------------------------------------===// 705 706let Predicates = [isGCN] in { 707 708// int_SI_vs_load_input 709def : Pat< 710 (SIload_input v4i32:$tlst, imm:$attr_offset, i32:$buf_idx_vgpr), 711 (BUFFER_LOAD_FORMAT_XYZW_IDXEN $buf_idx_vgpr, $tlst, (i32 0), imm:$attr_offset, 0, 0, 0) 712>; 713 714// Offset in an 32-bit VGPR 715def : Pat < 716 (SIload_constant v4i32:$sbase, i32:$voff), 717 (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0) 718>; 719 720 721//===----------------------------------------------------------------------===// 722// buffer_load/store_format patterns 723//===----------------------------------------------------------------------===// 724 725multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 726 string opcode> { 727 def : Pat< 728 (vt (name v4i32:$rsrc, 0, 729 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 730 imm:$glc, imm:$slc)), 731 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 732 (as_i1imm $glc), (as_i1imm $slc), 0) 733 >; 734 735 def : Pat< 736 (vt (name v4i32:$rsrc, i32:$vindex, 737 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 738 imm:$glc, imm:$slc)), 739 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 740 (as_i1imm $glc), (as_i1imm $slc), 0) 741 >; 742 743 def : Pat< 744 (vt (name v4i32:$rsrc, 0, 745 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 746 imm:$glc, imm:$slc)), 747 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 748 (as_i1imm $glc), (as_i1imm $slc), 0) 749 >; 750 751 def : Pat< 752 (vt (name v4i32:$rsrc, i32:$vindex, 753 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 754 imm:$glc, imm:$slc)), 755 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 756 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 757 $rsrc, $soffset, (as_i16imm $offset), 758 (as_i1imm $glc), (as_i1imm $slc), 0) 759 >; 760} 761 762defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 763defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 764defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 765defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 766defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 767defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 768 769multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 770 string opcode> { 771 def : Pat< 772 (name vt:$vdata, v4i32:$rsrc, 0, 773 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 774 imm:$glc, imm:$slc), 775 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 776 (as_i1imm $glc), (as_i1imm $slc), 0) 777 >; 778 779 def : Pat< 780 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 781 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 782 imm:$glc, imm:$slc), 783 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 784 (as_i16imm $offset), (as_i1imm $glc), 785 (as_i1imm $slc), 0) 786 >; 787 788 def : Pat< 789 (name vt:$vdata, v4i32:$rsrc, 0, 790 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 791 imm:$glc, imm:$slc), 792 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 793 (as_i16imm $offset), (as_i1imm $glc), 794 (as_i1imm $slc), 0) 795 >; 796 797 def : Pat< 798 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 799 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 800 imm:$glc, imm:$slc), 801 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 802 $vdata, 803 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 804 $rsrc, $soffset, (as_i16imm $offset), 805 (as_i1imm $glc), (as_i1imm $slc), 0) 806 >; 807} 808 809defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 810defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 811defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 812defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">; 813defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 814defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 815 816//===----------------------------------------------------------------------===// 817// buffer_atomic patterns 818//===----------------------------------------------------------------------===// 819 820multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 821 def : Pat< 822 (name i32:$vdata_in, v4i32:$rsrc, 0, 823 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 824 imm:$slc), 825 (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFSET) $vdata_in, $rsrc, $soffset, 826 (as_i16imm $offset), (as_i1imm $slc)) 827 >; 828 829 def : Pat< 830 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 831 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 832 imm:$slc), 833 (!cast<MUBUF_Pseudo>(opcode # _RTN_IDXEN) $vdata_in, $vindex, $rsrc, $soffset, 834 (as_i16imm $offset), (as_i1imm $slc)) 835 >; 836 837 def : Pat< 838 (name i32:$vdata_in, v4i32:$rsrc, 0, 839 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 840 imm:$slc), 841 (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFEN) $vdata_in, $voffset, $rsrc, $soffset, 842 (as_i16imm $offset), (as_i1imm $slc)) 843 >; 844 845 def : Pat< 846 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 847 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 848 imm:$slc), 849 (!cast<MUBUF_Pseudo>(opcode # _RTN_BOTHEN) 850 $vdata_in, 851 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 852 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)) 853 >; 854} 855 856defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 857defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">; 858defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 859defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 860defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 861defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 862defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 863defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">; 864defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">; 865defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 866 867def : Pat< 868 (int_amdgcn_buffer_atomic_cmpswap 869 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 870 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 871 imm:$slc), 872 (EXTRACT_SUBREG 873 (BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET 874 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 875 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 876 sub0) 877>; 878 879def : Pat< 880 (int_amdgcn_buffer_atomic_cmpswap 881 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 882 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 883 imm:$slc), 884 (EXTRACT_SUBREG 885 (BUFFER_ATOMIC_CMPSWAP_RTN_IDXEN 886 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 887 $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 888 sub0) 889>; 890 891def : Pat< 892 (int_amdgcn_buffer_atomic_cmpswap 893 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 894 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 895 imm:$slc), 896 (EXTRACT_SUBREG 897 (BUFFER_ATOMIC_CMPSWAP_RTN_OFFEN 898 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 899 $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 900 sub0) 901>; 902 903def : Pat< 904 (int_amdgcn_buffer_atomic_cmpswap 905 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 906 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 907 imm:$slc), 908 (EXTRACT_SUBREG 909 (BUFFER_ATOMIC_CMPSWAP_RTN_BOTHEN 910 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 911 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 912 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 913 sub0) 914>; 915 916 917class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 918 PatFrag constant_ld> : Pat < 919 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 920 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 921 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 922 >; 923 924multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 925 ValueType vt, PatFrag atomic_ld> { 926 def : Pat < 927 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 928 i16:$offset, i1:$slc))), 929 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0) 930 >; 931 932 def : Pat < 933 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 934 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0) 935 >; 936} 937 938let Predicates = [isSICI] in { 939def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 940def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 941def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 942def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 943 944defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 945defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 946} // End Predicates = [isSICI] 947 948multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 949 PatFrag ld> { 950 951 def : Pat < 952 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 953 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 954 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 955 >; 956} 957 958let Predicates = [Has16BitInsts] in { 959 960defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 961defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 962defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 963defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 964 965} // End Predicates = [Has16BitInsts] 966 967class MUBUFScratchLoadPat <MUBUF_Pseudo Instr, ValueType vt, PatFrag ld> : Pat < 968 (vt (ld (MUBUFScratch v4i32:$srsrc, i32:$vaddr, 969 i32:$soffset, u16imm:$offset))), 970 (Instr $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 971>; 972 973def : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, i32, sextloadi8_private>; 974def : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, i32, extloadi8_private>; 975def : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, i16, sextloadi8_private>; 976def : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, i16, extloadi8_private>; 977def : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, i32, sextloadi16_private>; 978def : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, i32, extloadi16_private>; 979def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, i32, load_private>; 980def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, v2i32, load_private>; 981def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, v4i32, load_private>; 982 983// BUFFER_LOAD_DWORD*, addr64=0 984multiclass MUBUF_Load_Dword <ValueType vt, 985 MUBUF_Pseudo offset, 986 MUBUF_Pseudo offen, 987 MUBUF_Pseudo idxen, 988 MUBUF_Pseudo bothen> { 989 990 def : Pat < 991 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset, 992 imm:$offset, 0, 0, imm:$glc, imm:$slc, 993 imm:$tfe)), 994 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 995 (as_i1imm $slc), (as_i1imm $tfe)) 996 >; 997 998 def : Pat < 999 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1000 imm:$offset, 1, 0, imm:$glc, imm:$slc, 1001 imm:$tfe)), 1002 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1003 (as_i1imm $tfe)) 1004 >; 1005 1006 def : Pat < 1007 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1008 imm:$offset, 0, 1, imm:$glc, imm:$slc, 1009 imm:$tfe)), 1010 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1011 (as_i1imm $slc), (as_i1imm $tfe)) 1012 >; 1013 1014 def : Pat < 1015 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset, 1016 imm:$offset, 1, 1, imm:$glc, imm:$slc, 1017 imm:$tfe)), 1018 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1019 (as_i1imm $tfe)) 1020 >; 1021} 1022 1023defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN, 1024 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>; 1025defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN, 1026 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>; 1027defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN, 1028 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>; 1029 1030multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1031 ValueType vt, PatFrag atomic_st> { 1032 // Store follows atomic op convention so address is forst 1033 def : Pat < 1034 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1035 i16:$offset, i1:$slc), vt:$val), 1036 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0) 1037 >; 1038 1039 def : Pat < 1040 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1041 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0) 1042 >; 1043} 1044let Predicates = [isSICI] in { 1045defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>; 1046defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>; 1047} // End Predicates = [isSICI] 1048 1049 1050multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1051 PatFrag st> { 1052 1053 def : Pat < 1054 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1055 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1056 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1057 >; 1058} 1059 1060defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1061defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>; 1062 1063class MUBUFScratchStorePat <MUBUF_Pseudo Instr, ValueType vt, PatFrag st> : Pat < 1064 (st vt:$value, (MUBUFScratch v4i32:$srsrc, i32:$vaddr, i32:$soffset, 1065 u16imm:$offset)), 1066 (Instr $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1067>; 1068 1069def : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, i32, truncstorei8_private>; 1070def : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, i32, truncstorei16_private>; 1071def : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, i16, truncstorei8_private>; 1072def : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, i16, store_private>; 1073def : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, i32, store_private>; 1074def : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, v2i32, store_private>; 1075def : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, v4i32, store_private>; 1076 1077//===----------------------------------------------------------------------===// 1078// MTBUF Patterns 1079//===----------------------------------------------------------------------===// 1080 1081// TBUFFER_STORE_FORMAT_*, addr64=0 1082class MTBUF_StoreResource <ValueType vt, int num_channels, MTBUF_Pseudo opcode> : Pat< 1083 (SItbuffer_store v4i32:$rsrc, vt:$vdata, num_channels, i32:$vaddr, 1084 i32:$soffset, imm:$inst_offset, imm:$dfmt, 1085 imm:$nfmt, imm:$offen, imm:$idxen, 1086 imm:$glc, imm:$slc, imm:$tfe), 1087 (opcode 1088 $vdata, (as_i16imm $inst_offset), (as_i1imm $offen), (as_i1imm $idxen), 1089 (as_i1imm $glc), 0, (as_i8imm $dfmt), (as_i8imm $nfmt), $vaddr, $rsrc, 1090 (as_i1imm $slc), (as_i1imm $tfe), $soffset) 1091>; 1092 1093def : MTBUF_StoreResource <i32, 1, TBUFFER_STORE_FORMAT_X>; 1094def : MTBUF_StoreResource <v2i32, 2, TBUFFER_STORE_FORMAT_XY>; 1095def : MTBUF_StoreResource <v4i32, 3, TBUFFER_STORE_FORMAT_XYZ>; 1096def : MTBUF_StoreResource <v4i32, 4, TBUFFER_STORE_FORMAT_XYZW>; 1097 1098} // End let Predicates = [isGCN] 1099 1100//===----------------------------------------------------------------------===// 1101// Target instructions, move to the appropriate target TD file 1102//===----------------------------------------------------------------------===// 1103 1104//===----------------------------------------------------------------------===// 1105// SI 1106//===----------------------------------------------------------------------===// 1107 1108class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1109 MUBUF_Real<op, ps>, 1110 Enc64, 1111 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1112 let AssemblerPredicate=isSICI; 1113 let DecoderNamespace="SICI"; 1114 1115 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1116 let Inst{12} = ps.offen; 1117 let Inst{13} = ps.idxen; 1118 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1119 let Inst{15} = ps.addr64; 1120 let Inst{16} = lds; 1121 let Inst{24-18} = op; 1122 let Inst{31-26} = 0x38; //encoding 1123 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1124 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1125 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1126 let Inst{54} = !if(ps.has_slc, slc, ?); 1127 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1128 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1129} 1130 1131multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1132 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1133 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1134 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1135 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1136 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1137} 1138 1139multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1140 def _RTN_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>; 1141 def _RTN_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_ADDR64")>; 1142 def _RTN_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>; 1143 def _RTN_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>; 1144 def _RTN_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>; 1145} 1146 1147defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_si <0x00>; 1148defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1149defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1150defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1151defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1152defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1153defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1154defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1155defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_si <0x08>; 1156defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_si <0x09>; 1157defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_si <0x0a>; 1158defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_si <0x0b>; 1159defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_si <0x0c>; 1160defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1161defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1162defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1163defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1164defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1165defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1166defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1167defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1168defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1169 1170defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1171defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1172defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1173defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1174//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1175defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1176defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1177defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1178defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1179defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1180defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1181defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1182defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1183defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1184 1185//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1186//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1187//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1188defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1189defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1190defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1191defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1192//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1193defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1194defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1195defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1196defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1197defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1198defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1199defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1200defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1201defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1202// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1203//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1204//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1205//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1206 1207def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1208def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1209 1210class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1211 MTBUF_Real<ps>, 1212 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1213 let AssemblerPredicate=isSICI; 1214 let DecoderNamespace="SICI"; 1215 1216 bits<1> addr64; 1217 let Inst{15} = addr64; 1218 let Inst{18-16} = op; 1219} 1220 1221def TBUFFER_LOAD_FORMAT_XYZW_si : MTBUF_Real_si <3, TBUFFER_LOAD_FORMAT_XYZW>; 1222def TBUFFER_STORE_FORMAT_X_si : MTBUF_Real_si <4, TBUFFER_STORE_FORMAT_X>; 1223def TBUFFER_STORE_FORMAT_XY_si : MTBUF_Real_si <5, TBUFFER_STORE_FORMAT_XY>; 1224def TBUFFER_STORE_FORMAT_XYZ_si : MTBUF_Real_si <6, TBUFFER_STORE_FORMAT_XYZ>; 1225def TBUFFER_STORE_FORMAT_XYZW_si : MTBUF_Real_si <7, TBUFFER_STORE_FORMAT_XYZW>; 1226 1227 1228//===----------------------------------------------------------------------===// 1229// CI 1230//===----------------------------------------------------------------------===// 1231 1232class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1233 MUBUF_Real_si<op, ps> { 1234 let AssemblerPredicate=isCIOnly; 1235 let DecoderNamespace="CI"; 1236} 1237 1238def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1239 1240 1241//===----------------------------------------------------------------------===// 1242// VI 1243//===----------------------------------------------------------------------===// 1244 1245class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1246 MUBUF_Real<op, ps>, 1247 Enc64, 1248 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1249 let AssemblerPredicate=isVI; 1250 let DecoderNamespace="VI"; 1251 1252 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1253 let Inst{12} = ps.offen; 1254 let Inst{13} = ps.idxen; 1255 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1256 let Inst{16} = lds; 1257 let Inst{17} = !if(ps.has_slc, slc, ?); 1258 let Inst{24-18} = op; 1259 let Inst{31-26} = 0x38; //encoding 1260 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1261 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1262 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1263 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1264 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1265} 1266 1267multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1268 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1269 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1270 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1271 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1272} 1273 1274multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1275 MUBUF_Real_AllAddr_vi<op> { 1276 def _RTN_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>; 1277 def _RTN_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>; 1278 def _RTN_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>; 1279 def _RTN_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>; 1280} 1281 1282defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_vi <0x00>; 1283defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1284defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1285defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1286defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1287defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1288defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1289defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1290defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_vi <0x10>; 1291defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_vi <0x11>; 1292defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_vi <0x12>; 1293defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_vi <0x13>; 1294defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_vi <0x14>; 1295defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 1296defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 1297defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 1298defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1299defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1300defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1301defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1302defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1303defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1304 1305defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1306defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1307defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1308defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1309defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1310defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1311defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1312defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1313defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1314defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1315defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1316defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1317defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1318 1319defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1320defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1321defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1322defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1323defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1324defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1325defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1326defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1327defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1328defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1329defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1330defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1331defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1332 1333def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 1334def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 1335 1336class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 1337 MTBUF_Real<ps>, 1338 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1339 let AssemblerPredicate=isVI; 1340 let DecoderNamespace="VI"; 1341 1342 let Inst{18-15} = op; 1343} 1344 1345def TBUFFER_LOAD_FORMAT_XYZW_vi : MTBUF_Real_vi <3, TBUFFER_LOAD_FORMAT_XYZW>; 1346def TBUFFER_STORE_FORMAT_X_vi : MTBUF_Real_vi <4, TBUFFER_STORE_FORMAT_X>; 1347def TBUFFER_STORE_FORMAT_XY_vi : MTBUF_Real_vi <5, TBUFFER_STORE_FORMAT_XY>; 1348def TBUFFER_STORE_FORMAT_XYZ_vi : MTBUF_Real_vi <6, TBUFFER_STORE_FORMAT_XYZ>; 1349def TBUFFER_STORE_FORMAT_XYZW_vi : MTBUF_Real_vi <7, TBUFFER_STORE_FORMAT_XYZW>; 1350 1351