1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 15def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 16 17def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 18def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 19def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 20 21class MubufLoad <SDPatternOperator op> : PatFrag < 22 (ops node:$ptr), (op node:$ptr), [{ 23 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 24 return AS == AMDGPUAS::GLOBAL_ADDRESS || 25 AS == AMDGPUAS::CONSTANT_ADDRESS; 26}]>; 27 28def mubuf_load : MubufLoad <load>; 29def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 30def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 31def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 32def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 33def mubuf_load_atomic : MubufLoad <atomic_load>; 34 35def BUFAddrKind { 36 int Offset = 0; 37 int OffEn = 1; 38 int IdxEn = 2; 39 int BothEn = 3; 40 int Addr64 = 4; 41} 42 43class getAddrName<int addrKind> { 44 string ret = 45 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 46 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 47 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 48 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 49 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 50 ""))))); 51} 52 53class MUBUFAddr64Table <bit is_addr64, string Name> { 54 bit IsAddr64 = is_addr64; 55 string OpName = Name; 56} 57 58class MUBUFLdsTable <bit is_lds, string Name> { 59 bit IsLds = is_lds; 60 string OpName = Name; 61} 62 63class MTBUFAddr64Table <bit is_addr64, string Name> { 64 bit IsAddr64 = is_addr64; 65 string OpName = Name; 66} 67 68//===----------------------------------------------------------------------===// 69// MTBUF classes 70//===----------------------------------------------------------------------===// 71 72class MTBUF_Pseudo <string opName, dag outs, dag ins, 73 string asmOps, list<dag> pattern=[]> : 74 InstSI<outs, ins, "", pattern>, 75 SIMCInstr<opName, SIEncodingFamily.NONE> { 76 77 let isPseudo = 1; 78 let isCodeGenOnly = 1; 79 let Size = 8; 80 let UseNamedOperandTable = 1; 81 82 string Mnemonic = opName; 83 string AsmOperands = asmOps; 84 85 let VM_CNT = 1; 86 let EXP_CNT = 1; 87 let MTBUF = 1; 88 let Uses = [EXEC]; 89 let hasSideEffects = 0; 90 let SchedRW = [WriteVMEM]; 91 92 let AsmMatchConverter = "cvtMtbuf"; 93 94 bits<1> offen = 0; 95 bits<1> idxen = 0; 96 bits<1> addr64 = 0; 97 bits<1> has_vdata = 1; 98 bits<1> has_vaddr = 1; 99 bits<1> has_glc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> has_srsrc = 1; 102 bits<1> has_soffset = 1; 103 bits<1> has_offset = 1; 104 bits<1> has_slc = 1; 105 bits<1> has_tfe = 1; 106} 107 108class MTBUF_Real <MTBUF_Pseudo ps> : 109 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 110 111 let isPseudo = 0; 112 let isCodeGenOnly = 0; 113 114 // copy relevant pseudo op flags 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 121 bits<12> offset; 122 bits<1> glc; 123 bits<7> format; 124 bits<8> vaddr; 125 bits<8> vdata; 126 bits<7> srsrc; 127 bits<1> slc; 128 bits<1> tfe; 129 bits<8> soffset; 130 131 bits<4> dfmt = format{3-0}; 132 bits<3> nfmt = format{6-4}; 133} 134 135class getMTBUFInsDA<list<RegisterClass> vdataList, 136 list<RegisterClass> vaddrList=[]> { 137 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 138 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 139 dag InsNoData = !if(!empty(vaddrList), 140 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 141 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe), 142 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 143 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe) 144 ); 145 dag InsData = !if(!empty(vaddrList), 146 (ins vdataClass:$vdata, SReg_128:$srsrc, 147 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 148 SLC:$slc, TFE:$tfe), 149 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 150 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 151 SLC:$slc, TFE:$tfe) 152 ); 153 dag ret = !if(!empty(vdataList), InsNoData, InsData); 154} 155 156class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 157 dag ret = 158 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 159 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 160 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 161 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 162 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 163 (ins)))))); 164} 165 166class getMTBUFAsmOps<int addrKind> { 167 string Pfx = 168 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 169 !if(!eq(addrKind, BUFAddrKind.OffEn), 170 "$vaddr, $srsrc, $format, $soffset offen", 171 !if(!eq(addrKind, BUFAddrKind.IdxEn), 172 "$vaddr, $srsrc, $format, $soffset idxen", 173 !if(!eq(addrKind, BUFAddrKind.BothEn), 174 "$vaddr, $srsrc, $format, $soffset idxen offen", 175 !if(!eq(addrKind, BUFAddrKind.Addr64), 176 "$vaddr, $srsrc, $format, $soffset addr64", 177 ""))))); 178 string ret = Pfx # "$offset"; 179} 180 181class MTBUF_SetupAddr<int addrKind> { 182 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 183 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 184 185 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 187 188 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 189 190 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 191} 192 193class MTBUF_Load_Pseudo <string opName, 194 int addrKind, 195 RegisterClass vdataClass, 196 list<dag> pattern=[], 197 // Workaround bug bz30254 198 int addrKindCopy = addrKind> 199 : MTBUF_Pseudo<opName, 200 (outs vdataClass:$vdata), 201 getMTBUFIns<addrKindCopy>.ret, 202 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 203 pattern>, 204 MTBUF_SetupAddr<addrKindCopy> { 205 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 206 let mayLoad = 1; 207 let mayStore = 0; 208} 209 210multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 211 ValueType load_vt = i32, 212 SDPatternOperator ld = null_frag> { 213 214 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 215 [(set load_vt:$vdata, 216 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 217 i1:$glc, i1:$slc, i1:$tfe)))]>, 218 MTBUFAddr64Table<0, NAME>; 219 220 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 221 [(set load_vt:$vdata, 222 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 223 i8:$format, i1:$glc, i1:$slc, i1:$tfe)))]>, 224 MTBUFAddr64Table<1, NAME>; 225 226 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 227 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 228 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 229 230 let DisableWQM = 1 in { 231 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 232 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 233 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 234 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 235 } 236} 237 238class MTBUF_Store_Pseudo <string opName, 239 int addrKind, 240 RegisterClass vdataClass, 241 list<dag> pattern=[], 242 // Workaround bug bz30254 243 int addrKindCopy = addrKind, 244 RegisterClass vdataClassCopy = vdataClass> 245 : MTBUF_Pseudo<opName, 246 (outs), 247 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 248 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 249 pattern>, 250 MTBUF_SetupAddr<addrKindCopy> { 251 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 252 let mayLoad = 0; 253 let mayStore = 1; 254} 255 256multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 257 ValueType store_vt = i32, 258 SDPatternOperator st = null_frag> { 259 260 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 261 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 262 i16:$offset, i8:$format, i1:$glc, 263 i1:$slc, i1:$tfe))]>, 264 MTBUFAddr64Table<0, NAME>; 265 266 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 267 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 268 i16:$offset, i8:$format, i1:$glc, 269 i1:$slc, i1:$tfe))]>, 270 MTBUFAddr64Table<1, NAME>; 271 272 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 273 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 274 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 275 276 let DisableWQM = 1 in { 277 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 278 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 279 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 280 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 281 } 282} 283 284 285//===----------------------------------------------------------------------===// 286// MUBUF classes 287//===----------------------------------------------------------------------===// 288 289class MUBUFGetBaseOpcode<string Op> { 290 string ret = !subst("DWORDX2", "DWORD", 291 !subst("DWORDX3", "DWORD", 292 !subst("DWORDX4", "DWORD", Op))); 293} 294 295class MUBUF_Pseudo <string opName, dag outs, dag ins, 296 string asmOps, list<dag> pattern=[]> : 297 InstSI<outs, ins, "", pattern>, 298 SIMCInstr<opName, SIEncodingFamily.NONE> { 299 300 let isPseudo = 1; 301 let isCodeGenOnly = 1; 302 let Size = 8; 303 let UseNamedOperandTable = 1; 304 305 string Mnemonic = opName; 306 string AsmOperands = asmOps; 307 308 Instruction Opcode = !cast<Instruction>(NAME); 309 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 310 311 let VM_CNT = 1; 312 let EXP_CNT = 1; 313 let MUBUF = 1; 314 let Uses = [EXEC]; 315 let hasSideEffects = 0; 316 let SchedRW = [WriteVMEM]; 317 318 let AsmMatchConverter = "cvtMubuf"; 319 320 bits<1> offen = 0; 321 bits<1> idxen = 0; 322 bits<1> addr64 = 0; 323 bits<1> lds = 0; 324 bits<1> has_vdata = 1; 325 bits<1> has_vaddr = 1; 326 bits<1> has_glc = 1; 327 bits<1> glc_value = 0; // the value for glc if no such operand 328 bits<1> has_srsrc = 1; 329 bits<1> has_soffset = 1; 330 bits<1> has_offset = 1; 331 bits<1> has_slc = 1; 332 bits<1> has_tfe = 1; 333 bits<4> dwords = 0; 334} 335 336class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 337 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 338 339 let isPseudo = 0; 340 let isCodeGenOnly = 0; 341 342 // copy relevant pseudo op flags 343 let SubtargetPredicate = ps.SubtargetPredicate; 344 let AsmMatchConverter = ps.AsmMatchConverter; 345 let Constraints = ps.Constraints; 346 let DisableEncoding = ps.DisableEncoding; 347 let TSFlags = ps.TSFlags; 348 349 bits<12> offset; 350 bits<1> glc; 351 bits<8> vaddr; 352 bits<8> vdata; 353 bits<7> srsrc; 354 bits<1> slc; 355 bits<1> tfe; 356 bits<8> soffset; 357} 358 359 360// For cache invalidation instructions. 361class MUBUF_Invalidate <string opName, SDPatternOperator node> : 362 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 363 364 let AsmMatchConverter = ""; 365 366 let hasSideEffects = 1; 367 let mayStore = 1; 368 369 // Set everything to 0. 370 let offen = 0; 371 let idxen = 0; 372 let addr64 = 0; 373 let has_vdata = 0; 374 let has_vaddr = 0; 375 let has_glc = 0; 376 let glc_value = 0; 377 let has_srsrc = 0; 378 let has_soffset = 0; 379 let has_offset = 0; 380 let has_slc = 0; 381 let has_tfe = 0; 382} 383 384class getMUBUFInsDA<list<RegisterClass> vdataList, 385 list<RegisterClass> vaddrList=[], 386 bit isLds = 0> { 387 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 388 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 389 dag InsNoData = !if(!empty(vaddrList), 390 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 391 offset:$offset, GLC:$glc, SLC:$slc), 392 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 393 offset:$offset, GLC:$glc, SLC:$slc) 394 ); 395 dag InsData = !if(!empty(vaddrList), 396 (ins vdataClass:$vdata, SReg_128:$srsrc, 397 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 398 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 399 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 400 ); 401 dag ret = !con( 402 !if(!empty(vdataList), InsNoData, InsData), 403 !if(isLds, (ins), (ins TFE:$tfe)) 404 ); 405} 406 407class getMUBUFDwords<RegisterClass regClass> { 408 string regClassAsInt = !cast<string>(regClass); 409 int ret = 410 !if(!eq(regClassAsInt, !cast<string>(VGPR_32)), 1, 411 !if(!eq(regClassAsInt, !cast<string>(VReg_64)), 2, 412 !if(!eq(regClassAsInt, !cast<string>(VReg_96)), 3, 413 !if(!eq(regClassAsInt, !cast<string>(VReg_128)), 4, 414 0)))); 415} 416 417class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 418 dag ret = 419 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 420 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 421 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 422 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 423 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 424 (ins)))))); 425} 426 427class getMUBUFAsmOps<int addrKind> { 428 string Pfx = 429 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 430 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 431 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 432 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 433 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 434 ""))))); 435 string ret = Pfx # "$offset"; 436} 437 438class MUBUF_SetupAddr<int addrKind> { 439 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 440 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 441 442 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 443 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 444 445 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 446 447 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 448} 449 450class MUBUF_Load_Pseudo <string opName, 451 int addrKind, 452 RegisterClass vdataClass, 453 bit HasTiedDest = 0, 454 bit isLds = 0, 455 list<dag> pattern=[], 456 // Workaround bug bz30254 457 int addrKindCopy = addrKind> 458 : MUBUF_Pseudo<opName, 459 (outs vdataClass:$vdata), 460 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 461 !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 462 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 463 !if(isLds, " lds", "$tfe"), 464 pattern>, 465 MUBUF_SetupAddr<addrKindCopy> { 466 let PseudoInstr = opName # !if(isLds, "_lds", "") # 467 "_" # getAddrName<addrKindCopy>.ret; 468 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 469 470 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 471 let mayLoad = 1; 472 let mayStore = 0; 473 let maybeAtomic = 1; 474 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 475 let has_tfe = !if(isLds, 0, 1); 476 let lds = isLds; 477 let dwords = getMUBUFDwords<vdataClass>.ret; 478} 479 480// FIXME: tfe can't be an operand because it requires a separate 481// opcode because it needs an N+1 register class dest register. 482multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 483 ValueType load_vt = i32, 484 SDPatternOperator ld = null_frag, 485 bit TiedDest = 0, 486 bit isLds = 0> { 487 488 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 489 TiedDest, isLds, 490 !if(isLds, 491 [], 492 [(set load_vt:$vdata, 493 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 494 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 495 496 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 497 TiedDest, isLds, 498 !if(isLds, 499 [], 500 [(set load_vt:$vdata, 501 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 502 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 503 504 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 505 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 506 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 507 508 let DisableWQM = 1 in { 509 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>; 510 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 511 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 512 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 513 } 514} 515 516multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass, 517 ValueType load_vt = i32, 518 SDPatternOperator ld_nolds = null_frag, 519 SDPatternOperator ld_lds = null_frag> { 520 defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>; 521 defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>; 522} 523 524class MUBUF_Store_Pseudo <string opName, 525 int addrKind, 526 RegisterClass vdataClass, 527 list<dag> pattern=[], 528 // Workaround bug bz30254 529 int addrKindCopy = addrKind, 530 RegisterClass vdataClassCopy = vdataClass> 531 : MUBUF_Pseudo<opName, 532 (outs), 533 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 534 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 535 pattern>, 536 MUBUF_SetupAddr<addrKindCopy> { 537 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 538 let mayLoad = 0; 539 let mayStore = 1; 540 let maybeAtomic = 1; 541 let dwords = getMUBUFDwords<vdataClass>.ret; 542} 543 544multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 545 ValueType store_vt = i32, 546 SDPatternOperator st = null_frag> { 547 548 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 549 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 550 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 551 MUBUFAddr64Table<0, NAME>; 552 553 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 554 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 555 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 556 MUBUFAddr64Table<1, NAME>; 557 558 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 559 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 560 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 561 562 let DisableWQM = 1 in { 563 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 564 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 565 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 566 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 567 } 568} 569 570class MUBUF_Pseudo_Store_Lds<string opName> 571 : MUBUF_Pseudo<opName, 572 (outs), 573 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 574 " $srsrc, $soffset$offset lds$glc$slc"> { 575 let mayLoad = 0; 576 let mayStore = 1; 577 let maybeAtomic = 1; 578 579 let has_vdata = 0; 580 let has_vaddr = 0; 581 let has_tfe = 0; 582 let lds = 1; 583 584 let Uses = [EXEC, M0]; 585 let AsmMatchConverter = "cvtMubufLds"; 586} 587 588class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 589 list<RegisterClass> vaddrList=[]> { 590 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 591 dag ret = !if(vdata_in, 592 !if(!empty(vaddrList), 593 (ins vdataClass:$vdata_in, 594 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 595 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 596 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 597 ), 598 !if(!empty(vaddrList), 599 (ins vdataClass:$vdata, 600 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 601 (ins vdataClass:$vdata, vaddrClass:$vaddr, 602 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 603 )); 604} 605 606class getMUBUFAtomicIns<int addrKind, 607 RegisterClass vdataClass, 608 bit vdata_in, 609 // Workaround bug bz30254 610 RegisterClass vdataClassCopy=vdataClass> { 611 dag ret = 612 !if(!eq(addrKind, BUFAddrKind.Offset), 613 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 614 !if(!eq(addrKind, BUFAddrKind.OffEn), 615 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 616 !if(!eq(addrKind, BUFAddrKind.IdxEn), 617 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 618 !if(!eq(addrKind, BUFAddrKind.BothEn), 619 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 620 !if(!eq(addrKind, BUFAddrKind.Addr64), 621 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 622 (ins)))))); 623} 624 625class MUBUF_Atomic_Pseudo<string opName, 626 int addrKind, 627 dag outs, 628 dag ins, 629 string asmOps, 630 list<dag> pattern=[], 631 // Workaround bug bz30254 632 int addrKindCopy = addrKind> 633 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 634 MUBUF_SetupAddr<addrKindCopy> { 635 let mayStore = 1; 636 let mayLoad = 1; 637 let hasPostISelHook = 1; 638 let hasSideEffects = 1; 639 let DisableWQM = 1; 640 let has_glc = 0; 641 let has_tfe = 0; 642 let maybeAtomic = 1; 643} 644 645class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 646 RegisterClass vdataClass, 647 list<dag> pattern=[], 648 // Workaround bug bz30254 649 int addrKindCopy = addrKind, 650 RegisterClass vdataClassCopy = vdataClass> 651 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 652 (outs), 653 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 654 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 655 pattern>, 656 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 657 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 658 let glc_value = 0; 659 let AsmMatchConverter = "cvtMubufAtomic"; 660} 661 662class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 663 RegisterClass vdataClass, 664 list<dag> pattern=[], 665 // Workaround bug bz30254 666 int addrKindCopy = addrKind, 667 RegisterClass vdataClassCopy = vdataClass> 668 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 669 (outs vdataClassCopy:$vdata), 670 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 671 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 672 pattern>, 673 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 674 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 675 let glc_value = 1; 676 let Constraints = "$vdata = $vdata_in"; 677 let DisableEncoding = "$vdata_in"; 678 let AsmMatchConverter = "cvtMubufAtomicReturn"; 679} 680 681multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 682 RegisterClass vdataClass, 683 ValueType vdataType, 684 SDPatternOperator atomic> { 685 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 686 MUBUFAddr64Table <0, NAME>; 687 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 688 MUBUFAddr64Table <1, NAME>; 689 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 690 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 691 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 692} 693 694multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 695 RegisterClass vdataClass, 696 ValueType vdataType, 697 SDPatternOperator atomic> { 698 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 699 [(set vdataType:$vdata, 700 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 701 vdataType:$vdata_in))]>, 702 MUBUFAddr64Table <0, NAME # "_RTN">; 703 704 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 705 [(set vdataType:$vdata, 706 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 707 vdataType:$vdata_in))]>, 708 MUBUFAddr64Table <1, NAME # "_RTN">; 709 710 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 711 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 712 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 713} 714 715multiclass MUBUF_Pseudo_Atomics <string opName, 716 RegisterClass vdataClass, 717 ValueType vdataType, 718 SDPatternOperator atomic> : 719 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 720 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 721 722 723//===----------------------------------------------------------------------===// 724// MUBUF Instructions 725//===----------------------------------------------------------------------===// 726 727defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 728 "buffer_load_format_x", VGPR_32 729>; 730defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 731 "buffer_load_format_xy", VReg_64 732>; 733defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 734 "buffer_load_format_xyz", VReg_96 735>; 736defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 737 "buffer_load_format_xyzw", VReg_128 738>; 739defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 740 "buffer_store_format_x", VGPR_32 741>; 742defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 743 "buffer_store_format_xy", VReg_64 744>; 745defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 746 "buffer_store_format_xyz", VReg_96 747>; 748defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 749 "buffer_store_format_xyzw", VReg_128 750>; 751 752let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 753 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 754 "buffer_load_format_d16_x", VGPR_32 755 >; 756 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 757 "buffer_load_format_d16_xy", VReg_64 758 >; 759 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 760 "buffer_load_format_d16_xyz", VReg_96 761 >; 762 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 763 "buffer_load_format_d16_xyzw", VReg_128 764 >; 765 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 766 "buffer_store_format_d16_x", VGPR_32 767 >; 768 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 769 "buffer_store_format_d16_xy", VReg_64 770 >; 771 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 772 "buffer_store_format_d16_xyz", VReg_96 773 >; 774 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 775 "buffer_store_format_d16_xyzw", VReg_128 776 >; 777} // End HasUnpackedD16VMem. 778 779let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 780 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 781 "buffer_load_format_d16_x", VGPR_32 782 >; 783 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 784 "buffer_load_format_d16_xy", VGPR_32 785 >; 786 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 787 "buffer_load_format_d16_xyz", VReg_64 788 >; 789 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 790 "buffer_load_format_d16_xyzw", VReg_64 791 >; 792 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 793 "buffer_store_format_d16_x", VGPR_32 794 >; 795 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 796 "buffer_store_format_d16_xy", VGPR_32 797 >; 798 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 799 "buffer_store_format_d16_xyz", VReg_64 800 >; 801 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 802 "buffer_store_format_d16_xyzw", VReg_64 803 >; 804} // End HasPackedD16VMem. 805 806defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 807 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 808>; 809defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 810 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 811>; 812defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 813 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 814>; 815defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 816 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 817>; 818defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 819 "buffer_load_dword", VGPR_32, i32, mubuf_load 820>; 821defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 822 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 823>; 824defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 825 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 826>; 827defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 828 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 829>; 830 831// This is not described in AMD documentation, 832// but 'lds' versions of these opcodes are available 833// in at least GFX8+ chips. See Bug 37653. 834let SubtargetPredicate = isVI in { 835defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 836 "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1 837>; 838defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 839 "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1 840>; 841defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 842 "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1 843>; 844} 845 846defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 847 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 848>; 849defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 850 "buffer_store_short", VGPR_32, i32, truncstorei16_global 851>; 852defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 853 "buffer_store_dword", VGPR_32, i32, store_global 854>; 855defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 856 "buffer_store_dwordx2", VReg_64, v2i32, store_global 857>; 858defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 859 "buffer_store_dwordx3", VReg_96, untyped, store_global 860>; 861defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 862 "buffer_store_dwordx4", VReg_128, v4i32, store_global 863>; 864defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 865 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 866>; 867defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 868 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 869>; 870defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 871 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 872>; 873defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 874 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 875>; 876defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 877 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 878>; 879defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 880 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 881>; 882defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 883 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 884>; 885defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 886 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 887>; 888defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 889 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 890>; 891defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 892 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 893>; 894defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 895 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 896>; 897defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 898 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 899>; 900defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 901 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 902>; 903defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 904 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 905>; 906defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 907 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 908>; 909defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 910 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 911>; 912defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 913 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 914>; 915defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 916 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 917>; 918defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 919 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 920>; 921defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 922 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 923>; 924defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 925 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 926>; 927defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 928 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 929>; 930defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 931 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 932>; 933defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 934 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 935>; 936defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 937 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 938>; 939defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 940 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 941>; 942 943let SubtargetPredicate = isVI in { 944def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 945} 946 947let SubtargetPredicate = isSI in { // isn't on CI & VI 948/* 949defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 950defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 951defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 952defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 953defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 954defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 955defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 956defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 957*/ 958 959def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 960 int_amdgcn_buffer_wbinvl1_sc>; 961} 962 963let SubtargetPredicate = HasD16LoadStore in { 964 965defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 966 "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 967>; 968 969defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 970 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 971>; 972 973defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 974 "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 975>; 976 977defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 978 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 979>; 980 981defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 982 "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 983>; 984 985defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 986 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 987>; 988 989defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 990 "buffer_store_byte_d16_hi", VGPR_32, i32 991>; 992 993defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 994 "buffer_store_short_d16_hi", VGPR_32, i32 995>; 996 997defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 998 "buffer_load_format_d16_hi_x", VGPR_32 999>; 1000defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1001 "buffer_store_format_d16_hi_x", VGPR_32 1002>; 1003 1004} // End HasD16LoadStore 1005 1006def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1007 int_amdgcn_buffer_wbinvl1>; 1008 1009//===----------------------------------------------------------------------===// 1010// MTBUF Instructions 1011//===----------------------------------------------------------------------===// 1012 1013defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 1014defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 1015defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>; 1016defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 1017defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 1018defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 1019defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>; 1020defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 1021 1022let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1023 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1024 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64>; 1025 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96>; 1026 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128>; 1027 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1028 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64>; 1029 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96>; 1030 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>; 1031} // End HasUnpackedD16VMem. 1032 1033let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1034 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1035 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32>; 1036 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64>; 1037 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64>; 1038 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1039 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32>; 1040 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64>; 1041 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>; 1042} // End HasPackedD16VMem. 1043 1044let SubtargetPredicate = isCIVI in { 1045 1046//===----------------------------------------------------------------------===// 1047// Instruction definitions for CI and newer. 1048//===----------------------------------------------------------------------===// 1049// Remaining instructions: 1050// BUFFER_LOAD_DWORDX3 1051// BUFFER_STORE_DWORDX3 1052 1053def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1054 int_amdgcn_buffer_wbinvl1_vol>; 1055 1056} // End let SubtargetPredicate = isCIVI 1057 1058//===----------------------------------------------------------------------===// 1059// MUBUF Patterns 1060//===----------------------------------------------------------------------===// 1061 1062def extract_glc : SDNodeXForm<imm, [{ 1063 return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8); 1064}]>; 1065 1066def extract_slc : SDNodeXForm<imm, [{ 1067 return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8); 1068}]>; 1069 1070//===----------------------------------------------------------------------===// 1071// buffer_load/store_format patterns 1072//===----------------------------------------------------------------------===// 1073 1074multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1075 string opcode> { 1076 def : GCNPat< 1077 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1078 imm:$cachepolicy, 0)), 1079 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1080 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1081 >; 1082 1083 def : GCNPat< 1084 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1085 imm:$cachepolicy, 0)), 1086 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1087 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1088 >; 1089 1090 def : GCNPat< 1091 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1092 imm:$cachepolicy, imm)), 1093 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1094 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1095 >; 1096 1097 def : GCNPat< 1098 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1099 imm:$cachepolicy, imm)), 1100 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1101 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1102 $rsrc, $soffset, (as_i16imm $offset), 1103 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1104 >; 1105} 1106 1107defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1108defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1109defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1110defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1111defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1112defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1113 1114let SubtargetPredicate = HasUnpackedD16VMem in { 1115 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1116 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1117 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1118 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1119} // End HasUnpackedD16VMem. 1120 1121let SubtargetPredicate = HasPackedD16VMem in { 1122 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1123 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1124 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1125 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1126 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1127 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1128} // End HasPackedD16VMem. 1129 1130defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1131defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1132defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1133defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1134defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1135defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1136 1137multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1138 string opcode> { 1139 def : GCNPat< 1140 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1141 imm:$cachepolicy, 0), 1142 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1143 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1144 >; 1145 1146 def : GCNPat< 1147 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1148 imm:$cachepolicy, 0), 1149 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1150 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1151 >; 1152 1153 def : GCNPat< 1154 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1155 imm:$cachepolicy, imm), 1156 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1157 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1158 >; 1159 1160 def : GCNPat< 1161 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1162 imm:$cachepolicy, imm), 1163 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1164 $vdata, 1165 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1166 $rsrc, $soffset, (as_i16imm $offset), 1167 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1168 >; 1169} 1170 1171defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1172defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1173defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1174defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1175defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1176defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1177 1178let SubtargetPredicate = HasUnpackedD16VMem in { 1179 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1180 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1181 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1182 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1183} // End HasUnpackedD16VMem. 1184 1185let SubtargetPredicate = HasPackedD16VMem in { 1186 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1187 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1188 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1189 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1190 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1191 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1192} // End HasPackedD16VMem. 1193 1194defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1195defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1196defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1197defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1198defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1199defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1200 1201//===----------------------------------------------------------------------===// 1202// buffer_atomic patterns 1203//===----------------------------------------------------------------------===// 1204 1205multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 1206 def : GCNPat< 1207 (name i32:$vdata_in, v4i32:$rsrc, 0, 1208 0, i32:$soffset, imm:$offset, 1209 imm:$cachepolicy, 0), 1210 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1211 (as_i16imm $offset), (extract_slc $cachepolicy)) 1212 >; 1213 1214 def : GCNPat< 1215 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1216 0, i32:$soffset, imm:$offset, 1217 imm:$cachepolicy, imm), 1218 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1219 (as_i16imm $offset), (extract_slc $cachepolicy)) 1220 >; 1221 1222 def : GCNPat< 1223 (name i32:$vdata_in, v4i32:$rsrc, 0, 1224 i32:$voffset, i32:$soffset, imm:$offset, 1225 imm:$cachepolicy, 0), 1226 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1227 (as_i16imm $offset), (extract_slc $cachepolicy)) 1228 >; 1229 1230 def : GCNPat< 1231 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1232 i32:$voffset, i32:$soffset, imm:$offset, 1233 imm:$cachepolicy, imm), 1234 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1235 $vdata_in, 1236 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1237 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1238 >; 1239} 1240 1241defm : BufferAtomicPatterns<SIbuffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 1242defm : BufferAtomicPatterns<SIbuffer_atomic_add, "BUFFER_ATOMIC_ADD">; 1243defm : BufferAtomicPatterns<SIbuffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 1244defm : BufferAtomicPatterns<SIbuffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 1245defm : BufferAtomicPatterns<SIbuffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 1246defm : BufferAtomicPatterns<SIbuffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 1247defm : BufferAtomicPatterns<SIbuffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 1248defm : BufferAtomicPatterns<SIbuffer_atomic_and, "BUFFER_ATOMIC_AND">; 1249defm : BufferAtomicPatterns<SIbuffer_atomic_or, "BUFFER_ATOMIC_OR">; 1250defm : BufferAtomicPatterns<SIbuffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 1251 1252def : GCNPat< 1253 (SIbuffer_atomic_cmpswap 1254 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1255 0, i32:$soffset, imm:$offset, 1256 imm:$cachepolicy, 0), 1257 (EXTRACT_SUBREG 1258 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1259 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1260 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1261 sub0) 1262>; 1263 1264def : GCNPat< 1265 (SIbuffer_atomic_cmpswap 1266 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1267 0, i32:$soffset, imm:$offset, 1268 imm:$cachepolicy, imm), 1269 (EXTRACT_SUBREG 1270 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1271 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1272 $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1273 sub0) 1274>; 1275 1276def : GCNPat< 1277 (SIbuffer_atomic_cmpswap 1278 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1279 i32:$voffset, i32:$soffset, imm:$offset, 1280 imm:$cachepolicy, 0), 1281 (EXTRACT_SUBREG 1282 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1283 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1284 $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1285 sub0) 1286>; 1287 1288def : GCNPat< 1289 (SIbuffer_atomic_cmpswap 1290 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1291 i32:$voffset, i32:$soffset, imm:$offset, 1292 imm:$cachepolicy, imm), 1293 (EXTRACT_SUBREG 1294 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1295 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1296 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1297 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1298 sub0) 1299>; 1300 1301 1302class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1303 PatFrag constant_ld> : GCNPat < 1304 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1305 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1306 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1307 >; 1308 1309multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1310 ValueType vt, PatFrag atomic_ld> { 1311 def : GCNPat < 1312 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1313 i16:$offset, i1:$slc))), 1314 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1315 >; 1316 1317 def : GCNPat < 1318 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1319 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1320 >; 1321} 1322 1323let SubtargetPredicate = isSICI in { 1324def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1325def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1326def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1327def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1328 1329defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1330defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1331} // End SubtargetPredicate = isSICI 1332 1333multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1334 PatFrag ld> { 1335 1336 def : GCNPat < 1337 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1338 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1339 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1340 >; 1341} 1342 1343let OtherPredicates = [Has16BitInsts] in { 1344 1345defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1346defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1347defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1348defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1349 1350defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1351 1352} // End OtherPredicates = [Has16BitInsts] 1353 1354multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1355 MUBUF_Pseudo InstrOffset, 1356 ValueType vt, PatFrag ld> { 1357 def : GCNPat < 1358 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1359 i32:$soffset, u16imm:$offset))), 1360 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1361 >; 1362 1363 def : GCNPat < 1364 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1365 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1366 >; 1367} 1368 1369// XXX - Is it possible to have a complex pattern in a PatFrag? 1370multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen, 1371 MUBUF_Pseudo InstrOffset, 1372 ValueType vt, PatFrag ld> { 1373 def : GCNPat < 1374 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1375 i32:$soffset, u16imm:$offset)))), 1376 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1377 >; 1378 1379 def : GCNPat < 1380 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1381 i32:$soffset, u16imm:$offset)))))), 1382 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1383 >; 1384 1385 1386 def : GCNPat < 1387 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))), 1388 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1389 >; 1390 1391 def : GCNPat < 1392 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))), 1393 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1394 >; 1395} 1396 1397multiclass MUBUFScratchLoadPat_Lo16 <MUBUF_Pseudo InstrOffen, 1398 MUBUF_Pseudo InstrOffset, 1399 ValueType vt, PatFrag ld> { 1400 def : GCNPat < 1401 (build_vector (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1402 i32:$soffset, u16imm:$offset))), 1403 (vt (Hi16Elt vt:$hi))), 1404 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1405 >; 1406 1407 def : GCNPat < 1408 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1409 i32:$soffset, u16imm:$offset))))), 1410 (f16 (Hi16Elt f16:$hi))), 1411 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1412 >; 1413 1414 def : GCNPat < 1415 (build_vector (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1416 (vt (Hi16Elt vt:$hi))), 1417 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1418 >; 1419 1420 def : GCNPat < 1421 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))))), 1422 (f16 (Hi16Elt f16:$hi))), 1423 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1424 >; 1425} 1426 1427defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1428defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1429defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1430defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1431defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1432defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1433defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1434defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1435defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1436defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1437 1438let OtherPredicates = [D16PreservesUnusedBits] in { 1439defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>; 1440defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>; 1441defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>; 1442 1443defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, i16, load_private>; 1444defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, i16, az_extloadi8_private>; 1445defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, i16, sextloadi8_private>; 1446} 1447multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1448 ValueType vt, PatFrag atomic_st> { 1449 // Store follows atomic op convention so address is forst 1450 def : GCNPat < 1451 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1452 i16:$offset, i1:$slc), vt:$val), 1453 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1454 >; 1455 1456 def : GCNPat < 1457 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1458 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1459 >; 1460} 1461let SubtargetPredicate = isSICI in { 1462defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1463defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1464} // End Predicates = isSICI 1465 1466 1467multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1468 PatFrag st> { 1469 1470 def : GCNPat < 1471 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1472 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1473 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1474 >; 1475} 1476 1477defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1478defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1479 1480multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1481 MUBUF_Pseudo InstrOffset, 1482 ValueType vt, PatFrag st> { 1483 def : GCNPat < 1484 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1485 i32:$soffset, u16imm:$offset)), 1486 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1487 >; 1488 1489 def : GCNPat < 1490 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1491 u16imm:$offset)), 1492 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1493 >; 1494} 1495 1496defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1497defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1498defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1499defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1500defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1501defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1502defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1503 1504 1505let OtherPredicates = [D16PreservesUnusedBits] in { 1506 // Hiding the extract high pattern in the PatFrag seems to not 1507 // automatically increase the complexity. 1508let AddedComplexity = 1 in { 1509defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1510defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1511} 1512} 1513 1514//===----------------------------------------------------------------------===// 1515// MTBUF Patterns 1516//===----------------------------------------------------------------------===// 1517 1518//===----------------------------------------------------------------------===// 1519// tbuffer_load/store_format patterns 1520//===----------------------------------------------------------------------===// 1521 1522multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1523 string opcode> { 1524 def : GCNPat< 1525 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1526 imm:$format, imm:$cachepolicy, 0)), 1527 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1528 (as_i8imm $format), 1529 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1530 >; 1531 1532 def : GCNPat< 1533 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1534 imm:$format, imm:$cachepolicy, imm)), 1535 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1536 (as_i8imm $format), 1537 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1538 >; 1539 1540 def : GCNPat< 1541 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1542 imm:$format, imm:$cachepolicy, 0)), 1543 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1544 (as_i8imm $format), 1545 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1546 >; 1547 1548 def : GCNPat< 1549 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1550 imm:$format, imm:$cachepolicy, imm)), 1551 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1552 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1553 $rsrc, $soffset, (as_i16imm $offset), 1554 (as_i8imm $format), 1555 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1556 >; 1557} 1558 1559defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1560defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1561defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1562defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1563defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1564defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1565 1566let SubtargetPredicate = HasUnpackedD16VMem in { 1567 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1568 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1569 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1570} // End HasUnpackedD16VMem. 1571 1572let SubtargetPredicate = HasPackedD16VMem in { 1573 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1574 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1575 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1576} // End HasPackedD16VMem. 1577 1578multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1579 string opcode> { 1580 def : GCNPat< 1581 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1582 imm:$format, imm:$cachepolicy, 0), 1583 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1584 (as_i16imm $offset), (as_i8imm $format), 1585 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1586 >; 1587 1588 def : GCNPat< 1589 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1590 imm:$format, imm:$cachepolicy, imm), 1591 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1592 (as_i16imm $offset), (as_i8imm $format), 1593 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1594 >; 1595 1596 def : GCNPat< 1597 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1598 imm:$format, imm:$cachepolicy, 0), 1599 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1600 (as_i16imm $offset), (as_i8imm $format), 1601 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1602 >; 1603 1604 def : GCNPat< 1605 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1606 imm:$offset, imm:$format, imm:$cachepolicy, imm), 1607 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1608 $vdata, 1609 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1610 $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), 1611 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1612 >; 1613} 1614 1615defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1616defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1617defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">; 1618defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1619defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1620defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1621defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">; 1622defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1623 1624let SubtargetPredicate = HasUnpackedD16VMem in { 1625 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1626 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1627 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1628} // End HasUnpackedD16VMem. 1629 1630let SubtargetPredicate = HasPackedD16VMem in { 1631 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1632 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1633 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1634} // End HasPackedD16VMem. 1635 1636//===----------------------------------------------------------------------===// 1637// Target instructions, move to the appropriate target TD file 1638//===----------------------------------------------------------------------===// 1639 1640//===----------------------------------------------------------------------===// 1641// SI 1642//===----------------------------------------------------------------------===// 1643 1644class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1645 MUBUF_Real<op, ps>, 1646 Enc64, 1647 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1648 let AssemblerPredicate=isSICI; 1649 let DecoderNamespace="SICI"; 1650 1651 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1652 let Inst{12} = ps.offen; 1653 let Inst{13} = ps.idxen; 1654 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1655 let Inst{15} = ps.addr64; 1656 let Inst{16} = !if(ps.lds, 1, 0); 1657 let Inst{24-18} = op; 1658 let Inst{31-26} = 0x38; //encoding 1659 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1660 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1661 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1662 let Inst{54} = !if(ps.has_slc, slc, ?); 1663 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1664 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1665} 1666 1667multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1668 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1669 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1670 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1671 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1672 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1673} 1674 1675multiclass MUBUF_Real_AllAddr_Lds_si<bits<7> op> { 1676 1677 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1678 MUBUFLdsTable<0, NAME # "_OFFSET_si">; 1679 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1680 MUBUFLdsTable<0, NAME # "_ADDR64_si">; 1681 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1682 MUBUFLdsTable<0, NAME # "_OFFEN_si">; 1683 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1684 MUBUFLdsTable<0, NAME # "_IDXEN_si">; 1685 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1686 MUBUFLdsTable<0, NAME # "_BOTHEN_si">; 1687 1688 def _LDS_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1689 MUBUFLdsTable<1, NAME # "_OFFSET_si">; 1690 def _LDS_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1691 MUBUFLdsTable<1, NAME # "_ADDR64_si">; 1692 def _LDS_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1693 MUBUFLdsTable<1, NAME # "_OFFEN_si">; 1694 def _LDS_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1695 MUBUFLdsTable<1, NAME # "_IDXEN_si">; 1696 def _LDS_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1697 MUBUFLdsTable<1, NAME # "_BOTHEN_si">; 1698} 1699 1700multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1701 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1702 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1703 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1704 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1705 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1706} 1707 1708defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_si <0x00>; 1709defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1710defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1711defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1712defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1713defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1714defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1715defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1716defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_si <0x08>; 1717defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_si <0x09>; 1718defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_si <0x0a>; 1719defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_si <0x0b>; 1720defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_si <0x0c>; 1721defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1722defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1723defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1724defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1725defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1726defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1727defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1728defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1729defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1730 1731defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1732defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1733defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1734defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1735//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1736defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1737defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1738defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1739defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1740defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1741defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1742defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1743defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1744defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1745 1746//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1747//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1748//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1749defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1750defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1751defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1752defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1753//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1754defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1755defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1756defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1757defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1758defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1759defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1760defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1761defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1762defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1763// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1764//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1765//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1766//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1767 1768def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1769def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1770 1771class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1772 MTBUF_Real<ps>, 1773 Enc64, 1774 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1775 let AssemblerPredicate=isSICI; 1776 let DecoderNamespace="SICI"; 1777 1778 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1779 let Inst{12} = ps.offen; 1780 let Inst{13} = ps.idxen; 1781 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1782 let Inst{15} = ps.addr64; 1783 let Inst{18-16} = op; 1784 let Inst{22-19} = dfmt; 1785 let Inst{25-23} = nfmt; 1786 let Inst{31-26} = 0x3a; //encoding 1787 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1788 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1789 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1790 let Inst{54} = !if(ps.has_slc, slc, ?); 1791 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1792 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1793} 1794 1795multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1796 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1797 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1798 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1799 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1800 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1801} 1802 1803defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1804defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1805defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1806defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1807defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1808defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1809defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1810defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1811 1812//===----------------------------------------------------------------------===// 1813// CI 1814// MTBUF - GFX6, GFX7. 1815//===----------------------------------------------------------------------===// 1816 1817class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1818 MUBUF_Real_si<op, ps> { 1819 let AssemblerPredicate=isCIOnly; 1820 let DecoderNamespace="CI"; 1821} 1822 1823def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1824 1825 1826//===----------------------------------------------------------------------===// 1827// VI 1828//===----------------------------------------------------------------------===// 1829 1830class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1831 MUBUF_Real<op, ps>, 1832 Enc64, 1833 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1834 let AssemblerPredicate=isVI; 1835 let DecoderNamespace="VI"; 1836 1837 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1838 let Inst{12} = ps.offen; 1839 let Inst{13} = ps.idxen; 1840 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1841 let Inst{16} = !if(ps.lds, 1, 0); 1842 let Inst{17} = !if(ps.has_slc, slc, ?); 1843 let Inst{24-18} = op; 1844 let Inst{31-26} = 0x38; //encoding 1845 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1846 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1847 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1848 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1849 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1850} 1851 1852multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1853 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1854 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1855 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1856 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1857} 1858 1859multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 1860 1861 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1862 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 1863 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1864 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 1865 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1866 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 1867 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1868 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 1869 1870 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1871 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 1872 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1873 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 1874 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1875 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 1876 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1877 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 1878} 1879 1880class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 1881 MUBUF_Real<op, ps>, 1882 Enc64, 1883 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 1884 let AssemblerPredicate=HasUnpackedD16VMem; 1885 let DecoderNamespace="GFX80_UNPACKED"; 1886 1887 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1888 let Inst{12} = ps.offen; 1889 let Inst{13} = ps.idxen; 1890 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1891 let Inst{16} = !if(ps.lds, 1, 0); 1892 let Inst{17} = !if(ps.has_slc, slc, ?); 1893 let Inst{24-18} = op; 1894 let Inst{31-26} = 0x38; //encoding 1895 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1896 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1897 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1898 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1899 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1900} 1901 1902multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 1903 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1904 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1905 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1906 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1907} 1908 1909multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1910 MUBUF_Real_AllAddr_vi<op> { 1911 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1912 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1913 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1914 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1915} 1916 1917defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 1918defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1919defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1920defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1921defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1922defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1923defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1924defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1925let SubtargetPredicate = HasUnpackedD16VMem in { 1926 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 1927 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 1928 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 1929 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 1930 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 1931 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 1932 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 1933 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 1934} // End HasUnpackedD16VMem. 1935let SubtargetPredicate = HasPackedD16VMem in { 1936 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 1937 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 1938 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 1939 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 1940 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 1941 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 1942 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 1943 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 1944} // End HasPackedD16VMem. 1945defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 1946defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 1947defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 1948defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 1949defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 1950defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 1951defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 1952defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 1953defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1954defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1955defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1956defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1957defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1958defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1959defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1960defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1961 1962defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1963defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1964defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1965defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1966defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1967defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1968 1969defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 1970defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 1971 1972defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1973defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1974defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1975defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1976defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1977defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1978defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1979defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1980defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1981defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1982defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1983defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1984defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1985 1986defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1987defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1988defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1989defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1990defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1991defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1992defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1993defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1994defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1995defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1996defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1997defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1998defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1999 2000def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2001 2002def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2003def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2004 2005class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2006 MTBUF_Real<ps>, 2007 Enc64, 2008 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2009 let AssemblerPredicate=isVI; 2010 let DecoderNamespace="VI"; 2011 2012 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2013 let Inst{12} = ps.offen; 2014 let Inst{13} = ps.idxen; 2015 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2016 let Inst{18-15} = op; 2017 let Inst{22-19} = dfmt; 2018 let Inst{25-23} = nfmt; 2019 let Inst{31-26} = 0x3a; //encoding 2020 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2021 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2022 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2023 let Inst{54} = !if(ps.has_slc, slc, ?); 2024 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2025 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2026} 2027 2028multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2029 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2030 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2031 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2032 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2033} 2034 2035class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2036 MTBUF_Real<ps>, 2037 Enc64, 2038 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2039 let AssemblerPredicate=HasUnpackedD16VMem; 2040 let DecoderNamespace="GFX80_UNPACKED"; 2041 2042 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2043 let Inst{12} = ps.offen; 2044 let Inst{13} = ps.idxen; 2045 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2046 let Inst{18-15} = op; 2047 let Inst{22-19} = dfmt; 2048 let Inst{25-23} = nfmt; 2049 let Inst{31-26} = 0x3a; //encoding 2050 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2051 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2052 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2053 let Inst{54} = !if(ps.has_slc, slc, ?); 2054 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2055 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2056} 2057 2058multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2059 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2060 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2061 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2062 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2063} 2064 2065defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2066defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2067defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2068defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2069defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2070defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2071defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2072defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2073let SubtargetPredicate = HasUnpackedD16VMem in { 2074 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2075 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2076 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2077 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2078 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2079 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2080 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2081 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2082} // End HasUnpackedD16VMem. 2083let SubtargetPredicate = HasPackedD16VMem in { 2084 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2085 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2086 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2087 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2088 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2089 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2090 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2091 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2092} // End HasUnpackedD16VMem. 2093 2094def MUBUFInfoTable : GenericTable { 2095 let FilterClass = "MUBUF_Pseudo"; 2096 let CppTypeName = "MUBUFInfo"; 2097 let Fields = ["Opcode", "BaseOpcode", "dwords", "has_vaddr", "has_srsrc", "has_soffset"]; 2098 2099 let PrimaryKey = ["Opcode"]; 2100 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2101} 2102 2103def getMUBUFInfoFromOpcode : SearchIndex { 2104 let Table = MUBUFInfoTable; 2105 let Key = ["Opcode"]; 2106} 2107 2108def getMUBUFInfoFromBaseOpcodeAndDwords : SearchIndex { 2109 let Table = MUBUFInfoTable; 2110 let Key = ["BaseOpcode", "dwords"]; 2111} 2112