1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 15def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 16 17def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 18def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 19def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 20 21class MubufLoad <SDPatternOperator op> : PatFrag < 22 (ops node:$ptr), (op node:$ptr), [{ 23 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 24 return AS == AMDGPUAS::GLOBAL_ADDRESS || 25 AS == AMDGPUAS::CONSTANT_ADDRESS; 26}]>; 27 28def mubuf_load : MubufLoad <load>; 29def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 30def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 31def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 32def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 33def mubuf_load_atomic : MubufLoad <atomic_load>; 34 35def BUFAddrKind { 36 int Offset = 0; 37 int OffEn = 1; 38 int IdxEn = 2; 39 int BothEn = 3; 40 int Addr64 = 4; 41} 42 43class getAddrName<int addrKind> { 44 string ret = 45 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 46 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 47 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 48 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 49 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 50 ""))))); 51} 52 53class MUBUFAddr64Table <bit is_addr64, string Name> { 54 bit IsAddr64 = is_addr64; 55 string OpName = Name; 56} 57 58class MUBUFLdsTable <bit is_lds, string Name> { 59 bit IsLds = is_lds; 60 string OpName = Name; 61} 62 63class MTBUFAddr64Table <bit is_addr64, string Name> { 64 bit IsAddr64 = is_addr64; 65 string OpName = Name; 66} 67 68//===----------------------------------------------------------------------===// 69// MTBUF classes 70//===----------------------------------------------------------------------===// 71 72class MTBUF_Pseudo <string opName, dag outs, dag ins, 73 string asmOps, list<dag> pattern=[]> : 74 InstSI<outs, ins, "", pattern>, 75 SIMCInstr<opName, SIEncodingFamily.NONE> { 76 77 let isPseudo = 1; 78 let isCodeGenOnly = 1; 79 let Size = 8; 80 let UseNamedOperandTable = 1; 81 82 string Mnemonic = opName; 83 string AsmOperands = asmOps; 84 85 let VM_CNT = 1; 86 let EXP_CNT = 1; 87 let MTBUF = 1; 88 let Uses = [EXEC]; 89 let hasSideEffects = 0; 90 let SchedRW = [WriteVMEM]; 91 92 let AsmMatchConverter = "cvtMtbuf"; 93 94 bits<1> offen = 0; 95 bits<1> idxen = 0; 96 bits<1> addr64 = 0; 97 bits<1> has_vdata = 1; 98 bits<1> has_vaddr = 1; 99 bits<1> has_glc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> has_srsrc = 1; 102 bits<1> has_soffset = 1; 103 bits<1> has_offset = 1; 104 bits<1> has_slc = 1; 105 bits<1> has_tfe = 1; 106} 107 108class MTBUF_Real <MTBUF_Pseudo ps> : 109 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 110 111 let isPseudo = 0; 112 let isCodeGenOnly = 0; 113 114 // copy relevant pseudo op flags 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 121 bits<12> offset; 122 bits<1> glc; 123 bits<7> format; 124 bits<8> vaddr; 125 bits<8> vdata; 126 bits<7> srsrc; 127 bits<1> slc; 128 bits<1> tfe; 129 bits<8> soffset; 130 131 bits<4> dfmt = format{3-0}; 132 bits<3> nfmt = format{6-4}; 133} 134 135class getMTBUFInsDA<list<RegisterClass> vdataList, 136 list<RegisterClass> vaddrList=[]> { 137 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 138 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 139 dag InsNoData = !if(!empty(vaddrList), 140 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 141 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe), 142 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 143 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe) 144 ); 145 dag InsData = !if(!empty(vaddrList), 146 (ins vdataClass:$vdata, SReg_128:$srsrc, 147 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 148 SLC:$slc, TFE:$tfe), 149 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 150 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 151 SLC:$slc, TFE:$tfe) 152 ); 153 dag ret = !if(!empty(vdataList), InsNoData, InsData); 154} 155 156class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 157 dag ret = 158 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 159 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 160 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 161 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 162 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 163 (ins)))))); 164} 165 166class getMTBUFAsmOps<int addrKind> { 167 string Pfx = 168 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 169 !if(!eq(addrKind, BUFAddrKind.OffEn), 170 "$vaddr, $srsrc, $format, $soffset offen", 171 !if(!eq(addrKind, BUFAddrKind.IdxEn), 172 "$vaddr, $srsrc, $format, $soffset idxen", 173 !if(!eq(addrKind, BUFAddrKind.BothEn), 174 "$vaddr, $srsrc, $format, $soffset idxen offen", 175 !if(!eq(addrKind, BUFAddrKind.Addr64), 176 "$vaddr, $srsrc, $format, $soffset addr64", 177 ""))))); 178 string ret = Pfx # "$offset"; 179} 180 181class MTBUF_SetupAddr<int addrKind> { 182 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 183 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 184 185 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 187 188 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 189 190 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 191} 192 193class MTBUF_Load_Pseudo <string opName, 194 int addrKind, 195 RegisterClass vdataClass, 196 list<dag> pattern=[], 197 // Workaround bug bz30254 198 int addrKindCopy = addrKind> 199 : MTBUF_Pseudo<opName, 200 (outs vdataClass:$vdata), 201 getMTBUFIns<addrKindCopy>.ret, 202 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 203 pattern>, 204 MTBUF_SetupAddr<addrKindCopy> { 205 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 206 let mayLoad = 1; 207 let mayStore = 0; 208} 209 210multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 211 ValueType load_vt = i32, 212 SDPatternOperator ld = null_frag> { 213 214 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 215 [(set load_vt:$vdata, 216 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 217 i1:$glc, i1:$slc, i1:$tfe)))]>, 218 MTBUFAddr64Table<0, NAME>; 219 220 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 221 [(set load_vt:$vdata, 222 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 223 i8:$format, i1:$glc, i1:$slc, i1:$tfe)))]>, 224 MTBUFAddr64Table<1, NAME>; 225 226 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 227 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 228 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 229 230 let DisableWQM = 1 in { 231 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 232 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 233 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 234 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 235 } 236} 237 238class MTBUF_Store_Pseudo <string opName, 239 int addrKind, 240 RegisterClass vdataClass, 241 list<dag> pattern=[], 242 // Workaround bug bz30254 243 int addrKindCopy = addrKind, 244 RegisterClass vdataClassCopy = vdataClass> 245 : MTBUF_Pseudo<opName, 246 (outs), 247 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 248 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 249 pattern>, 250 MTBUF_SetupAddr<addrKindCopy> { 251 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 252 let mayLoad = 0; 253 let mayStore = 1; 254} 255 256multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 257 ValueType store_vt = i32, 258 SDPatternOperator st = null_frag> { 259 260 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 261 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 262 i16:$offset, i8:$format, i1:$glc, 263 i1:$slc, i1:$tfe))]>, 264 MTBUFAddr64Table<0, NAME>; 265 266 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 267 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 268 i16:$offset, i8:$format, i1:$glc, 269 i1:$slc, i1:$tfe))]>, 270 MTBUFAddr64Table<1, NAME>; 271 272 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 273 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 274 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 275 276 let DisableWQM = 1 in { 277 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 278 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 279 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 280 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 281 } 282} 283 284 285//===----------------------------------------------------------------------===// 286// MUBUF classes 287//===----------------------------------------------------------------------===// 288 289class MUBUF_Pseudo <string opName, dag outs, dag ins, 290 string asmOps, list<dag> pattern=[]> : 291 InstSI<outs, ins, "", pattern>, 292 SIMCInstr<opName, SIEncodingFamily.NONE> { 293 294 let isPseudo = 1; 295 let isCodeGenOnly = 1; 296 let Size = 8; 297 let UseNamedOperandTable = 1; 298 299 string Mnemonic = opName; 300 string AsmOperands = asmOps; 301 302 let VM_CNT = 1; 303 let EXP_CNT = 1; 304 let MUBUF = 1; 305 let Uses = [EXEC]; 306 let hasSideEffects = 0; 307 let SchedRW = [WriteVMEM]; 308 309 let AsmMatchConverter = "cvtMubuf"; 310 311 bits<1> offen = 0; 312 bits<1> idxen = 0; 313 bits<1> addr64 = 0; 314 bits<1> lds = 0; 315 bits<1> has_vdata = 1; 316 bits<1> has_vaddr = 1; 317 bits<1> has_glc = 1; 318 bits<1> glc_value = 0; // the value for glc if no such operand 319 bits<1> has_srsrc = 1; 320 bits<1> has_soffset = 1; 321 bits<1> has_offset = 1; 322 bits<1> has_slc = 1; 323 bits<1> has_tfe = 1; 324} 325 326class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 327 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 328 329 let isPseudo = 0; 330 let isCodeGenOnly = 0; 331 332 // copy relevant pseudo op flags 333 let SubtargetPredicate = ps.SubtargetPredicate; 334 let AsmMatchConverter = ps.AsmMatchConverter; 335 let Constraints = ps.Constraints; 336 let DisableEncoding = ps.DisableEncoding; 337 let TSFlags = ps.TSFlags; 338 339 bits<12> offset; 340 bits<1> glc; 341 bits<8> vaddr; 342 bits<8> vdata; 343 bits<7> srsrc; 344 bits<1> slc; 345 bits<1> tfe; 346 bits<8> soffset; 347} 348 349 350// For cache invalidation instructions. 351class MUBUF_Invalidate <string opName, SDPatternOperator node> : 352 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 353 354 let AsmMatchConverter = ""; 355 356 let hasSideEffects = 1; 357 let mayStore = 1; 358 359 // Set everything to 0. 360 let offen = 0; 361 let idxen = 0; 362 let addr64 = 0; 363 let has_vdata = 0; 364 let has_vaddr = 0; 365 let has_glc = 0; 366 let glc_value = 0; 367 let has_srsrc = 0; 368 let has_soffset = 0; 369 let has_offset = 0; 370 let has_slc = 0; 371 let has_tfe = 0; 372} 373 374class getMUBUFInsDA<list<RegisterClass> vdataList, 375 list<RegisterClass> vaddrList=[], 376 bit isLds = 0> { 377 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 378 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 379 dag InsNoData = !if(!empty(vaddrList), 380 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 381 offset:$offset, GLC:$glc, SLC:$slc), 382 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 383 offset:$offset, GLC:$glc, SLC:$slc) 384 ); 385 dag InsData = !if(!empty(vaddrList), 386 (ins vdataClass:$vdata, SReg_128:$srsrc, 387 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 388 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 389 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 390 ); 391 dag ret = !con( 392 !if(!empty(vdataList), InsNoData, InsData), 393 !if(isLds, (ins), (ins TFE:$tfe)) 394 ); 395} 396 397class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 398 dag ret = 399 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 400 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 401 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 402 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 403 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 404 (ins)))))); 405} 406 407class getMUBUFAsmOps<int addrKind> { 408 string Pfx = 409 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 410 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 411 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 412 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 413 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 414 ""))))); 415 string ret = Pfx # "$offset"; 416} 417 418class MUBUF_SetupAddr<int addrKind> { 419 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 420 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 421 422 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 423 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 424 425 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 426 427 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 428} 429 430class MUBUF_Load_Pseudo <string opName, 431 int addrKind, 432 RegisterClass vdataClass, 433 bit HasTiedDest = 0, 434 bit isLds = 0, 435 list<dag> pattern=[], 436 // Workaround bug bz30254 437 int addrKindCopy = addrKind> 438 : MUBUF_Pseudo<opName, 439 (outs vdataClass:$vdata), 440 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 441 !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 442 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 443 !if(isLds, " lds", "$tfe"), 444 pattern>, 445 MUBUF_SetupAddr<addrKindCopy> { 446 let PseudoInstr = opName # !if(isLds, "_lds", "") # 447 "_" # getAddrName<addrKindCopy>.ret; 448 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 449 450 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 451 let mayLoad = 1; 452 let mayStore = 0; 453 let maybeAtomic = 1; 454 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 455 let has_tfe = !if(isLds, 0, 1); 456 let lds = isLds; 457} 458 459// FIXME: tfe can't be an operand because it requires a separate 460// opcode because it needs an N+1 register class dest register. 461multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 462 ValueType load_vt = i32, 463 SDPatternOperator ld = null_frag, 464 bit TiedDest = 0, 465 bit isLds = 0> { 466 467 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 468 TiedDest, isLds, 469 !if(isLds, 470 [], 471 [(set load_vt:$vdata, 472 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 473 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 474 475 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 476 TiedDest, isLds, 477 !if(isLds, 478 [], 479 [(set load_vt:$vdata, 480 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 481 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 482 483 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 484 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 485 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 486 487 let DisableWQM = 1 in { 488 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>; 489 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 490 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 491 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 492 } 493} 494 495multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass, 496 ValueType load_vt = i32, 497 SDPatternOperator ld_nolds = null_frag, 498 SDPatternOperator ld_lds = null_frag> { 499 defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>; 500 defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>; 501} 502 503class MUBUF_Store_Pseudo <string opName, 504 int addrKind, 505 RegisterClass vdataClass, 506 list<dag> pattern=[], 507 // Workaround bug bz30254 508 int addrKindCopy = addrKind, 509 RegisterClass vdataClassCopy = vdataClass> 510 : MUBUF_Pseudo<opName, 511 (outs), 512 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 513 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 514 pattern>, 515 MUBUF_SetupAddr<addrKindCopy> { 516 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 517 let mayLoad = 0; 518 let mayStore = 1; 519 let maybeAtomic = 1; 520} 521 522multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 523 ValueType store_vt = i32, 524 SDPatternOperator st = null_frag> { 525 526 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 527 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 528 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 529 MUBUFAddr64Table<0, NAME>; 530 531 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 532 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 533 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 534 MUBUFAddr64Table<1, NAME>; 535 536 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 537 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 538 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 539 540 let DisableWQM = 1 in { 541 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 542 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 543 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 544 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 545 } 546} 547 548class MUBUF_Pseudo_Store_Lds<string opName> 549 : MUBUF_Pseudo<opName, 550 (outs), 551 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 552 " $srsrc, $soffset$offset lds$glc$slc"> { 553 let mayLoad = 0; 554 let mayStore = 1; 555 let maybeAtomic = 1; 556 557 let has_vdata = 0; 558 let has_vaddr = 0; 559 let has_tfe = 0; 560 let lds = 1; 561 562 let Uses = [EXEC, M0]; 563 let AsmMatchConverter = "cvtMubufLds"; 564} 565 566class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 567 list<RegisterClass> vaddrList=[]> { 568 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 569 dag ret = !if(vdata_in, 570 !if(!empty(vaddrList), 571 (ins vdataClass:$vdata_in, 572 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 573 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 574 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 575 ), 576 !if(!empty(vaddrList), 577 (ins vdataClass:$vdata, 578 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 579 (ins vdataClass:$vdata, vaddrClass:$vaddr, 580 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 581 )); 582} 583 584class getMUBUFAtomicIns<int addrKind, 585 RegisterClass vdataClass, 586 bit vdata_in, 587 // Workaround bug bz30254 588 RegisterClass vdataClassCopy=vdataClass> { 589 dag ret = 590 !if(!eq(addrKind, BUFAddrKind.Offset), 591 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 592 !if(!eq(addrKind, BUFAddrKind.OffEn), 593 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 594 !if(!eq(addrKind, BUFAddrKind.IdxEn), 595 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 596 !if(!eq(addrKind, BUFAddrKind.BothEn), 597 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 598 !if(!eq(addrKind, BUFAddrKind.Addr64), 599 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 600 (ins)))))); 601} 602 603class MUBUF_Atomic_Pseudo<string opName, 604 int addrKind, 605 dag outs, 606 dag ins, 607 string asmOps, 608 list<dag> pattern=[], 609 // Workaround bug bz30254 610 int addrKindCopy = addrKind> 611 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 612 MUBUF_SetupAddr<addrKindCopy> { 613 let mayStore = 1; 614 let mayLoad = 1; 615 let hasPostISelHook = 1; 616 let hasSideEffects = 1; 617 let DisableWQM = 1; 618 let has_glc = 0; 619 let has_tfe = 0; 620 let maybeAtomic = 1; 621} 622 623class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 624 RegisterClass vdataClass, 625 list<dag> pattern=[], 626 // Workaround bug bz30254 627 int addrKindCopy = addrKind, 628 RegisterClass vdataClassCopy = vdataClass> 629 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 630 (outs), 631 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 632 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 633 pattern>, 634 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 635 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 636 let glc_value = 0; 637 let AsmMatchConverter = "cvtMubufAtomic"; 638} 639 640class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 641 RegisterClass vdataClass, 642 list<dag> pattern=[], 643 // Workaround bug bz30254 644 int addrKindCopy = addrKind, 645 RegisterClass vdataClassCopy = vdataClass> 646 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 647 (outs vdataClassCopy:$vdata), 648 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 649 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 650 pattern>, 651 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 652 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 653 let glc_value = 1; 654 let Constraints = "$vdata = $vdata_in"; 655 let DisableEncoding = "$vdata_in"; 656 let AsmMatchConverter = "cvtMubufAtomicReturn"; 657} 658 659multiclass MUBUF_Pseudo_Atomics <string opName, 660 RegisterClass vdataClass, 661 ValueType vdataType, 662 SDPatternOperator atomic> { 663 664 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 665 MUBUFAddr64Table <0, NAME>; 666 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 667 MUBUFAddr64Table <1, NAME>; 668 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 669 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 670 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 671 672 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 673 [(set vdataType:$vdata, 674 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 675 vdataType:$vdata_in))]>, 676 MUBUFAddr64Table <0, NAME # "_RTN">; 677 678 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 679 [(set vdataType:$vdata, 680 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 681 vdataType:$vdata_in))]>, 682 MUBUFAddr64Table <1, NAME # "_RTN">; 683 684 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 685 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 686 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 687} 688 689 690//===----------------------------------------------------------------------===// 691// MUBUF Instructions 692//===----------------------------------------------------------------------===// 693 694defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 695 "buffer_load_format_x", VGPR_32 696>; 697defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 698 "buffer_load_format_xy", VReg_64 699>; 700defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 701 "buffer_load_format_xyz", VReg_96 702>; 703defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 704 "buffer_load_format_xyzw", VReg_128 705>; 706defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 707 "buffer_store_format_x", VGPR_32 708>; 709defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 710 "buffer_store_format_xy", VReg_64 711>; 712defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 713 "buffer_store_format_xyz", VReg_96 714>; 715defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 716 "buffer_store_format_xyzw", VReg_128 717>; 718 719let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 720 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 721 "buffer_load_format_d16_x", VGPR_32 722 >; 723 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 724 "buffer_load_format_d16_xy", VReg_64 725 >; 726 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 727 "buffer_load_format_d16_xyz", VReg_96 728 >; 729 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 730 "buffer_load_format_d16_xyzw", VReg_128 731 >; 732 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 733 "buffer_store_format_d16_x", VGPR_32 734 >; 735 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 736 "buffer_store_format_d16_xy", VReg_64 737 >; 738 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 739 "buffer_store_format_d16_xyz", VReg_96 740 >; 741 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 742 "buffer_store_format_d16_xyzw", VReg_128 743 >; 744} // End HasUnpackedD16VMem. 745 746let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 747 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 748 "buffer_load_format_d16_x", VGPR_32 749 >; 750 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 751 "buffer_load_format_d16_xy", VGPR_32 752 >; 753 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 754 "buffer_load_format_d16_xyz", VReg_64 755 >; 756 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 757 "buffer_load_format_d16_xyzw", VReg_64 758 >; 759 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 760 "buffer_store_format_d16_x", VGPR_32 761 >; 762 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 763 "buffer_store_format_d16_xy", VGPR_32 764 >; 765 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 766 "buffer_store_format_d16_xyz", VReg_64 767 >; 768 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 769 "buffer_store_format_d16_xyzw", VReg_64 770 >; 771} // End HasPackedD16VMem. 772 773defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 774 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 775>; 776defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 777 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 778>; 779defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 780 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 781>; 782defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 783 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 784>; 785defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 786 "buffer_load_dword", VGPR_32, i32, mubuf_load 787>; 788defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 789 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 790>; 791defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 792 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 793>; 794defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 795 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 796>; 797 798// This is not described in AMD documentation, 799// but 'lds' versions of these opcodes are available 800// in at least GFX8+ chips. See Bug 37653. 801let SubtargetPredicate = isVI in { 802defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 803 "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1 804>; 805defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 806 "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1 807>; 808defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 809 "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1 810>; 811} 812 813defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 814 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 815>; 816defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 817 "buffer_store_short", VGPR_32, i32, truncstorei16_global 818>; 819defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 820 "buffer_store_dword", VGPR_32, i32, store_global 821>; 822defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 823 "buffer_store_dwordx2", VReg_64, v2i32, store_global 824>; 825defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 826 "buffer_store_dwordx3", VReg_96, untyped, store_global 827>; 828defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 829 "buffer_store_dwordx4", VReg_128, v4i32, store_global 830>; 831defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 832 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 833>; 834defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 835 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 836>; 837defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 838 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 839>; 840defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 841 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 842>; 843defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 844 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 845>; 846defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 847 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 848>; 849defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 850 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 851>; 852defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 853 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 854>; 855defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 856 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 857>; 858defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 859 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 860>; 861defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 862 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 863>; 864defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 865 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 866>; 867defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 868 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 869>; 870defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 871 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 872>; 873defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 874 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 875>; 876defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 877 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 878>; 879defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 880 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 881>; 882defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 883 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 884>; 885defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 886 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 887>; 888defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 889 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 890>; 891defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 892 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 893>; 894defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 895 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 896>; 897defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 898 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 899>; 900defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 901 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 902>; 903defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 904 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 905>; 906defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 907 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 908>; 909 910let SubtargetPredicate = isVI in { 911def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 912} 913 914let SubtargetPredicate = isSI in { // isn't on CI & VI 915/* 916defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 917defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 918defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 919defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 920defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 921defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 922defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 923defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 924*/ 925 926def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 927 int_amdgcn_buffer_wbinvl1_sc>; 928} 929 930let SubtargetPredicate = HasD16LoadStore in { 931 932defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 933 "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 934>; 935 936defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 937 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 938>; 939 940defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 941 "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 942>; 943 944defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 945 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 946>; 947 948defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 949 "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 950>; 951 952defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 953 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 954>; 955 956defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 957 "buffer_store_byte_d16_hi", VGPR_32, i32 958>; 959 960defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 961 "buffer_store_short_d16_hi", VGPR_32, i32 962>; 963 964defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 965 "buffer_load_format_d16_hi_x", VGPR_32 966>; 967defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 968 "buffer_store_format_d16_hi_x", VGPR_32 969>; 970 971} // End HasD16LoadStore 972 973def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 974 int_amdgcn_buffer_wbinvl1>; 975 976//===----------------------------------------------------------------------===// 977// MTBUF Instructions 978//===----------------------------------------------------------------------===// 979 980defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 981defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 982defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>; 983defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 984defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 985defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 986defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>; 987defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 988 989let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 990 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 991 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64>; 992 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96>; 993 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128>; 994 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 995 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64>; 996 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96>; 997 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>; 998} // End HasUnpackedD16VMem. 999 1000let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1001 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1002 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32>; 1003 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64>; 1004 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64>; 1005 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1006 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32>; 1007 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64>; 1008 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>; 1009} // End HasPackedD16VMem. 1010 1011let SubtargetPredicate = isCIVI in { 1012 1013//===----------------------------------------------------------------------===// 1014// Instruction definitions for CI and newer. 1015//===----------------------------------------------------------------------===// 1016// Remaining instructions: 1017// BUFFER_LOAD_DWORDX3 1018// BUFFER_STORE_DWORDX3 1019 1020def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1021 int_amdgcn_buffer_wbinvl1_vol>; 1022 1023} // End let SubtargetPredicate = isCIVI 1024 1025//===----------------------------------------------------------------------===// 1026// MUBUF Patterns 1027//===----------------------------------------------------------------------===// 1028 1029def extract_glc : SDNodeXForm<imm, [{ 1030 return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8); 1031}]>; 1032 1033def extract_slc : SDNodeXForm<imm, [{ 1034 return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8); 1035}]>; 1036 1037//===----------------------------------------------------------------------===// 1038// buffer_load/store_format patterns 1039//===----------------------------------------------------------------------===// 1040 1041multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1042 string opcode> { 1043 def : GCNPat< 1044 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1045 imm:$cachepolicy, 0)), 1046 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1047 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1048 >; 1049 1050 def : GCNPat< 1051 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1052 imm:$cachepolicy, 0)), 1053 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1054 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1055 >; 1056 1057 def : GCNPat< 1058 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1059 imm:$cachepolicy, imm)), 1060 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1061 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1062 >; 1063 1064 def : GCNPat< 1065 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1066 imm:$cachepolicy, imm)), 1067 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1068 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1069 $rsrc, $soffset, (as_i16imm $offset), 1070 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1071 >; 1072} 1073 1074defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1075defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1076defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1077defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1078defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1079defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1080 1081let SubtargetPredicate = HasUnpackedD16VMem in { 1082 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1083 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1084 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1085 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1086} // End HasUnpackedD16VMem. 1087 1088let SubtargetPredicate = HasPackedD16VMem in { 1089 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1090 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1091 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1092 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1093 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1094 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1095} // End HasPackedD16VMem. 1096 1097defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1098defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1099defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1100defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1101defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1102defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1103 1104multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1105 string opcode> { 1106 def : GCNPat< 1107 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1108 imm:$cachepolicy, 0), 1109 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1110 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1111 >; 1112 1113 def : GCNPat< 1114 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1115 imm:$cachepolicy, 0), 1116 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1117 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1118 >; 1119 1120 def : GCNPat< 1121 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1122 imm:$cachepolicy, imm), 1123 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1124 (as_i16imm $offset), (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1125 >; 1126 1127 def : GCNPat< 1128 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1129 imm:$cachepolicy, imm), 1130 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1131 $vdata, 1132 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1133 $rsrc, $soffset, (as_i16imm $offset), 1134 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1135 >; 1136} 1137 1138defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1139defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1140defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1141defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1142defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1143defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1144 1145let SubtargetPredicate = HasUnpackedD16VMem in { 1146 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1147 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1148 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1149 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1150} // End HasUnpackedD16VMem. 1151 1152let SubtargetPredicate = HasPackedD16VMem in { 1153 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1154 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1155 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1156 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1157 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1158 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1159} // End HasPackedD16VMem. 1160 1161defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1162defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1163defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1164defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1165defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1166defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1167 1168//===----------------------------------------------------------------------===// 1169// buffer_atomic patterns 1170//===----------------------------------------------------------------------===// 1171 1172multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 1173 def : GCNPat< 1174 (name i32:$vdata_in, v4i32:$rsrc, 0, 1175 0, i32:$soffset, imm:$offset, 1176 imm:$cachepolicy, 0), 1177 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1178 (as_i16imm $offset), (extract_slc $cachepolicy)) 1179 >; 1180 1181 def : GCNPat< 1182 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1183 0, i32:$soffset, imm:$offset, 1184 imm:$cachepolicy, imm), 1185 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1186 (as_i16imm $offset), (extract_slc $cachepolicy)) 1187 >; 1188 1189 def : GCNPat< 1190 (name i32:$vdata_in, v4i32:$rsrc, 0, 1191 i32:$voffset, i32:$soffset, imm:$offset, 1192 imm:$cachepolicy, 0), 1193 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1194 (as_i16imm $offset), (extract_slc $cachepolicy)) 1195 >; 1196 1197 def : GCNPat< 1198 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1199 i32:$voffset, i32:$soffset, imm:$offset, 1200 imm:$cachepolicy, imm), 1201 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1202 $vdata_in, 1203 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1204 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1205 >; 1206} 1207 1208defm : BufferAtomicPatterns<SIbuffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 1209defm : BufferAtomicPatterns<SIbuffer_atomic_add, "BUFFER_ATOMIC_ADD">; 1210defm : BufferAtomicPatterns<SIbuffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 1211defm : BufferAtomicPatterns<SIbuffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 1212defm : BufferAtomicPatterns<SIbuffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 1213defm : BufferAtomicPatterns<SIbuffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 1214defm : BufferAtomicPatterns<SIbuffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 1215defm : BufferAtomicPatterns<SIbuffer_atomic_and, "BUFFER_ATOMIC_AND">; 1216defm : BufferAtomicPatterns<SIbuffer_atomic_or, "BUFFER_ATOMIC_OR">; 1217defm : BufferAtomicPatterns<SIbuffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 1218 1219def : GCNPat< 1220 (SIbuffer_atomic_cmpswap 1221 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1222 0, i32:$soffset, imm:$offset, 1223 imm:$cachepolicy, 0), 1224 (EXTRACT_SUBREG 1225 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1226 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1227 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1228 sub0) 1229>; 1230 1231def : GCNPat< 1232 (SIbuffer_atomic_cmpswap 1233 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1234 0, i32:$soffset, imm:$offset, 1235 imm:$cachepolicy, imm), 1236 (EXTRACT_SUBREG 1237 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1238 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1239 $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1240 sub0) 1241>; 1242 1243def : GCNPat< 1244 (SIbuffer_atomic_cmpswap 1245 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1246 i32:$voffset, i32:$soffset, imm:$offset, 1247 imm:$cachepolicy, 0), 1248 (EXTRACT_SUBREG 1249 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1250 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1251 $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1252 sub0) 1253>; 1254 1255def : GCNPat< 1256 (SIbuffer_atomic_cmpswap 1257 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1258 i32:$voffset, i32:$soffset, imm:$offset, 1259 imm:$cachepolicy, imm), 1260 (EXTRACT_SUBREG 1261 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1262 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1263 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1264 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1265 sub0) 1266>; 1267 1268 1269class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1270 PatFrag constant_ld> : GCNPat < 1271 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1272 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1273 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1274 >; 1275 1276multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1277 ValueType vt, PatFrag atomic_ld> { 1278 def : GCNPat < 1279 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1280 i16:$offset, i1:$slc))), 1281 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1282 >; 1283 1284 def : GCNPat < 1285 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1286 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1287 >; 1288} 1289 1290let SubtargetPredicate = isSICI in { 1291def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1292def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1293def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1294def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1295 1296defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1297defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1298} // End SubtargetPredicate = isSICI 1299 1300multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1301 PatFrag ld> { 1302 1303 def : GCNPat < 1304 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1305 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1306 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1307 >; 1308} 1309 1310let OtherPredicates = [Has16BitInsts] in { 1311 1312defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1313defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1314defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1315defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1316 1317defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1318 1319} // End OtherPredicates = [Has16BitInsts] 1320 1321multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1322 MUBUF_Pseudo InstrOffset, 1323 ValueType vt, PatFrag ld> { 1324 def : GCNPat < 1325 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1326 i32:$soffset, u16imm:$offset))), 1327 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1328 >; 1329 1330 def : GCNPat < 1331 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1332 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1333 >; 1334} 1335 1336// XXX - Is it possible to have a complex pattern in a PatFrag? 1337multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen, 1338 MUBUF_Pseudo InstrOffset, 1339 ValueType vt, PatFrag ld> { 1340 def : GCNPat < 1341 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1342 i32:$soffset, u16imm:$offset)))), 1343 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1344 >; 1345 1346 def : GCNPat < 1347 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1348 i32:$soffset, u16imm:$offset)))))), 1349 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1350 >; 1351 1352 1353 def : GCNPat < 1354 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))), 1355 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1356 >; 1357 1358 def : GCNPat < 1359 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))), 1360 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1361 >; 1362} 1363 1364multiclass MUBUFScratchLoadPat_Lo16 <MUBUF_Pseudo InstrOffen, 1365 MUBUF_Pseudo InstrOffset, 1366 ValueType vt, PatFrag ld> { 1367 def : GCNPat < 1368 (build_vector (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1369 i32:$soffset, u16imm:$offset))), 1370 (vt (Hi16Elt vt:$hi))), 1371 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1372 >; 1373 1374 def : GCNPat < 1375 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1376 i32:$soffset, u16imm:$offset))))), 1377 (f16 (Hi16Elt f16:$hi))), 1378 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1379 >; 1380 1381 def : GCNPat < 1382 (build_vector (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1383 (vt (Hi16Elt vt:$hi))), 1384 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1385 >; 1386 1387 def : GCNPat < 1388 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))))), 1389 (f16 (Hi16Elt f16:$hi))), 1390 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1391 >; 1392} 1393 1394defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1395defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1396defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1397defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1398defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1399defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1400defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1401defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1402defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1403defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1404 1405let OtherPredicates = [D16PreservesUnusedBits] in { 1406defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>; 1407defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>; 1408defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>; 1409 1410defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, i16, load_private>; 1411defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, i16, az_extloadi8_private>; 1412defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, i16, sextloadi8_private>; 1413} 1414 1415// BUFFER_LOAD_DWORD*, addr64=0 1416multiclass MUBUF_Load_Dword <ValueType vt, 1417 MUBUF_Pseudo offset, 1418 MUBUF_Pseudo offen, 1419 MUBUF_Pseudo idxen, 1420 MUBUF_Pseudo bothen> { 1421 1422 def : GCNPat < 1423 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset, 1424 imm:$offset, 0, 0, imm:$glc, imm:$slc, 1425 imm:$tfe)), 1426 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1427 (as_i1imm $slc), (as_i1imm $tfe)) 1428 >; 1429 1430 def : GCNPat < 1431 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1432 imm:$offset, 1, 0, imm:$glc, imm:$slc, 1433 imm:$tfe)), 1434 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1435 (as_i1imm $tfe)) 1436 >; 1437 1438 def : GCNPat < 1439 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1440 imm:$offset, 0, 1, imm:$glc, imm:$slc, 1441 imm:$tfe)), 1442 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1443 (as_i1imm $slc), (as_i1imm $tfe)) 1444 >; 1445 1446 def : GCNPat < 1447 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset, 1448 imm:$offset, 1, 1, imm:$glc, imm:$slc, 1449 imm:$tfe)), 1450 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1451 (as_i1imm $tfe)) 1452 >; 1453} 1454 1455defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN, 1456 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>; 1457defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN, 1458 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>; 1459defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN, 1460 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>; 1461 1462multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1463 ValueType vt, PatFrag atomic_st> { 1464 // Store follows atomic op convention so address is forst 1465 def : GCNPat < 1466 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1467 i16:$offset, i1:$slc), vt:$val), 1468 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1469 >; 1470 1471 def : GCNPat < 1472 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1473 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1474 >; 1475} 1476let SubtargetPredicate = isSICI in { 1477defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1478defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1479} // End Predicates = isSICI 1480 1481 1482multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1483 PatFrag st> { 1484 1485 def : GCNPat < 1486 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1487 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1488 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1489 >; 1490} 1491 1492defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1493defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1494 1495multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1496 MUBUF_Pseudo InstrOffset, 1497 ValueType vt, PatFrag st> { 1498 def : GCNPat < 1499 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1500 i32:$soffset, u16imm:$offset)), 1501 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1502 >; 1503 1504 def : GCNPat < 1505 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1506 u16imm:$offset)), 1507 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1508 >; 1509} 1510 1511defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1512defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1513defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1514defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1515defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1516defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1517defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1518 1519 1520let OtherPredicates = [D16PreservesUnusedBits] in { 1521 // Hiding the extract high pattern in the PatFrag seems to not 1522 // automatically increase the complexity. 1523let AddedComplexity = 1 in { 1524defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1525defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1526} 1527} 1528 1529//===----------------------------------------------------------------------===// 1530// MTBUF Patterns 1531//===----------------------------------------------------------------------===// 1532 1533//===----------------------------------------------------------------------===// 1534// tbuffer_load/store_format patterns 1535//===----------------------------------------------------------------------===// 1536 1537multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1538 string opcode> { 1539 def : GCNPat< 1540 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1541 imm:$format, imm:$cachepolicy, 0)), 1542 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1543 (as_i8imm $format), 1544 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1545 >; 1546 1547 def : GCNPat< 1548 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1549 imm:$format, imm:$cachepolicy, imm)), 1550 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1551 (as_i8imm $format), 1552 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1553 >; 1554 1555 def : GCNPat< 1556 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1557 imm:$format, imm:$cachepolicy, 0)), 1558 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1559 (as_i8imm $format), 1560 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1561 >; 1562 1563 def : GCNPat< 1564 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1565 imm:$format, imm:$cachepolicy, imm)), 1566 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1567 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1568 $rsrc, $soffset, (as_i16imm $offset), 1569 (as_i8imm $format), 1570 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1571 >; 1572} 1573 1574defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1575defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1576defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1577defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1578defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1579defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1580 1581let SubtargetPredicate = HasUnpackedD16VMem in { 1582 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1583 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1584 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1585} // End HasUnpackedD16VMem. 1586 1587let SubtargetPredicate = HasPackedD16VMem in { 1588 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1589 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1590 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1591} // End HasPackedD16VMem. 1592 1593multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1594 string opcode> { 1595 def : GCNPat< 1596 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1597 imm:$format, imm:$cachepolicy, 0), 1598 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1599 (as_i16imm $offset), (as_i8imm $format), 1600 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1601 >; 1602 1603 def : GCNPat< 1604 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1605 imm:$format, imm:$cachepolicy, imm), 1606 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1607 (as_i16imm $offset), (as_i8imm $format), 1608 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1609 >; 1610 1611 def : GCNPat< 1612 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1613 imm:$format, imm:$cachepolicy, 0), 1614 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1615 (as_i16imm $offset), (as_i8imm $format), 1616 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1617 >; 1618 1619 def : GCNPat< 1620 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1621 imm:$offset, imm:$format, imm:$cachepolicy, imm), 1622 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1623 $vdata, 1624 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1625 $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), 1626 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0) 1627 >; 1628} 1629 1630defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1631defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1632defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">; 1633defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1634defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1635defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1636defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">; 1637defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1638 1639let SubtargetPredicate = HasUnpackedD16VMem in { 1640 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1641 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1642 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1643} // End HasUnpackedD16VMem. 1644 1645let SubtargetPredicate = HasPackedD16VMem in { 1646 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1647 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1648 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1649} // End HasPackedD16VMem. 1650 1651//===----------------------------------------------------------------------===// 1652// Target instructions, move to the appropriate target TD file 1653//===----------------------------------------------------------------------===// 1654 1655//===----------------------------------------------------------------------===// 1656// SI 1657//===----------------------------------------------------------------------===// 1658 1659class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1660 MUBUF_Real<op, ps>, 1661 Enc64, 1662 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1663 let AssemblerPredicate=isSICI; 1664 let DecoderNamespace="SICI"; 1665 1666 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1667 let Inst{12} = ps.offen; 1668 let Inst{13} = ps.idxen; 1669 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1670 let Inst{15} = ps.addr64; 1671 let Inst{16} = !if(ps.lds, 1, 0); 1672 let Inst{24-18} = op; 1673 let Inst{31-26} = 0x38; //encoding 1674 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1675 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1676 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1677 let Inst{54} = !if(ps.has_slc, slc, ?); 1678 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1679 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1680} 1681 1682multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1683 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1684 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1685 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1686 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1687 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1688} 1689 1690multiclass MUBUF_Real_AllAddr_Lds_si<bits<7> op> { 1691 1692 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1693 MUBUFLdsTable<0, NAME # "_OFFSET_si">; 1694 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1695 MUBUFLdsTable<0, NAME # "_ADDR64_si">; 1696 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1697 MUBUFLdsTable<0, NAME # "_OFFEN_si">; 1698 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1699 MUBUFLdsTable<0, NAME # "_IDXEN_si">; 1700 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1701 MUBUFLdsTable<0, NAME # "_BOTHEN_si">; 1702 1703 def _LDS_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1704 MUBUFLdsTable<1, NAME # "_OFFSET_si">; 1705 def _LDS_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1706 MUBUFLdsTable<1, NAME # "_ADDR64_si">; 1707 def _LDS_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1708 MUBUFLdsTable<1, NAME # "_OFFEN_si">; 1709 def _LDS_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1710 MUBUFLdsTable<1, NAME # "_IDXEN_si">; 1711 def _LDS_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1712 MUBUFLdsTable<1, NAME # "_BOTHEN_si">; 1713} 1714 1715multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1716 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1717 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1718 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1719 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1720 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1721} 1722 1723defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_si <0x00>; 1724defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1725defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1726defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1727defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1728defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1729defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1730defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1731defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_si <0x08>; 1732defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_si <0x09>; 1733defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_si <0x0a>; 1734defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_si <0x0b>; 1735defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_si <0x0c>; 1736defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1737defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1738defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1739defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1740defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1741defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1742defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1743defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1744defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1745 1746defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1747defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1748defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1749defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1750//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1751defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1752defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1753defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1754defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1755defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1756defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1757defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1758defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1759defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1760 1761//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1762//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1763//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1764defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1765defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1766defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1767defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1768//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1769defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1770defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1771defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1772defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1773defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1774defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1775defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1776defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1777defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1778// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1779//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1780//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1781//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1782 1783def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1784def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1785 1786class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1787 MTBUF_Real<ps>, 1788 Enc64, 1789 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1790 let AssemblerPredicate=isSICI; 1791 let DecoderNamespace="SICI"; 1792 1793 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1794 let Inst{12} = ps.offen; 1795 let Inst{13} = ps.idxen; 1796 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1797 let Inst{15} = ps.addr64; 1798 let Inst{18-16} = op; 1799 let Inst{22-19} = dfmt; 1800 let Inst{25-23} = nfmt; 1801 let Inst{31-26} = 0x3a; //encoding 1802 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1803 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1804 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1805 let Inst{54} = !if(ps.has_slc, slc, ?); 1806 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1807 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1808} 1809 1810multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1811 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1812 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1813 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1814 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1815 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1816} 1817 1818defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1819defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1820defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1821defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1822defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1823defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1824defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1825defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1826 1827//===----------------------------------------------------------------------===// 1828// CI 1829// MTBUF - GFX6, GFX7. 1830//===----------------------------------------------------------------------===// 1831 1832class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1833 MUBUF_Real_si<op, ps> { 1834 let AssemblerPredicate=isCIOnly; 1835 let DecoderNamespace="CI"; 1836} 1837 1838def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1839 1840 1841//===----------------------------------------------------------------------===// 1842// VI 1843//===----------------------------------------------------------------------===// 1844 1845class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1846 MUBUF_Real<op, ps>, 1847 Enc64, 1848 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1849 let AssemblerPredicate=isVI; 1850 let DecoderNamespace="VI"; 1851 1852 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1853 let Inst{12} = ps.offen; 1854 let Inst{13} = ps.idxen; 1855 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1856 let Inst{16} = !if(ps.lds, 1, 0); 1857 let Inst{17} = !if(ps.has_slc, slc, ?); 1858 let Inst{24-18} = op; 1859 let Inst{31-26} = 0x38; //encoding 1860 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1861 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1862 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1863 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1864 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1865} 1866 1867multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1868 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1869 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1870 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1871 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1872} 1873 1874multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 1875 1876 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1877 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 1878 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1879 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 1880 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1881 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 1882 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1883 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 1884 1885 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1886 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 1887 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1888 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 1889 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1890 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 1891 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1892 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 1893} 1894 1895class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 1896 MUBUF_Real<op, ps>, 1897 Enc64, 1898 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 1899 let AssemblerPredicate=HasUnpackedD16VMem; 1900 let DecoderNamespace="GFX80_UNPACKED"; 1901 1902 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1903 let Inst{12} = ps.offen; 1904 let Inst{13} = ps.idxen; 1905 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1906 let Inst{16} = !if(ps.lds, 1, 0); 1907 let Inst{17} = !if(ps.has_slc, slc, ?); 1908 let Inst{24-18} = op; 1909 let Inst{31-26} = 0x38; //encoding 1910 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1911 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1912 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1913 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1914 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1915} 1916 1917multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 1918 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1919 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1920 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1921 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1922} 1923 1924multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1925 MUBUF_Real_AllAddr_vi<op> { 1926 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1927 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1928 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1929 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1930} 1931 1932defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 1933defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1934defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1935defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1936defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1937defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1938defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1939defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1940let SubtargetPredicate = HasUnpackedD16VMem in { 1941 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 1942 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 1943 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 1944 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 1945 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 1946 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 1947 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 1948 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 1949} // End HasUnpackedD16VMem. 1950let SubtargetPredicate = HasPackedD16VMem in { 1951 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 1952 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 1953 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 1954 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 1955 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 1956 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 1957 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 1958 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 1959} // End HasPackedD16VMem. 1960defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 1961defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 1962defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 1963defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 1964defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 1965defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 1966defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 1967defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 1968defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1969defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1970defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1971defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1972defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1973defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1974defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1975defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1976 1977defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1978defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1979defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1980defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1981defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1982defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1983 1984defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 1985defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 1986 1987defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1988defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1989defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1990defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1991defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1992defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1993defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1994defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1995defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1996defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1997defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1998defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1999defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2000 2001defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2002defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2003defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2004defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2005defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2006defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2007defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2008defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2009defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2010defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2011defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2012defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2013defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2014 2015def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2016 2017def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2018def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2019 2020class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2021 MTBUF_Real<ps>, 2022 Enc64, 2023 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2024 let AssemblerPredicate=isVI; 2025 let DecoderNamespace="VI"; 2026 2027 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2028 let Inst{12} = ps.offen; 2029 let Inst{13} = ps.idxen; 2030 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2031 let Inst{18-15} = op; 2032 let Inst{22-19} = dfmt; 2033 let Inst{25-23} = nfmt; 2034 let Inst{31-26} = 0x3a; //encoding 2035 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2036 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2037 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2038 let Inst{54} = !if(ps.has_slc, slc, ?); 2039 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2040 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2041} 2042 2043multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2044 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2045 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2046 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2047 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2048} 2049 2050class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2051 MTBUF_Real<ps>, 2052 Enc64, 2053 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2054 let AssemblerPredicate=HasUnpackedD16VMem; 2055 let DecoderNamespace="GFX80_UNPACKED"; 2056 2057 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2058 let Inst{12} = ps.offen; 2059 let Inst{13} = ps.idxen; 2060 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2061 let Inst{18-15} = op; 2062 let Inst{22-19} = dfmt; 2063 let Inst{25-23} = nfmt; 2064 let Inst{31-26} = 0x3a; //encoding 2065 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2066 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2067 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2068 let Inst{54} = !if(ps.has_slc, slc, ?); 2069 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2070 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2071} 2072 2073multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2074 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2075 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2076 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2077 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2078} 2079 2080defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2081defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2082defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2083defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2084defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2085defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2086defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2087defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2088let SubtargetPredicate = HasUnpackedD16VMem in { 2089 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2090 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2091 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2092 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2093 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2094 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2095 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2096 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2097} // End HasUnpackedD16VMem. 2098let SubtargetPredicate = HasPackedD16VMem in { 2099 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2100 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2101 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2102 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2103 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2104 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2105 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2106 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2107} // End HasUnpackedD16VMem. 2108