1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MUBUFLdsTable <bit is_lds, string Name> { 39 bit IsLds = is_lds; 40 string OpName = Name; 41} 42 43class MTBUFAddr64Table <bit is_addr64, string Name> { 44 bit IsAddr64 = is_addr64; 45 string OpName = Name; 46} 47 48//===----------------------------------------------------------------------===// 49// MTBUF classes 50//===----------------------------------------------------------------------===// 51 52class MTBUFGetBaseOpcode<string Op> { 53 string ret = !subst("FORMAT_XY", "FORMAT_X", 54 !subst("FORMAT_XYZ", "FORMAT_X", 55 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 56} 57 58class getMTBUFElements<string Op> { 59 int ret = 1; 60} 61 62 63class MTBUF_Pseudo <string opName, dag outs, dag ins, 64 string asmOps, list<dag> pattern=[]> : 65 InstSI<outs, ins, "", pattern>, 66 SIMCInstr<opName, SIEncodingFamily.NONE> { 67 68 let isPseudo = 1; 69 let isCodeGenOnly = 1; 70 let Size = 8; 71 let UseNamedOperandTable = 1; 72 73 string Mnemonic = opName; 74 string AsmOperands = asmOps; 75 76 Instruction Opcode = !cast<Instruction>(NAME); 77 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 78 79 let VM_CNT = 1; 80 let EXP_CNT = 1; 81 let MTBUF = 1; 82 let Uses = [EXEC]; 83 let hasSideEffects = 0; 84 let SchedRW = [WriteVMEM]; 85 86 let AsmMatchConverter = "cvtMtbuf"; 87 88 bits<1> offen = 0; 89 bits<1> idxen = 0; 90 bits<1> addr64 = 0; 91 bits<1> has_vdata = 1; 92 bits<1> has_vaddr = 1; 93 bits<1> has_glc = 1; 94 bits<1> has_dlc = 1; 95 bits<1> glc_value = 0; // the value for glc if no such operand 96 bits<1> dlc_value = 0; // the value for dlc if no such operand 97 bits<1> has_srsrc = 1; 98 bits<1> has_soffset = 1; 99 bits<1> has_offset = 1; 100 bits<1> has_slc = 1; 101 bits<1> has_tfe = 1; 102 bits<4> elements = 0; 103 bits<1> has_sccb = 1; 104 bits<1> sccb_value = 0; 105} 106 107class MTBUF_Real <MTBUF_Pseudo ps> : 108 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 109 110 let isPseudo = 0; 111 let isCodeGenOnly = 0; 112 113 // copy relevant pseudo op flags 114 let UseNamedOperandTable = ps.UseNamedOperandTable; 115 let SubtargetPredicate = ps.SubtargetPredicate; 116 let AsmMatchConverter = ps.AsmMatchConverter; 117 let Constraints = ps.Constraints; 118 let DisableEncoding = ps.DisableEncoding; 119 let TSFlags = ps.TSFlags; 120 let SchedRW = ps.SchedRW; 121 122 bits<12> offset; 123 bits<5> cpol; 124 bits<7> format; 125 bits<8> vaddr; 126 bits<10> vdata; 127 bits<7> srsrc; 128 bits<1> tfe; 129 bits<8> soffset; 130 131 bits<4> dfmt = format{3-0}; 132 bits<3> nfmt = format{6-4}; 133 134 // GFX90A+ only: instruction uses AccVGPR for data 135 // Bit superceedes tfe. 136 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 137} 138 139class getMTBUFInsDA<list<RegisterClass> vdataList, 140 list<RegisterClass> vaddrList=[]> { 141 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 142 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 143 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 144 dag InsNoData = !if(!empty(vaddrList), 145 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 146 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 147 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 148 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 149 ); 150 dag InsData = !if(!empty(vaddrList), 151 (ins vdata_op:$vdata, SReg_128:$srsrc, 152 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 153 TFE:$tfe, SWZ:$swz), 154 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 155 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 156 TFE:$tfe, SWZ:$swz) 157 ); 158 dag ret = !if(!empty(vdataList), InsNoData, InsData); 159} 160 161class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 162 dag ret = 163 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 164 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 168 (ins)))))); 169} 170 171class getMTBUFAsmOps<int addrKind> { 172 string Pfx = 173 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 174 !if(!eq(addrKind, BUFAddrKind.OffEn), 175 "$vaddr, $srsrc,$format $soffset offen", 176 !if(!eq(addrKind, BUFAddrKind.IdxEn), 177 "$vaddr, $srsrc,$format $soffset idxen", 178 !if(!eq(addrKind, BUFAddrKind.BothEn), 179 "$vaddr, $srsrc,$format $soffset idxen offen", 180 !if(!eq(addrKind, BUFAddrKind.Addr64), 181 "$vaddr, $srsrc,$format $soffset addr64", 182 ""))))); 183 string ret = Pfx # "$offset"; 184} 185 186class MTBUF_SetupAddr<int addrKind> { 187 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 188 !eq(addrKind, BUFAddrKind.BothEn)); 189 190 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 191 !eq(addrKind, BUFAddrKind.BothEn)); 192 193 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 194 195 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 196} 197 198class MTBUF_Load_Pseudo <string opName, 199 int addrKind, 200 RegisterClass vdataClass, 201 int elems, 202 list<dag> pattern=[], 203 // Workaround bug bz30254 204 int addrKindCopy = addrKind> 205 : MTBUF_Pseudo<opName, 206 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 207 getMTBUFIns<addrKindCopy>.ret, 208 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 209 pattern>, 210 MTBUF_SetupAddr<addrKindCopy> { 211 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 212 let mayLoad = 1; 213 let mayStore = 0; 214 let elements = elems; 215} 216 217multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 218 int elems, ValueType load_vt = i32, 219 SDPatternOperator ld = null_frag> { 220 221 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 222 MTBUFAddr64Table<0, NAME>; 223 224 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 225 MTBUFAddr64Table<1, NAME>; 226 227 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 228 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 229 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 230 231 let DisableWQM = 1 in { 232 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 233 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 234 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 235 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 236 } 237} 238 239class MTBUF_Store_Pseudo <string opName, 240 int addrKind, 241 RegisterClass vdataClass, 242 int elems, 243 list<dag> pattern=[], 244 // Workaround bug bz30254 245 int addrKindCopy = addrKind, 246 RegisterClass vdataClassCopy = vdataClass> 247 : MTBUF_Pseudo<opName, 248 (outs), 249 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 250 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 251 pattern>, 252 MTBUF_SetupAddr<addrKindCopy> { 253 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 254 let mayLoad = 0; 255 let mayStore = 1; 256 let elements = elems; 257} 258 259multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 260 int elems, ValueType store_vt = i32, 261 SDPatternOperator st = null_frag> { 262 263 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 264 MTBUFAddr64Table<0, NAME>; 265 266 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 267 MTBUFAddr64Table<1, NAME>; 268 269 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 270 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 271 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 272 273 let DisableWQM = 1 in { 274 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 275 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 276 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 277 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 278 } 279} 280 281 282//===----------------------------------------------------------------------===// 283// MUBUF classes 284//===----------------------------------------------------------------------===// 285 286class MUBUFGetBaseOpcode<string Op> { 287 string ret = !subst("DWORDX2", "DWORD", 288 !subst("DWORDX3", "DWORD", 289 !subst("DWORDX4", "DWORD", Op))); 290} 291 292class MUBUF_Pseudo <string opName, dag outs, dag ins, 293 string asmOps, list<dag> pattern=[]> : 294 InstSI<outs, ins, "", pattern>, 295 SIMCInstr<opName, SIEncodingFamily.NONE> { 296 297 let isPseudo = 1; 298 let isCodeGenOnly = 1; 299 let Size = 8; 300 let UseNamedOperandTable = 1; 301 302 string Mnemonic = opName; 303 string AsmOperands = asmOps; 304 305 Instruction Opcode = !cast<Instruction>(NAME); 306 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 307 308 let VM_CNT = 1; 309 let EXP_CNT = 1; 310 let MUBUF = 1; 311 let Uses = [EXEC]; 312 let hasSideEffects = 0; 313 let SchedRW = [WriteVMEM]; 314 315 let AsmMatchConverter = "cvtMubuf"; 316 317 bits<1> offen = 0; 318 bits<1> idxen = 0; 319 bits<1> addr64 = 0; 320 bits<1> lds = 0; 321 bits<1> has_vdata = 1; 322 bits<1> has_vaddr = 1; 323 bits<1> has_glc = 1; 324 bits<1> has_dlc = 1; 325 bits<1> glc_value = 0; // the value for glc if no such operand 326 bits<1> dlc_value = 0; // the value for dlc if no such operand 327 bits<1> has_srsrc = 1; 328 bits<1> has_soffset = 1; 329 bits<1> has_offset = 1; 330 bits<1> has_slc = 1; 331 bits<1> has_tfe = 1; 332 bits<4> elements = 0; 333 bits<1> has_sccb = 1; 334 bits<1> sccb_value = 0; 335} 336 337class MUBUF_Real <MUBUF_Pseudo ps> : 338 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 339 340 let isPseudo = 0; 341 let isCodeGenOnly = 0; 342 343 // copy relevant pseudo op flags 344 let SubtargetPredicate = ps.SubtargetPredicate; 345 let AsmMatchConverter = ps.AsmMatchConverter; 346 let OtherPredicates = ps.OtherPredicates; 347 let Constraints = ps.Constraints; 348 let DisableEncoding = ps.DisableEncoding; 349 let TSFlags = ps.TSFlags; 350 let UseNamedOperandTable = ps.UseNamedOperandTable; 351 let SchedRW = ps.SchedRW; 352 353 bits<12> offset; 354 bits<5> cpol; 355 bits<8> vaddr; 356 bits<10> vdata; 357 bits<7> srsrc; 358 bits<1> tfe; 359 bits<8> soffset; 360 361 // GFX90A+ only: instruction uses AccVGPR for data 362 // Bit superceedes tfe. 363 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 364} 365 366 367// For cache invalidation instructions. 368class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 369 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 370 371 let AsmMatchConverter = ""; 372 373 let hasSideEffects = 1; 374 let mayLoad = 0; 375 let mayStore = 0; 376 377 // Set everything to 0. 378 let offen = 0; 379 let idxen = 0; 380 let addr64 = 0; 381 let has_vdata = 0; 382 let has_vaddr = 0; 383 let has_glc = 0; 384 let has_dlc = 0; 385 let glc_value = 0; 386 let dlc_value = 0; 387 let has_srsrc = 0; 388 let has_soffset = 0; 389 let has_offset = 0; 390 let has_slc = 0; 391 let has_tfe = 0; 392 let has_sccb = 0; 393 let sccb_value = 0; 394} 395 396class getMUBUFInsDA<list<RegisterClass> vdataList, 397 list<RegisterClass> vaddrList=[], 398 bit isLds = 0> { 399 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 400 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 401 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 402 dag InsNoData = !if(!empty(vaddrList), 403 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 404 offset:$offset, CPol_0:$cpol), 405 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 406 offset:$offset, CPol_0:$cpol) 407 ); 408 dag InsData = !if(!empty(vaddrList), 409 (ins vdata_op:$vdata, SReg_128:$srsrc, 410 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 411 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 412 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 413 ); 414 dag ret = !con( 415 !if(!empty(vdataList), InsNoData, InsData), 416 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 417 ); 418} 419 420class getMUBUFElements<ValueType vt> { 421 // eq does not support ValueType for some reason. 422 string vtAsStr = !cast<string>(vt); 423 424 int ret = 425 !if(!eq(vtAsStr, "f16"), 1, 426 !if(!eq(vtAsStr, "v2f16"), 2, 427 !if(!eq(vtAsStr, "v3f16"), 3, 428 !if(!eq(vtAsStr, "v4f16"), 4, 429 !if(!eq(vt.Size, 32), 1, 430 !if(!eq(vt.Size, 64), 2, 431 !if(!eq(vt.Size, 96), 3, 432 !if(!eq(vt.Size, 128), 4, 0) 433 ) 434 ) 435 ) 436 ) 437 ) 438 ) 439 ); 440} 441 442class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 443 dag ret = 444 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 445 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 446 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 447 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 448 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 449 (ins)))))); 450} 451 452class getMUBUFAsmOps<int addrKind> { 453 string Pfx = 454 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 455 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 456 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 457 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 458 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 459 ""))))); 460 string ret = Pfx # "$offset"; 461} 462 463class MUBUF_SetupAddr<int addrKind> { 464 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 465 !eq(addrKind, BUFAddrKind.BothEn)); 466 467 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 468 !eq(addrKind, BUFAddrKind.BothEn)); 469 470 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 471 472 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 473} 474 475class MUBUF_Load_Pseudo <string opName, 476 int addrKind, 477 ValueType vdata_vt, 478 bit HasTiedDest = 0, 479 bit isLds = 0, 480 list<dag> pattern=[], 481 // Workaround bug bz30254 482 int addrKindCopy = addrKind, 483 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 484 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 485 : MUBUF_Pseudo<opName, 486 (outs vdata_op:$vdata), 487 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 488 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 489 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 490 !if(isLds, " lds", "$tfe") # "$swz", 491 pattern>, 492 MUBUF_SetupAddr<addrKindCopy> { 493 let PseudoInstr = opName # !if(isLds, "_lds", "") # 494 "_" # getAddrName<addrKindCopy>.ret; 495 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 496 497 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 498 let mayLoad = 1; 499 let mayStore = 0; 500 let maybeAtomic = 1; 501 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 502 let has_tfe = !not(isLds); 503 let lds = isLds; 504 let elements = getMUBUFElements<vdata_vt>.ret; 505} 506 507class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 508 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 509 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 510>; 511 512class MUBUF_Addr64_Load_Pat <Instruction inst, 513 ValueType load_vt = i32, 514 SDPatternOperator ld = null_frag> : Pat < 515 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 516 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 517>; 518 519multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 520 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 521 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 522} 523 524 525// FIXME: tfe can't be an operand because it requires a separate 526// opcode because it needs an N+1 register class dest register. 527multiclass MUBUF_Pseudo_Loads<string opName, 528 ValueType load_vt = i32, 529 SDPatternOperator ld = null_frag, 530 bit TiedDest = 0, 531 bit isLds = 0> { 532 533 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 534 535 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 536 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 537 538 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 539 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 540 541 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 542 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 543 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 544 545 let DisableWQM = 1 in { 546 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 547 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 548 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 549 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 550 } 551} 552 553multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 554 SDPatternOperator ld_nolds = null_frag, 555 SDPatternOperator ld_lds = null_frag> { 556 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 557 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 558} 559 560class MUBUF_Store_Pseudo <string opName, 561 int addrKind, 562 ValueType store_vt, 563 list<dag> pattern=[], 564 // Workaround bug bz30254 565 int addrKindCopy = addrKind> 566 : MUBUF_Pseudo<opName, 567 (outs), 568 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 569 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 570 pattern>, 571 MUBUF_SetupAddr<addrKindCopy> { 572 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 573 let mayLoad = 0; 574 let mayStore = 1; 575 let maybeAtomic = 1; 576 let elements = getMUBUFElements<store_vt>.ret; 577} 578 579multiclass MUBUF_Pseudo_Stores<string opName, 580 ValueType store_vt = i32, 581 SDPatternOperator st = null_frag> { 582 583 defvar legal_store_vt = !if(!eq(!cast<string>(store_vt), !cast<string>(v3f16)), v4f16, store_vt); 584 585 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 586 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 587 i16:$offset))]>, 588 MUBUFAddr64Table<0, NAME>; 589 590 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 591 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 592 i16:$offset))]>, 593 MUBUFAddr64Table<1, NAME>; 594 595 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 596 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 597 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 598 599 let DisableWQM = 1 in { 600 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 601 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 602 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 603 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 604 } 605} 606 607class MUBUF_Pseudo_Store_Lds<string opName> 608 : MUBUF_Pseudo<opName, 609 (outs), 610 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 611 " $srsrc, $soffset$offset lds$cpol$swz"> { 612 let mayLoad = 0; 613 let mayStore = 1; 614 let maybeAtomic = 1; 615 616 let has_vdata = 0; 617 let has_vaddr = 0; 618 let has_tfe = 0; 619 let lds = 1; 620 621 let Uses = [EXEC, M0]; 622 let AsmMatchConverter = "cvtMubufLds"; 623} 624 625class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 626 list<RegisterClass> vaddrList=[]> { 627 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 628 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 629 dag ret = !if(vdata_in, 630 !if(!empty(vaddrList), 631 (ins vdata_op:$vdata_in, 632 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 633 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 634 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 635 ), 636 !if(!empty(vaddrList), 637 (ins vdata_op:$vdata, 638 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 639 (ins vdata_op:$vdata, vaddrClass:$vaddr, 640 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 641 )); 642} 643 644class getMUBUFAtomicIns<int addrKind, 645 RegisterClass vdataClass, 646 bit vdata_in, 647 // Workaround bug bz30254 648 RegisterClass vdataClassCopy=vdataClass> { 649 dag ret = 650 !if(!eq(addrKind, BUFAddrKind.Offset), 651 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 652 !if(!eq(addrKind, BUFAddrKind.OffEn), 653 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 654 !if(!eq(addrKind, BUFAddrKind.IdxEn), 655 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 656 !if(!eq(addrKind, BUFAddrKind.BothEn), 657 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 658 !if(!eq(addrKind, BUFAddrKind.Addr64), 659 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 660 (ins)))))); 661} 662 663class MUBUF_Atomic_Pseudo<string opName, 664 int addrKind, 665 dag outs, 666 dag ins, 667 string asmOps, 668 list<dag> pattern=[], 669 // Workaround bug bz30254 670 int addrKindCopy = addrKind> 671 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 672 MUBUF_SetupAddr<addrKindCopy> { 673 let mayStore = 1; 674 let mayLoad = 1; 675 let hasPostISelHook = 1; 676 let hasSideEffects = 1; 677 let DisableWQM = 1; 678 let has_glc = 0; 679 let has_dlc = 0; 680 let has_tfe = 0; 681 let has_sccb = 1; 682 let maybeAtomic = 1; 683 let AsmMatchConverter = "cvtMubufAtomic"; 684} 685 686class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 687 RegisterClass vdataClass, 688 list<dag> pattern=[], 689 // Workaround bug bz30254 690 int addrKindCopy = addrKind, 691 RegisterClass vdataClassCopy = vdataClass> 692 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 693 (outs), 694 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 695 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 696 pattern>, 697 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 698 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 699 let glc_value = 0; 700 let dlc_value = 0; 701 let sccb_value = 0; 702 let IsAtomicNoRet = 1; 703} 704 705class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 706 RegisterClass vdataClass, 707 list<dag> pattern=[], 708 // Workaround bug bz30254 709 int addrKindCopy = addrKind, 710 RegisterClass vdataClassCopy = vdataClass, 711 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 712 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 713 (outs vdata_op:$vdata), 714 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 715 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 716 pattern>, 717 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 718 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 719 let glc_value = 1; 720 let dlc_value = 0; 721 let sccb_value = 0; 722 let IsAtomicRet = 1; 723 let Constraints = "$vdata = $vdata_in"; 724 let DisableEncoding = "$vdata_in"; 725} 726 727multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 728 RegisterClass vdataClass, 729 ValueType vdataType, 730 SDPatternOperator atomic, 731 bit isFP = isFloatType<vdataType>.ret> { 732 let FPAtomic = isFP in 733 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 734 MUBUFAddr64Table <0, NAME>; 735 736 let FPAtomic = isFP in 737 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 738 MUBUFAddr64Table <1, NAME>; 739 740 let FPAtomic = isFP in 741 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 742 743 let FPAtomic = isFP in 744 745 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 746 747 let FPAtomic = isFP in 748 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 749} 750 751multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 752 RegisterClass vdataClass, 753 ValueType vdataType, 754 SDPatternOperator atomic, 755 bit isFP = isFloatType<vdataType>.ret> { 756 let FPAtomic = isFP in 757 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 758 [(set vdataType:$vdata, 759 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 760 vdataType:$vdata_in))]>, 761 MUBUFAddr64Table <0, NAME # "_RTN">; 762 763 let FPAtomic = isFP in 764 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 765 [(set vdataType:$vdata, 766 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 767 vdataType:$vdata_in))]>, 768 MUBUFAddr64Table <1, NAME # "_RTN">; 769 770 let FPAtomic = isFP in 771 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 772 773 let FPAtomic = isFP in 774 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 775 776 let FPAtomic = isFP in 777 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 778} 779 780multiclass MUBUF_Pseudo_Atomics <string opName, 781 RegisterClass vdataClass, 782 ValueType vdataType, 783 SDPatternOperator atomic> : 784 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 785 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 786 787 788//===----------------------------------------------------------------------===// 789// MUBUF Instructions 790//===----------------------------------------------------------------------===// 791 792defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 793 "buffer_load_format_x", f32 794>; 795defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 796 "buffer_load_format_xy", v2f32 797>; 798defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 799 "buffer_load_format_xyz", v3f32 800>; 801defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 802 "buffer_load_format_xyzw", v4f32 803>; 804defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 805 "buffer_store_format_x", f32 806>; 807defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 808 "buffer_store_format_xy", v2f32 809>; 810defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 811 "buffer_store_format_xyz", v3f32 812>; 813defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 814 "buffer_store_format_xyzw", v4f32 815>; 816 817let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 818 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 819 "buffer_load_format_d16_x", i32 820 >; 821 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 822 "buffer_load_format_d16_xy", v2i32 823 >; 824 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 825 "buffer_load_format_d16_xyz", v3i32 826 >; 827 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 828 "buffer_load_format_d16_xyzw", v4i32 829 >; 830 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 831 "buffer_store_format_d16_x", i32 832 >; 833 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 834 "buffer_store_format_d16_xy", v2i32 835 >; 836 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 837 "buffer_store_format_d16_xyz", v3i32 838 >; 839 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 840 "buffer_store_format_d16_xyzw", v4i32 841 >; 842} // End HasUnpackedD16VMem. 843 844let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 845 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 846 "buffer_load_format_d16_x", f16 847 >; 848 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 849 "buffer_load_format_d16_xy", v2f16 850 >; 851 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 852 "buffer_load_format_d16_xyz", v3f16 853 >; 854 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 855 "buffer_load_format_d16_xyzw", v4f16 856 >; 857 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 858 "buffer_store_format_d16_x", f16 859 >; 860 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 861 "buffer_store_format_d16_xy", v2f16 862 >; 863 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 864 "buffer_store_format_d16_xyz", v3f16 865 >; 866 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 867 "buffer_store_format_d16_xyzw", v4f16 868 >; 869} // End HasPackedD16VMem. 870 871defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 872 "buffer_load_ubyte", i32 873>; 874defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 875 "buffer_load_sbyte", i32 876>; 877defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 878 "buffer_load_ushort", i32 879>; 880defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 881 "buffer_load_sshort", i32 882>; 883defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 884 "buffer_load_dword", i32 885>; 886defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 887 "buffer_load_dwordx2", v2i32 888>; 889defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 890 "buffer_load_dwordx3", v3i32 891>; 892defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 893 "buffer_load_dwordx4", v4i32 894>; 895 896defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 897defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 898defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 899defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 900defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 901defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 902defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 903defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 904defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 905defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 906 907// This is not described in AMD documentation, 908// but 'lds' versions of these opcodes are available 909// in at least GFX8+ chips. See Bug 37653. 910let SubtargetPredicate = isGFX8GFX9 in { 911defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 912 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 913>; 914defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 915 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 916>; 917defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 918 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 919>; 920} 921 922defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 923 "buffer_store_byte", i32, truncstorei8_global 924>; 925defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 926 "buffer_store_short", i32, truncstorei16_global 927>; 928defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 929 "buffer_store_dword", i32, store_global 930>; 931defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 932 "buffer_store_dwordx2", v2i32, store_global 933>; 934defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 935 "buffer_store_dwordx3", v3i32, store_global 936>; 937defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 938 "buffer_store_dwordx4", v4i32, store_global 939>; 940defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 941 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 942>; 943defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 944 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 945>; 946defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 947 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 948>; 949defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 950 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 951>; 952defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 953 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 954>; 955defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 957>; 958defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 960>; 961defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 963>; 964defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 966>; 967defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 969>; 970defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 972>; 973defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 975>; 976defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 978>; 979defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 981>; 982defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 984>; 985defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 987>; 988defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 990>; 991defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 993>; 994defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 996>; 997defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 999>; 1000defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 1002>; 1003defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 1005>; 1006defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 1008>; 1009defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 1011>; 1012defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1014>; 1015defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1017>; 1018 1019let SubtargetPredicate = HasGFX10_BEncoding in 1020defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1021 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1022>; 1023 1024let SubtargetPredicate = isGFX8GFX9 in { 1025def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1026} 1027 1028let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1029/* 1030defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1031defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1032*/ 1033 1034def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1035 int_amdgcn_buffer_wbinvl1_sc>; 1036} 1037 1038let SubtargetPredicate = isGFX6GFX7GFX10 in { 1039 1040defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1041 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1042>; 1043defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1044 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1045>; 1046defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1047 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1048>; 1049defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1050 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1051>; 1052defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1053 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1054>; 1055defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1056 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1057>; 1058 1059} 1060 1061let SubtargetPredicate = HasD16LoadStore in { 1062 1063defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1064 "buffer_load_ubyte_d16", i32, null_frag, 1 1065>; 1066 1067defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1068 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1069>; 1070 1071defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1072 "buffer_load_sbyte_d16", i32, null_frag, 1 1073>; 1074 1075defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1076 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1077>; 1078 1079defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1080 "buffer_load_short_d16", i32, null_frag, 1 1081>; 1082 1083defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1084 "buffer_load_short_d16_hi", i32, null_frag, 1 1085>; 1086 1087defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1088 "buffer_store_byte_d16_hi", i32 1089>; 1090 1091defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1092 "buffer_store_short_d16_hi", i32 1093>; 1094 1095defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1096 "buffer_load_format_d16_hi_x", i32 1097>; 1098defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1099 "buffer_store_format_d16_hi_x", i32 1100>; 1101 1102} // End HasD16LoadStore 1103 1104def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1105 int_amdgcn_buffer_wbinvl1>; 1106 1107let SubtargetPredicate = HasAtomicFaddInsts in { 1108defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1109 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_noret_32 1110>; 1111defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1112 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_noret_32 1113>; 1114 1115let OtherPredicates = [isGFX90APlus] in { 1116defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN < 1117 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1118>; 1119defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1120 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1121>; 1122} 1123} // End SubtargetPredicate = HasAtomicFaddInsts 1124 1125//===----------------------------------------------------------------------===// 1126// MTBUF Instructions 1127//===----------------------------------------------------------------------===// 1128 1129defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1130defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1131defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1132defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1133defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1134defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1135defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1136defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1137 1138let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1139 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1140 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1141 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1142 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1143 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1144 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1145 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1146 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1147} // End HasUnpackedD16VMem. 1148 1149let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1150 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1151 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1152 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1153 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1154 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1155 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1156 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1157 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1158} // End HasPackedD16VMem. 1159 1160let SubtargetPredicate = isGFX7Plus in { 1161 1162//===----------------------------------------------------------------------===// 1163// Instruction definitions for CI and newer. 1164//===----------------------------------------------------------------------===// 1165 1166def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1167 int_amdgcn_buffer_wbinvl1_vol>; 1168 1169} // End let SubtargetPredicate = isGFX7Plus 1170 1171let SubtargetPredicate = isGFX90APlus in { 1172 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1173 } 1174 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1175 } 1176 1177 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 1178 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 1179 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 1180} // End SubtargetPredicate = isGFX90APlus 1181 1182let SubtargetPredicate = isGFX10Plus in { 1183 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1184 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1185} // End SubtargetPredicate = isGFX10Plus 1186 1187//===----------------------------------------------------------------------===// 1188// MUBUF Patterns 1189//===----------------------------------------------------------------------===// 1190 1191//===----------------------------------------------------------------------===// 1192// buffer_load/store_format patterns 1193//===----------------------------------------------------------------------===// 1194 1195multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1196 string opcode, ValueType memoryVt = vt> { 1197 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_load<name, memoryVt>); 1198 1199 def : GCNPat< 1200 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1201 timm:$auxiliary, 0)), 1202 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1203 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1204 >; 1205 1206 def : GCNPat< 1207 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1208 timm:$auxiliary, 0)), 1209 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1210 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1211 >; 1212 1213 def : GCNPat< 1214 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1215 timm:$auxiliary, timm)), 1216 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1217 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1218 >; 1219 1220 def : GCNPat< 1221 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1222 timm:$auxiliary, timm)), 1223 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1224 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1225 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1226 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1227 >; 1228} 1229 1230defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1231defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1232defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1233defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1234defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1235defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1236defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1237defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1238 1239let SubtargetPredicate = HasUnpackedD16VMem in { 1240 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1241 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1242 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1243 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1244 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1245 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1246} // End HasUnpackedD16VMem. 1247 1248let SubtargetPredicate = HasPackedD16VMem in { 1249 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1250 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1251 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1252 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1253 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1254 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1255 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1256 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1257 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1258} // End HasPackedD16VMem. 1259 1260defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1261defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1262defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1263defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1264defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1265defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1266defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1267defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1268defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1269defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1270defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1271defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1272defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1273defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1274defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1275defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1276 1277multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1278 string opcode, ValueType memoryVt = vt> { 1279 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_store<name, memoryVt>); 1280 1281 def : GCNPat< 1282 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1283 timm:$auxiliary, 0), 1284 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1285 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1286 >; 1287 1288 def : GCNPat< 1289 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1290 timm:$auxiliary, 0), 1291 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1292 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1293 >; 1294 1295 def : GCNPat< 1296 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1297 timm:$auxiliary, timm), 1298 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1299 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1300 >; 1301 1302 def : GCNPat< 1303 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1304 timm:$auxiliary, timm), 1305 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1306 getVregSrcForVT<vt>.ret:$vdata, 1307 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1308 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1309 0, (extract_swz $auxiliary)) 1310 >; 1311} 1312 1313defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1314defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1315defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1316defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1317defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1318defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1319defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1320defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1321 1322let SubtargetPredicate = HasUnpackedD16VMem in { 1323 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1324 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1325 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1326 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1327 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1328 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1329} // End HasUnpackedD16VMem. 1330 1331let SubtargetPredicate = HasPackedD16VMem in { 1332 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1333 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1334 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1335 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1336 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1337 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1338 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1339 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1340 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1341} // End HasPackedD16VMem. 1342 1343defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1344defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1345defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1346defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1347defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1348defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1349defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1350defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1351defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1352defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1353defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1354defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1355defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1356defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1357 1358//===----------------------------------------------------------------------===// 1359// buffer_atomic patterns 1360//===----------------------------------------------------------------------===// 1361 1362multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1363 string opcode> { 1364 def : GCNPat< 1365 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1366 timm:$offset, timm:$cachepolicy, 0)), 1367 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1368 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1369 (as_i16timm $offset), (set_glc $cachepolicy)) 1370 >; 1371 1372 def : GCNPat< 1373 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1374 timm:$offset, timm:$cachepolicy, timm)), 1375 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1376 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1377 (as_i16timm $offset), (set_glc $cachepolicy)) 1378 >; 1379 1380 def : GCNPat< 1381 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1382 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1383 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1384 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1385 (as_i16timm $offset), (set_glc $cachepolicy)) 1386 >; 1387 1388 def : GCNPat< 1389 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1390 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1391 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1392 getVregSrcForVT<vt>.ret:$vdata_in, 1393 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1394 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1395 (set_glc $cachepolicy)) 1396 >; 1397} 1398 1399defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1400defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">; 1401defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1402defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1403defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1404defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1405defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1406defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1407defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1408defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1409defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1410defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1411defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1412defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1413defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1414defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1415defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1416defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1417defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1418defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1419defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1420defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1421defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1422defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1423defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1424defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1425 1426class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1427 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1428 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1429 [{ return SDValue(N, 0).use_empty(); }]> { 1430 1431 let GISelPredicateCode = [{ 1432 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1433 }]; 1434} 1435 1436multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1437 string opcode> { 1438 def : GCNPat< 1439 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1440 0, i32:$soffset, timm:$offset, 1441 timm:$cachepolicy, 0), 1442 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1443 (as_i16timm $offset), $cachepolicy) 1444 >; 1445 1446 def : GCNPat< 1447 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1448 0, i32:$soffset, timm:$offset, 1449 timm:$cachepolicy, timm), 1450 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1451 (as_i16timm $offset), $cachepolicy) 1452 >; 1453 1454 def : GCNPat< 1455 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1456 i32:$voffset, i32:$soffset, timm:$offset, 1457 timm:$cachepolicy, 0), 1458 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1459 (as_i16timm $offset), $cachepolicy) 1460 >; 1461 1462 def : GCNPat< 1463 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1464 i32:$voffset, i32:$soffset, timm:$offset, 1465 timm:$cachepolicy, timm), 1466 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1467 getVregSrcForVT<vt>.ret:$vdata_in, 1468 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1469 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1470 >; 1471} 1472 1473let SubtargetPredicate = HasAtomicFaddInsts in { 1474defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1475defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1476} 1477 1478let SubtargetPredicate = isGFX90APlus in { 1479 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1480 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1481 1482 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f64, "BUFFER_ATOMIC_ADD_F64">; 1483 defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f64, "BUFFER_ATOMIC_MIN_F64">; 1484 defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f64, "BUFFER_ATOMIC_MAX_F64">; 1485} // End SubtargetPredicate = isGFX90APlus 1486 1487def : GCNPat< 1488 (SIbuffer_atomic_cmpswap 1489 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1490 timm:$offset, timm:$cachepolicy, 0), 1491 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1492 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1493 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1494 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1495 (set_glc $cachepolicy)), VReg_64)), sub0) 1496>; 1497 1498def : GCNPat< 1499 (SIbuffer_atomic_cmpswap 1500 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1501 0, i32:$soffset, timm:$offset, 1502 timm:$cachepolicy, timm), 1503 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1504 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1505 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1506 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1507 (set_glc $cachepolicy)), VReg_64)), 1508 sub0) 1509>; 1510 1511def : GCNPat< 1512 (SIbuffer_atomic_cmpswap 1513 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1514 i32:$voffset, i32:$soffset, timm:$offset, 1515 timm:$cachepolicy, 0), 1516 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1517 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1518 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1519 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1520 (set_glc $cachepolicy)), VReg_64)), 1521 sub0) 1522>; 1523 1524def : GCNPat< 1525 (SIbuffer_atomic_cmpswap 1526 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1527 i32:$voffset, i32:$soffset, timm:$offset, 1528 timm:$cachepolicy, timm), 1529 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1530 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1531 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1532 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1533 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1534 (set_glc $cachepolicy)), VReg_64)), 1535 sub0) 1536>; 1537 1538class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1539 PatFrag constant_ld> : GCNPat < 1540 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1541 i16:$offset))), 1542 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1543 >; 1544 1545multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1546 ValueType vt, PatFrag atomic_ld> { 1547 def : GCNPat < 1548 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1549 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1550 >; 1551 1552 def : GCNPat < 1553 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1554 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1555 >; 1556} 1557 1558let SubtargetPredicate = isGFX6GFX7 in { 1559def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1560def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1561def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1562def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1563def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1564def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1565 1566defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1567defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1568} // End SubtargetPredicate = isGFX6GFX7 1569 1570multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1571 PatFrag ld> { 1572 1573 def : GCNPat < 1574 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1575 (Instr_OFFSET $srsrc, $soffset, $offset) 1576 >; 1577} 1578 1579let OtherPredicates = [Has16BitInsts] in { 1580 1581defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1582defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1583defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1584defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1585defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1586defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1587 1588defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1589 1590} // End OtherPredicates = [Has16BitInsts] 1591 1592multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1593 MUBUF_Pseudo InstrOffset, 1594 ValueType vt, PatFrag ld> { 1595 def : GCNPat < 1596 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1597 i32:$soffset, u16imm:$offset))), 1598 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1599 >; 1600 1601 def : GCNPat < 1602 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1603 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1604 >; 1605} 1606 1607// XXX - Is it possible to have a complex pattern in a PatFrag? 1608multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1609 MUBUF_Pseudo InstrOffset, 1610 ValueType vt, PatFrag ld_frag> { 1611 def : GCNPat < 1612 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1613 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1614 >; 1615 1616 def : GCNPat < 1617 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1618 (InstrOffset $srsrc, $soffset, $offset, $in) 1619 >; 1620} 1621 1622let OtherPredicates = [DisableFlatScratch] in { 1623defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1624defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1625defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1626defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1627defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1628defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1629defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1630defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1631defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1632defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1633 1634foreach vt = Reg32Types.types in { 1635defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1636} 1637defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1638defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1639defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1640 1641let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1642defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1643defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1644defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1645defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1646defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1647defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1648 1649defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1650defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1651defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1652defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1653defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1654defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1655} 1656 1657} // End OtherPredicates = [DisableFlatScratch] 1658 1659multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1660 ValueType vt, PatFrag atomic_st> { 1661 // Store follows atomic op convention so address is first 1662 def : GCNPat < 1663 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1664 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1665 >; 1666 1667 def : GCNPat < 1668 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1669 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1670 >; 1671} 1672let SubtargetPredicate = isGFX6GFX7 in { 1673defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1674defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1675} // End Predicates = isGFX6GFX7 1676 1677 1678multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1679 PatFrag st> { 1680 1681 def : GCNPat < 1682 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1683 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1684 >; 1685} 1686 1687defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1688defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1689 1690multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1691 MUBUF_Pseudo InstrOffset, 1692 ValueType vt, PatFrag st, 1693 RegisterClass rc = VGPR_32> { 1694 def : GCNPat < 1695 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1696 i32:$soffset, u16imm:$offset)), 1697 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1698 >; 1699 1700 def : GCNPat < 1701 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1702 u16imm:$offset)), 1703 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1704 >; 1705} 1706 1707let OtherPredicates = [DisableFlatScratch] in { 1708defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1709defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1710defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1711defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1712 1713foreach vt = Reg32Types.types in { 1714defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1715} 1716 1717defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1718defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1719defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1720 1721 1722let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1723 // Hiding the extract high pattern in the PatFrag seems to not 1724 // automatically increase the complexity. 1725let AddedComplexity = 1 in { 1726defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1727defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1728} 1729} 1730} // End OtherPredicates = [DisableFlatScratch] 1731 1732//===----------------------------------------------------------------------===// 1733// MTBUF Patterns 1734//===----------------------------------------------------------------------===// 1735 1736//===----------------------------------------------------------------------===// 1737// tbuffer_load/store_format patterns 1738//===----------------------------------------------------------------------===// 1739 1740multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1741 string opcode, ValueType memoryVt = vt> { 1742 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_load<name, memoryVt>); 1743 1744 def : GCNPat< 1745 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1746 timm:$format, timm:$auxiliary, 0)), 1747 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1748 (as_i8timm $format), 1749 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1750 >; 1751 1752 def : GCNPat< 1753 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1754 timm:$format, timm:$auxiliary, timm)), 1755 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1756 (as_i8timm $format), 1757 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1758 >; 1759 1760 def : GCNPat< 1761 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1762 timm:$format, timm:$auxiliary, 0)), 1763 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1764 (as_i8timm $format), 1765 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1766 >; 1767 1768 def : GCNPat< 1769 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1770 timm:$format, timm:$auxiliary, timm)), 1771 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1772 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1773 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1774 (as_i8timm $format), 1775 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1776 >; 1777} 1778 1779defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1780defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1781defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1782defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1783defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1784defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1785defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1786defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1787 1788let SubtargetPredicate = HasUnpackedD16VMem in { 1789 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1790 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1791 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1792 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1793 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1794} // End HasUnpackedD16VMem. 1795 1796let SubtargetPredicate = HasPackedD16VMem in { 1797 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1798 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1799 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1800 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1801 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1802} // End HasPackedD16VMem. 1803 1804multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1805 string opcode, ValueType memoryVt = vt> { 1806 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_store<name, memoryVt>); 1807 1808 def : GCNPat< 1809 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1810 timm:$format, timm:$auxiliary, 0), 1811 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1812 (as_i16timm $offset), (as_i8timm $format), 1813 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1814 >; 1815 1816 def : GCNPat< 1817 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1818 timm:$format, timm:$auxiliary, timm), 1819 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1820 (as_i16timm $offset), (as_i8timm $format), 1821 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1822 >; 1823 1824 def : GCNPat< 1825 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1826 timm:$format, timm:$auxiliary, 0), 1827 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1828 (as_i16timm $offset), (as_i8timm $format), 1829 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1830 >; 1831 1832 def : GCNPat< 1833 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1834 timm:$offset, timm:$format, timm:$auxiliary, timm), 1835 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1836 getVregSrcForVT<vt>.ret:$vdata, 1837 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1838 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1839 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1840 >; 1841} 1842 1843defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1844defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1845defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1846defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1847defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1848defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1849defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1850defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1851 1852let SubtargetPredicate = HasUnpackedD16VMem in { 1853 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1854 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1855 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1856 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1857 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1858} // End HasUnpackedD16VMem. 1859 1860let SubtargetPredicate = HasPackedD16VMem in { 1861 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1862 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1863 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1864 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1865 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1866} // End HasPackedD16VMem. 1867 1868//===----------------------------------------------------------------------===// 1869// Target-specific instruction encodings. 1870//===----------------------------------------------------------------------===// 1871 1872//===----------------------------------------------------------------------===// 1873// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1874//===----------------------------------------------------------------------===// 1875 1876class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1877 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1878 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1879 let Inst{12} = ps.offen; 1880 let Inst{13} = ps.idxen; 1881 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 1882 let Inst{16} = ps.lds; 1883 let Inst{24-18} = op; 1884 let Inst{31-26} = 0x38; 1885 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1886 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 1887 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1888 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 1889 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1890 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1891} 1892 1893class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1894 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1895 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 1896 let Inst{25} = op{7}; 1897} 1898 1899class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1900 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1901 let Inst{15} = ps.addr64; 1902} 1903 1904//===----------------------------------------------------------------------===// 1905// MUBUF - GFX10. 1906//===----------------------------------------------------------------------===// 1907 1908let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1909 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1910 string asmName> { 1911 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1912 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1913 let AsmString = asmName # ps.AsmOperands; 1914 } 1915 } 1916 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1917 def _BOTHEN_gfx10 : 1918 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1919 def _IDXEN_gfx10 : 1920 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1921 def _OFFEN_gfx10 : 1922 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1923 def _OFFSET_gfx10 : 1924 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1925 } 1926 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1927 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1928 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1929 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1930 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1931 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1932 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1933 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1934 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1935 1936 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1937 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1938 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1939 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1940 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1941 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1942 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1943 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1944 } 1945 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1946 def _BOTHEN_RTN_gfx10 : 1947 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 1948 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 1949 def _IDXEN_RTN_gfx10 : 1950 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 1951 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 1952 def _OFFEN_RTN_gfx10 : 1953 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 1954 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 1955 def _OFFSET_RTN_gfx10 : 1956 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 1957 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 1958 } 1959 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1960 MUBUF_Real_Atomics_RTN_gfx10<op> { 1961 def _BOTHEN_gfx10 : 1962 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1963 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 1964 def _IDXEN_gfx10 : 1965 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1966 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 1967 def _OFFEN_gfx10 : 1968 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1969 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 1970 def _OFFSET_gfx10 : 1971 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1972 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 1973 } 1974} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1975 1976defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1977defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1978defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1979defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1980defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1981defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1982defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1983defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1984// FIXME-GFX10: Add following instructions: 1985//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1986//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1987defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1988defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1989defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1990defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1991defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1992defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1993defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1994defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1995 1996def BUFFER_GL0_INV_gfx10 : 1997 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1998def BUFFER_GL1_INV_gfx10 : 1999 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2000 2001//===----------------------------------------------------------------------===// 2002// MUBUF - GFX6, GFX7, GFX10. 2003//===----------------------------------------------------------------------===// 2004 2005let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2006 multiclass MUBUF_Real_gfx6<bits<8> op> { 2007 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2008 } 2009} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2010 2011let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2012 multiclass MUBUF_Real_gfx7<bits<8> op> { 2013 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2014 } 2015} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2016 2017let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2018 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2019 def _ADDR64_gfx6_gfx7 : 2020 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2021 def _BOTHEN_gfx6_gfx7 : 2022 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2023 def _IDXEN_gfx6_gfx7 : 2024 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2025 def _OFFEN_gfx6_gfx7 : 2026 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2027 def _OFFSET_gfx6_gfx7 : 2028 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2029 } 2030 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2031 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2032 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2033 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2034 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2035 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2036 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2037 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2038 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2039 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2040 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2041 2042 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2043 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2044 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2045 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2046 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2047 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2048 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2049 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2050 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2051 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2052 } 2053 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2054 def _ADDR64_gfx6_gfx7 : 2055 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2056 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2057 def _BOTHEN_gfx6_gfx7 : 2058 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2059 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2060 def _IDXEN_gfx6_gfx7 : 2061 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2062 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2063 def _OFFEN_gfx6_gfx7 : 2064 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2065 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2066 def _OFFSET_gfx6_gfx7 : 2067 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2068 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2069 2070 def _ADDR64_RTN_gfx6_gfx7 : 2071 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2072 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2073 def _BOTHEN_RTN_gfx6_gfx7 : 2074 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2075 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2076 def _IDXEN_RTN_gfx6_gfx7 : 2077 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2078 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2079 def _OFFEN_RTN_gfx6_gfx7 : 2080 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2081 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2082 def _OFFSET_RTN_gfx6_gfx7 : 2083 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2084 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2085 } 2086} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2087 2088multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2089 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2090 2091multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2092 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2093 2094multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2095 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2096 2097// FIXME-GFX6: Following instructions are available only on GFX6. 2098//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2099//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2100 2101defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2102defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2103defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2104defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2105defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2106defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2107defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2108defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2109defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2110defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2111defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2112defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2113defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2114defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2115defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2116defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2117defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2118defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2119defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2120defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2121defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2122defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2123 2124defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2125defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2126defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2127defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2128defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2129defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2130defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2131defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2132defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2133defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2134defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2135defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2136defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2137defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2138defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2139defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2140defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2141defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2142defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2143defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2144defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2145defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2146defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2147defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2148defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2149defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2150defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2151defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2152defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2153// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2154defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2155defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2156defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2157 2158defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2159 2160defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2161defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2162def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2163 2164//===----------------------------------------------------------------------===// 2165// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2166//===----------------------------------------------------------------------===// 2167 2168class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2169 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2170 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2171 let Inst{12} = ps.offen; 2172 let Inst{13} = ps.idxen; 2173 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2174 let Inst{18-16} = op; 2175 let Inst{31-26} = 0x3a; //encoding 2176 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2177 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2178 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2179 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2180 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2181 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2182} 2183 2184//===----------------------------------------------------------------------===// 2185// MTBUF - GFX10. 2186//===----------------------------------------------------------------------===// 2187 2188class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2189 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2190 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2191 let Inst{25-19} = format; 2192 let Inst{53} = op{3}; 2193} 2194 2195let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2196 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2197 def _BOTHEN_gfx10 : 2198 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2199 def _IDXEN_gfx10 : 2200 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2201 def _OFFEN_gfx10 : 2202 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2203 def _OFFSET_gfx10 : 2204 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2205 } 2206} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2207 2208defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2209defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2210defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2211defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2212defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2213defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2214defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2215defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2216 2217//===----------------------------------------------------------------------===// 2218// MTBUF - GFX6, GFX7, GFX10. 2219//===----------------------------------------------------------------------===// 2220 2221class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2222 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2223 let Inst{15} = ps.addr64; 2224 let Inst{22-19} = dfmt; 2225 let Inst{25-23} = nfmt; 2226} 2227 2228let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2229 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2230 def _ADDR64_gfx6_gfx7 : 2231 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2232 def _BOTHEN_gfx6_gfx7 : 2233 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2234 def _IDXEN_gfx6_gfx7 : 2235 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2236 def _OFFEN_gfx6_gfx7 : 2237 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2238 def _OFFSET_gfx6_gfx7 : 2239 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2240 } 2241} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2242 2243multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2244 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2245 2246defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2247defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2248defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2249defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2250defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2251defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2252defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2253defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2254 2255//===----------------------------------------------------------------------===// 2256// GFX8, GFX9 (VI). 2257//===----------------------------------------------------------------------===// 2258 2259class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2260 bit has_sccb = ps.has_sccb> : 2261 MUBUF_Real<ps>, 2262 Enc64, 2263 SIMCInstr<ps.PseudoInstr, Enc>, 2264 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2265 !if(ps.IsAtomicRet, 1, ?))> { 2266 2267 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2268 let Inst{12} = ps.offen; 2269 let Inst{13} = ps.idxen; 2270 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2271 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2272 let Inst{16} = ps.lds; 2273 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2274 let Inst{24-18} = op; 2275 let Inst{31-26} = 0x38; //encoding 2276 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2277 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2278 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2279 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2280} 2281 2282class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2283 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2284 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2285 let DecoderNamespace = "GFX8"; 2286 2287 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2288} 2289 2290class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2291 bit has_sccb = ps.has_sccb> : 2292 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2293 let AssemblerPredicate = isGFX90APlus; 2294 let DecoderNamespace = "GFX90A"; 2295 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2296 !subst("$tfe", "", ps.AsmOperands)); 2297 2298 let Inst{55} = acc; 2299} 2300 2301multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2302 def _vi : MUBUF_Real_vi<op, ps>; 2303 def _gfx90a : MUBUF_Real_gfx90a<op, ps, !and(ps.has_sccb,!not(ps.FPAtomic))>; 2304} 2305 2306multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2307 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2308 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2309 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2310 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2311} 2312 2313multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2314 2315 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2316 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2317 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2318 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2319 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2320 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2321 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2322 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2323 2324 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2325 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2326 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2327 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2328 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2329 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2330 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2331 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2332 2333 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2334 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2335 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2336 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2337 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2338 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2339 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2340 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2341 2342 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2343 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2344 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2345 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2346 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2347 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2348 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2349 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2350} 2351 2352class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2353 MUBUF_Real<ps>, 2354 Enc64, 2355 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2356 let AssemblerPredicate=HasUnpackedD16VMem; 2357 let DecoderNamespace="GFX80_UNPACKED"; 2358 2359 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2360 let Inst{12} = ps.offen; 2361 let Inst{13} = ps.idxen; 2362 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2363 let Inst{16} = ps.lds; 2364 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2365 let Inst{24-18} = op; 2366 let Inst{31-26} = 0x38; //encoding 2367 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2368 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2369 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2370 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2371 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2372} 2373 2374multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2375 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2376 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2377 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2378 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2379} 2380 2381multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2382 MUBUF_Real_AllAddr_vi<op> { 2383 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2384 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2385 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2386 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2387} 2388 2389defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2390defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2391defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2392defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2393defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2394defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2395defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2396defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2397let SubtargetPredicate = HasUnpackedD16VMem in { 2398 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2399 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2400 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2401 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2402 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2403 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2404 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2405 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2406} // End HasUnpackedD16VMem. 2407let SubtargetPredicate = HasPackedD16VMem in { 2408 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2409 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2410 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2411 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2412 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2413 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2414 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2415 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2416} // End HasPackedD16VMem. 2417defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2418defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2419defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2420defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2421defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2422defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2423defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2424defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2425defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2426defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2427defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2428defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2429defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2430defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2431defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2432defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2433 2434defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2435defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2436defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2437defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2438defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2439defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2440 2441defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2442defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2443 2444defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2445defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2446defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2447defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2448defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2449defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2450defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2451defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2452defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2453defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2454defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2455defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2456defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2457 2458defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2459defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2460defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2461defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2462defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2463defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2464defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2465defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2466defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2467defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2468defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2469defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2470defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2471 2472defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2473 2474let AssemblerPredicate = isGFX8GFX9 in { 2475def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2476def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2477} // End AssemblerPredicate = isGFX8GFX9 2478 2479let SubtargetPredicate = HasAtomicFaddInsts in { 2480 2481defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2482defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2483 2484} // End SubtargetPredicate = HasAtomicFaddInsts 2485 2486let SubtargetPredicate = isGFX90APlus in { 2487 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2488 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2489 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2490} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2491 2492def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2493} 2494def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2495 2496class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2497 MTBUF_Real<ps>, 2498 Enc64, 2499 SIMCInstr<ps.PseudoInstr, Enc> { 2500 2501 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2502 let Inst{12} = ps.offen; 2503 let Inst{13} = ps.idxen; 2504 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2505 let Inst{18-15} = op; 2506 let Inst{22-19} = dfmt; 2507 let Inst{25-23} = nfmt; 2508 let Inst{31-26} = 0x3a; //encoding 2509 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2510 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2511 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2512 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2513 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2514 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2515 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2516} 2517 2518class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2519 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2520 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2521 let DecoderNamespace = "GFX8"; 2522 2523 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2524} 2525 2526class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2527 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2528 let AssemblerPredicate = isGFX90APlus; 2529 let DecoderNamespace = "GFX90A"; 2530 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2531 2532 let Inst{55} = acc; 2533} 2534 2535multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2536 def _vi : MTBUF_Real_vi<op, ps>; 2537 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2538} 2539 2540multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2541 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2542 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2543 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2544 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2545} 2546 2547class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2548 MTBUF_Real<ps>, 2549 Enc64, 2550 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2551 let AssemblerPredicate=HasUnpackedD16VMem; 2552 let DecoderNamespace="GFX80_UNPACKED"; 2553 2554 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2555 let Inst{12} = ps.offen; 2556 let Inst{13} = ps.idxen; 2557 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2558 let Inst{18-15} = op; 2559 let Inst{22-19} = dfmt; 2560 let Inst{25-23} = nfmt; 2561 let Inst{31-26} = 0x3a; //encoding 2562 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2563 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2564 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2565 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2566 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2567 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2568} 2569 2570multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2571 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2572 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2573 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2574 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2575} 2576 2577defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2578defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2579defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2580defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2581defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2582defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2583defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2584defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2585let SubtargetPredicate = HasUnpackedD16VMem in { 2586 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2587 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2588 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2589 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2590 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2591 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2592 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2593 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2594} // End HasUnpackedD16VMem. 2595let SubtargetPredicate = HasPackedD16VMem in { 2596 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2597 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2598 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2599 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2600 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2601 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2602 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2603 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2604} // End HasUnpackedD16VMem. 2605 2606def MUBUFInfoTable : GenericTable { 2607 let FilterClass = "MUBUF_Pseudo"; 2608 let CppTypeName = "MUBUFInfo"; 2609 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2610 2611 let PrimaryKey = ["Opcode"]; 2612 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2613} 2614 2615def getMUBUFInfoFromOpcode : SearchIndex { 2616 let Table = MUBUFInfoTable; 2617 let Key = ["Opcode"]; 2618} 2619 2620def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2621 let Table = MUBUFInfoTable; 2622 let Key = ["BaseOpcode", "elements"]; 2623} 2624 2625def MTBUFInfoTable : GenericTable { 2626 let FilterClass = "MTBUF_Pseudo"; 2627 let CppTypeName = "MTBUFInfo"; 2628 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2629 2630 let PrimaryKey = ["Opcode"]; 2631 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2632} 2633 2634def getMTBUFInfoFromOpcode : SearchIndex { 2635 let Table = MTBUFInfoTable; 2636 let Key = ["Opcode"]; 2637} 2638 2639def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2640 let Table = MTBUFInfoTable; 2641 let Key = ["BaseOpcode", "elements"]; 2642} 2643