1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14class SMEMOffset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21def smem_offset : SMEMOffset; 22 23def smem_offset_mod : SMEMOffset { 24 let PrintMethod = "printSMEMOffsetMod"; 25} 26 27//===----------------------------------------------------------------------===// 28// Scalar Memory classes 29//===----------------------------------------------------------------------===// 30 31class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 32 InstSI <outs, ins, "", pattern>, 33 SIMCInstr<opName, SIEncodingFamily.NONE> { 34 let isPseudo = 1; 35 let isCodeGenOnly = 1; 36 37 let LGKM_CNT = 1; 38 let SMRD = 1; 39 let mayStore = 0; 40 let mayLoad = 1; 41 let hasSideEffects = 0; 42 let UseNamedOperandTable = 1; 43 let SchedRW = [WriteSMEM]; 44 45 string Mnemonic = opName; 46 string AsmOperands = asmOps; 47 48 bits<1> has_sbase = 1; 49 bits<1> has_sdst = 1; 50 bit has_glc = 0; 51 bit has_dlc = 0; 52 bit has_offset = 0; 53 bit has_soffset = 0; 54 bit is_buffer = 0; 55} 56 57class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic> 58 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> { 59 60 let isPseudo = 0; 61 let isCodeGenOnly = 0; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 // copy relevant pseudo op flags 66 let LGKM_CNT = ps.LGKM_CNT; 67 let SMRD = ps.SMRD; 68 let mayStore = ps.mayStore; 69 let mayLoad = ps.mayLoad; 70 let hasSideEffects = ps.hasSideEffects; 71 let UseNamedOperandTable = ps.UseNamedOperandTable; 72 let SchedRW = ps.SchedRW; 73 let SubtargetPredicate = ps.SubtargetPredicate; 74 let AsmMatchConverter = ps.AsmMatchConverter; 75 let IsAtomicRet = ps.IsAtomicRet; 76 let IsAtomicNoRet = ps.IsAtomicNoRet; 77 78 let TSFlags = ps.TSFlags; 79 80 bit is_buffer = ps.is_buffer; 81 82 // encoding 83 bits<7> sbase; 84 bits<7> sdst; 85 bits<32> offset; 86 bits<8> soffset; 87 bits<5> cpol; 88} 89 90class OffsetMode<bit hasOffset, bit hasSOffset, string variant, 91 dag ins, string asm> { 92 bit HasOffset = hasOffset; 93 bit HasSOffset = hasSOffset; 94 string Variant = variant; 95 dag Ins = ins; 96 string Asm = asm; 97} 98 99def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins smem_offset:$offset), "$offset">; 100def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">; 101def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM", 102 (ins SReg_32:$soffset, smem_offset_mod:$offset), 103 "$soffset$offset">; 104 105class SM_Probe_Pseudo <string opName, string variant, RegisterClass baseClass, 106 dag offsets, string asmOffsets, 107 bit hasOffset, bit hasSOffset> 108 : SM_Pseudo<opName, (outs), 109 !con((ins i8imm:$sdata, baseClass:$sbase), offsets), 110 " $sdata, $sbase, " # asmOffsets> { 111 let mayLoad = 0; 112 let mayStore = 0; 113 let has_glc = 0; 114 let LGKM_CNT = 0; 115 let ScalarStore = 0; 116 let hasSideEffects = 1; 117 let has_offset = hasOffset; 118 let has_soffset = hasSOffset; 119 let PseudoInstr = opName # variant; 120} 121 122class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 123 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 124 RegisterClass BaseClass; 125 let mayLoad = 1; 126 let mayStore = 0; 127 let has_glc = 1; 128 let has_dlc = 1; 129} 130 131class SM_Store_Pseudo <string opName, RegisterClass baseClass, 132 RegisterClass srcClass, dag ins, string asmOps> 133 : SM_Pseudo<opName, (outs), ins, asmOps, []> { 134 RegisterClass BaseClass = baseClass; 135 RegisterClass SrcClass = srcClass; 136 let mayLoad = 0; 137 let mayStore = 1; 138 let has_glc = 1; 139 let has_dlc = 1; 140 let ScalarStore = 1; 141} 142 143class SM_Discard_Pseudo <string opName, string variant, dag offsets, 144 string asmOffsets, bit hasOffset, bit hasSOffset> 145 : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets), 146 " $sbase, " # asmOffsets> { 147 let mayLoad = 0; 148 let mayStore = 0; 149 let has_glc = 0; 150 let has_sdst = 0; 151 let ScalarStore = 0; 152 let hasSideEffects = 1; 153 let has_offset = hasOffset; 154 let has_soffset = hasSOffset; 155 let PseudoInstr = opName # variant; 156} 157 158multiclass SM_Pseudo_Loads<string opName, 159 RegisterClass baseClass, 160 RegisterClass dstClass> { 161 def _IMM : SM_Load_Pseudo <opName, 162 (outs dstClass:$sdst), 163 (ins baseClass:$sbase, i32imm:$offset, CPol:$cpol), 164 " $sdst, $sbase, $offset$cpol", []> { 165 let has_offset = 1; 166 let BaseClass = baseClass; 167 let PseudoInstr = opName # "_IMM"; 168 let has_glc = 1; 169 let has_dlc = 1; 170 } 171 172 def _SGPR : SM_Load_Pseudo <opName, 173 (outs dstClass:$sdst), 174 (ins baseClass:$sbase, SReg_32:$soffset, CPol:$cpol), 175 " $sdst, $sbase, $soffset$cpol", []> { 176 let has_soffset = 1; 177 let BaseClass = baseClass; 178 let PseudoInstr = opName # "_SGPR"; 179 let has_glc = 1; 180 let has_dlc = 1; 181 } 182 183 def _SGPR_IMM : SM_Load_Pseudo <opName, 184 (outs dstClass:$sdst), 185 (ins baseClass:$sbase, SReg_32:$soffset, 186 i32imm:$offset, CPol:$cpol), 187 " $sdst, $sbase, $soffset$offset$cpol", []> { 188 let has_offset = 1; 189 let has_soffset = 1; 190 let BaseClass = baseClass; 191 let PseudoInstr = opName # "_SGPR_IMM"; 192 let has_glc = 1; 193 let has_dlc = 1; 194 } 195} 196 197multiclass SM_Pseudo_Stores<string opName, 198 RegisterClass baseClass, 199 RegisterClass srcClass> { 200 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 201 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol), 202 " $sdata, $sbase, $offset$cpol"> { 203 let has_offset = 1; 204 let PseudoInstr = opName # "_IMM"; 205 } 206 207 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, 208 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPol:$cpol), 209 " $sdata, $sbase, $soffset$cpol"> { 210 let has_soffset = 1; 211 let PseudoInstr = opName # "_SGPR"; 212 } 213 214 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 215 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, i32imm:$offset, 216 CPol:$cpol), 217 " $sdata, $sbase, $soffset$offset$cpol"> { 218 let has_offset = 1; 219 let has_soffset = 1; 220 let PseudoInstr = opName # "_SGPR_IMM"; 221 } 222} 223 224multiclass SM_Pseudo_Discards<string opName> { 225 def _IMM : SM_Discard_Pseudo <opName, "_IMM", 226 (ins smem_offset:$offset), "$offset", 1, 0>; 227 def _SGPR : SM_Discard_Pseudo <opName, "_SGPR", 228 (ins SReg_32:$soffset), "$soffset", 0, 1>; 229 def _SGPR_IMM : SM_Discard_Pseudo <opName, "_SGPR_IMM", 230 (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>; 231} 232 233class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 234 opName, (outs SReg_64_XEXEC:$sdst), (ins), 235 " $sdst", [(set i64:$sdst, (node))]> { 236 let hasSideEffects = 1; 237 238 let mayStore = 0; 239 let mayLoad = 0; 240 let has_sbase = 0; 241} 242 243class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 244 opName, (outs), (ins), "", [(node)]> { 245 let hasSideEffects = 1; 246 let mayLoad = 0; 247 let mayStore = 0; 248 let has_sdst = 0; 249 let has_sbase = 0; 250} 251 252multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 253 def _IMM : SM_Probe_Pseudo <opName, "_IMM", baseClass, 254 (ins smem_offset:$offset), "$offset", 1, 0>; 255 def _SGPR : SM_Probe_Pseudo <opName, "_SGPR", baseClass, 256 (ins SReg_32:$soffset), "$soffset", 0, 1>; 257 def _SGPR_IMM : SM_Probe_Pseudo <opName, "_SGPR_IMM", baseClass, 258 (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>; 259} 260 261class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 262 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 263 " $sdst", [(set i32:$sdst, (node))]> { 264 let hasSideEffects = 1; 265 let mayStore = 0; 266 let mayLoad = 0; 267 let has_sbase = 0; 268} 269 270//===----------------------------------------------------------------------===// 271// Scalar Atomic Memory Classes 272//===----------------------------------------------------------------------===// 273 274class SM_Atomic_Pseudo <string opName, 275 dag outs, dag ins, string asmOps, bit isRet> 276 : SM_Pseudo<opName, outs, ins, asmOps, []> { 277 278 bit glc = isRet; 279 280 let mayLoad = 1; 281 let mayStore = 1; 282 let has_glc = 1; 283 let has_dlc = 1; 284 let has_soffset = 1; 285 286 // Should these be set? 287 let ScalarStore = 1; 288 let hasSideEffects = 1; 289 let maybeAtomic = 1; 290 291 let IsAtomicNoRet = !not(isRet); 292 let IsAtomicRet = isRet; 293 294 let AsmMatchConverter = "cvtSMEMAtomic"; 295} 296 297class SM_Pseudo_Atomic<string opName, 298 RegisterClass baseClass, 299 RegisterClass dataClass, 300 OffsetMode offsets, 301 bit isRet, 302 string opNameWithSuffix = 303 opName # offsets.Variant # !if(isRet, "_RTN", ""), 304 Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> : 305 SM_Atomic_Pseudo<opName, 306 !if(isRet, (outs dataClass:$sdst), (outs)), 307 !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins, 308 (ins CPolTy:$cpol)), 309 !if(isRet, " $sdst", " $sdata") # 310 ", $sbase, " # offsets.Asm # "$cpol", 311 isRet>, 312 AtomicNoRet <opNameWithSuffix, isRet> { 313 let has_offset = offsets.HasOffset; 314 let has_soffset = offsets.HasSOffset; 315 let PseudoInstr = opNameWithSuffix; 316 317 let Constraints = !if(isRet, "$sdst = $sdata", ""); 318 let DisableEncoding = !if(isRet, "$sdata", ""); 319} 320 321multiclass SM_Pseudo_Atomics<string opName, 322 RegisterClass baseClass, 323 RegisterClass dataClass> { 324 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>; 325 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>; 326 def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>; 327 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>; 328 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>; 329 def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>; 330} 331 332//===----------------------------------------------------------------------===// 333// Scalar Memory Instructions 334//===----------------------------------------------------------------------===// 335 336// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 337// SMRD instructions, because the SReg_32_XM0 register class does not include M0 338// and writing to M0 from an SMRD instruction will hang the GPU. 339 340// XXX - SMEM instructions do not allow exec for data operand, but 341// does sdst for SMRD on SI/CI? 342defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 343defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 344defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 345defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 346defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 347 348let is_buffer = 1 in { 349defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 350 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 351>; 352 353// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 354// SI/CI, bit disallowed for SMEM on VI. 355defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 356 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 357>; 358 359defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 360 "s_buffer_load_dwordx4", SReg_128, SReg_128 361>; 362 363defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 364 "s_buffer_load_dwordx8", SReg_128, SReg_256 365>; 366 367defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 368 "s_buffer_load_dwordx16", SReg_128, SReg_512 369>; 370} 371 372let SubtargetPredicate = HasScalarStores in { 373defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 374defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 375defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 376 377let is_buffer = 1 in { 378defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 379 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 380>; 381 382defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 383 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 384>; 385 386defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 387 "s_buffer_store_dwordx4", SReg_128, SReg_128 388>; 389} 390} // End SubtargetPredicate = HasScalarStores 391 392let SubtargetPredicate = HasSMemTimeInst in 393def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 394def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 395 396let SubtargetPredicate = isGFX7GFX8GFX9 in { 397def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 398} // let SubtargetPredicate = isGFX7GFX8GFX9 399 400let SubtargetPredicate = isGFX8Plus in { 401let OtherPredicates = [HasScalarStores] in { 402def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 403def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 404} // End OtherPredicates = [HasScalarStores] 405 406defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 407let is_buffer = 1 in { 408defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 409} 410} // SubtargetPredicate = isGFX8Plus 411 412let SubtargetPredicate = HasSMemRealTime in 413def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 414 415let SubtargetPredicate = isGFX10Plus in 416def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 417let SubtargetPredicate = HasGetWaveIdInst in 418def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 419 420 421let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 422defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 423defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 424defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 425 426defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 427defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 428defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 429} // SubtargetPredicate = HasScalarFlatScratchInsts 430 431let SubtargetPredicate = HasScalarAtomics in { 432 433let is_buffer = 1 in { 434defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 435defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 436defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 437defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 438defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 439defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 440defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 441defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 442defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 443defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 444defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 445defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 446defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 447 448defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 449defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 450defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 451defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 452defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 453defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 454defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 455defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 456defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 457defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 458defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 459defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 460defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 461} 462 463defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 464defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 465defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 466defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 467defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 468defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 469defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 470defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 471defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 472defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 473defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 474defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 475defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 476 477defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 478defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 479defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 480defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 481defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 482defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 483defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 484defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 485defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 486defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 487defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 488defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 489defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 490 491} // let SubtargetPredicate = HasScalarAtomics 492 493let SubtargetPredicate = HasScalarAtomics in { 494defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 495defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 496} 497 498//===----------------------------------------------------------------------===// 499// Targets 500//===----------------------------------------------------------------------===// 501 502//===----------------------------------------------------------------------===// 503// SI 504//===----------------------------------------------------------------------===// 505 506class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 507 : SM_Real<ps> 508 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 509 , Enc32 { 510 511 let AssemblerPredicate = isGFX6GFX7; 512 let DecoderNamespace = "GFX6GFX7"; 513 514 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 515 let Inst{8} = ps.has_offset; 516 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 517 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 518 let Inst{26-22} = op; 519 let Inst{31-27} = 0x18; //encoding 520} 521 522multiclass SM_Real_Loads_si<bits<5> op, string ps, 523 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 524 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 525 526 def _IMM_si : SMRD_Real_si <op, immPs> { 527 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 528 } 529 530 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 531 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 532 } 533 534} 535 536defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 537defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 538defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 539defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 540defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 541defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 542defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 543defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 544defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 545defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 546 547def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 548def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 549 550 551//===----------------------------------------------------------------------===// 552// VI and GFX9. 553//===----------------------------------------------------------------------===// 554 555class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 556 : SM_Real<ps> 557 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 558 , Enc64 { 559 field bit IsGFX9SpecificEncoding = false; 560 let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9); 561 let DecoderNamespace = "GFX8"; 562 563 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 564 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 565 566 // Note that for GFX9 instructions with immediate offsets, soffset_en 567 // must be defined, whereas in GFX8 it's undefined in all cases, 568 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite 569 // documentation suggesting otherwise. 570 field bit SOffsetEn = !if(IsGFX9SpecificEncoding, 571 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)), 572 ?); 573 let Inst{14} = SOffsetEn; 574 575 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 576 577 // imm 578 // TODO: Shall not be defined if the instruction has no offset nor 579 // soffset. 580 let Inst{17} = ps.has_offset; 581 582 let Inst{25-18} = op; 583 let Inst{31-26} = 0x30; //encoding 584 585 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 586 // Offset value is corrected accordingly when offset is encoded/decoded. 587 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics. 588 field bits<21> Offset; 589 let Offset{6-0} = !if(ps.has_offset, offset{6-0}, 590 !if(ps.has_soffset, soffset{6-0}, ?)); 591 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?); 592 let Inst{52-32} = Offset; 593 594 // soffset 595 let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset), 596 soffset{6-0}, ?); 597} 598 599class SMEM_Real_Load_vi<bits<8> op, string ps, dag offsets> 600 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)> { 601 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps).BaseClass; 602 let InOperandList = !con((ins BaseClass:$sbase), offsets, (ins CPol:$cpol)); 603} 604 605// The alternative GFX9 SGPR encoding using soffset to encode the 606// offset register. Not available in assembler and goes to the GFX9 607// encoding family to avoid conflicts with the primary SGPR variant. 608class SMEM_Real_SGPR_alt_gfx9 { 609 bit IsGFX9SpecificEncoding = true; 610 bit SOffsetEn = 1; 611 bit Offset = ?; 612 int Subtarget = SIEncodingFamily.GFX9; 613 string AsmVariantName = "NonParsable"; 614} 615 616multiclass SM_Real_Loads_vi<bits<8> op, string ps> { 617 def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM", (ins smem_offset:$offset)>; 618 def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR", (ins SReg_32:$soffset)>; 619 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR", 620 (ins SReg_32:$soffset)>, 621 SMEM_Real_SGPR_alt_gfx9; 622 let IsGFX9SpecificEncoding = true in 623 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi < 624 op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 625} 626 627class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 628 // encoding 629 bits<7> sdata; 630 631 let sdst = ?; 632 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 633} 634 635class SMEM_Real_Store_vi <bits<8> op, string ps, dag offsets> 636 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)> { 637 RegisterClass SrcClass = !cast<SM_Store_Pseudo>(ps).SrcClass; 638 RegisterClass BaseClass = !cast<SM_Store_Pseudo>(ps).BaseClass; 639 let InOperandList = !con((ins SrcClass:$sdata, BaseClass:$sbase), 640 offsets, (ins CPol:$cpol)); 641} 642 643multiclass SM_Real_Stores_vi<bits<8> op, string ps> { 644 def _IMM_vi : SMEM_Real_Store_vi <op, ps#_IMM, (ins smem_offset:$offset)>; 645 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#_SGPR, (ins SReg_32:$soffset)>; 646 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR", 647 (ins SReg_32:$soffset)>, 648 SMEM_Real_SGPR_alt_gfx9; 649 let IsGFX9SpecificEncoding = true in 650 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi < 651 op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 652} 653 654multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 655 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 656 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 657 def _SGPR_alt_gfx9 658 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>, 659 SMEM_Real_SGPR_alt_gfx9; 660 let IsGFX9SpecificEncoding = true in 661 def _SGPR_IMM_gfx9 662 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 663} 664 665defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 666defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 667defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 668defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 669defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 670defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 671defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 672defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 673defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 674defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 675 676defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 677defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 678defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 679 680defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 681defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 682defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 683 684// These instructions use same encoding 685def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 686def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 687def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 688def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 689def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 690def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 691 692defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 693defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 694defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 695 696defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 697defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 698defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 699 700defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 701defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 702 703//===----------------------------------------------------------------------===// 704// GFX9 705//===----------------------------------------------------------------------===// 706 707class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 708 : SMEM_Real_vi <op, ps>, 709 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 710 711 bits<7> sdata; 712 713 let Constraints = ps.Constraints; 714 let DisableEncoding = ps.DisableEncoding; 715 716 let cpol{CPolBit.GLC} = ps.glc; 717 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 718} 719 720multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 721 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 722 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 723 def _SGPR_alt_gfx9 724 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>, 725 SMEM_Real_SGPR_alt_gfx9; 726 let IsGFX9SpecificEncoding = true in 727 def _SGPR_IMM_gfx9 728 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 729 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 730 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 731 def _SGPR_RTN_alt_gfx9 732 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>, 733 SMEM_Real_SGPR_alt_gfx9; 734 let IsGFX9SpecificEncoding = true in 735 def _SGPR_IMM_RTN_gfx9 736 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 737} 738 739defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 740defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 741defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 742defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 743defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 744defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 745defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 746defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 747defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 748defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 749defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 750defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 751defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 752 753defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 754defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 755defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 756defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 757defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 758defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 759defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 760defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 761defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 762defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 763defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 764defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 765defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 766 767defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 768defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 769defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 770defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 771defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 772defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 773defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 774defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 775defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 776defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 777defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 778defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 779defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 780 781defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 782defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 783defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 784defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 785defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 786defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 787defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 788defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 789defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 790defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 791defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 792defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 793defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 794 795multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 796 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 797 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 798 def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>, 799 SMEM_Real_SGPR_alt_gfx9; 800 let IsGFX9SpecificEncoding = true in 801 def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>; 802} 803 804defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 805defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 806 807//===----------------------------------------------------------------------===// 808// CI 809//===----------------------------------------------------------------------===// 810 811def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 812 NamedMatchClass<"SMRDLiteralOffset">> { 813 let OperandType = "OPERAND_IMMEDIATE"; 814} 815 816class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 817 SM_Real<ps>, 818 Enc64 { 819 820 let AssemblerPredicate = isGFX7Only; 821 let DecoderNamespace = "GFX7"; 822 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 823 824 let Inst{7-0} = 0xff; 825 let Inst{8} = 0; 826 let Inst{14-9} = sbase{6-1}; 827 let Inst{21-15} = sdst{6-0}; 828 let Inst{26-22} = op; 829 let Inst{31-27} = 0x18; //encoding 830 let Inst{63-32} = offset{31-0}; 831} 832 833def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 834def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 835def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 836def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 837def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 838def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 839def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 840def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 841def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 842def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 843 844class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 845 : SM_Real<ps> 846 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 847 , Enc32 { 848 849 let AssemblerPredicate = isGFX7Only; 850 let DecoderNamespace = "GFX7"; 851 852 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 853 let Inst{8} = ps.has_offset; 854 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 855 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 856 let Inst{26-22} = op; 857 let Inst{31-27} = 0x18; //encoding 858} 859 860def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 861 862//===----------------------------------------------------------------------===// 863// Scalar Memory Patterns 864//===----------------------------------------------------------------------===// 865 866def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 867 let GISelPredicateCode = [{ 868 if (!MI.hasOneMemOperand()) 869 return false; 870 if (!isInstrUniform(MI)) 871 return false; 872 873 // FIXME: We should probably be caching this. 874 SmallVector<GEPInfo, 4> AddrInfo; 875 getAddrModeInfo(MI, MRI, AddrInfo); 876 877 if (hasVgprParts(AddrInfo)) 878 return false; 879 return true; 880 }]; 881} 882 883def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 884def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 885def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 886def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 887def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 888 889multiclass SMRD_Pattern <string Instr, ValueType vt> { 890 891 // 1. IMM offset 892 def : GCNPat < 893 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 894 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 895 >; 896 897 // 2. 32-bit IMM offset on CI 898 def : GCNPat < 899 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 900 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 901 let OtherPredicates = [isGFX7Only]; 902 } 903 904 // 3. SGPR offset 905 def : GCNPat < 906 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 907 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0)) 908 >; 909 910 // 4. No offset 911 def : GCNPat < 912 (vt (smrd_load (i64 SReg_64:$sbase))), 913 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0)) 914 >; 915} 916 917multiclass SMLoad_Pattern <string Instr, ValueType vt> { 918 // 1. Offset as an immediate 919 def : GCNPat < 920 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 921 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 922 let AddedComplexity = 2; 923 } 924 925 // 2. 32-bit IMM offset on CI 926 def : GCNPat < 927 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 928 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 929 (extract_cpol $cachepolicy))> { 930 let OtherPredicates = [isGFX7Only]; 931 let AddedComplexity = 1; 932 } 933 934 // 3. Offset loaded in an 32bit SGPR 935 def : GCNPat < 936 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 937 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_cpol $cachepolicy))) 938 >; 939} 940 941// Global and constant loads can be selected to either MUBUF or SMRD 942// instructions, but SMRD instructions are faster so we want the instruction 943// selector to prefer those. 944let AddedComplexity = 100 in { 945 946foreach vt = Reg32Types.types in { 947defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 948} 949 950foreach vt = SReg_64.RegTypes in { 951defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 952} 953 954foreach vt = SReg_128.RegTypes in { 955defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 956} 957 958foreach vt = SReg_256.RegTypes in { 959defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 960} 961 962foreach vt = SReg_512.RegTypes in { 963defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 964} 965 966defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 967defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 968defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 969defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 970defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 971 972defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 973defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 974defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 975defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 976defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 977} // End let AddedComplexity = 100 978 979let OtherPredicates = [HasSMemTimeInst] in { 980def : GCNPat < 981 (i64 (readcyclecounter)), 982 (S_MEMTIME) 983>; 984} // let OtherPredicates = [HasSMemTimeInst] 985 986let OtherPredicates = [HasShaderCyclesRegister] in { 987def : GCNPat < 988 (i64 (readcyclecounter)), 989 (REG_SEQUENCE SReg_64, 990 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 991 (S_MOV_B32 (i32 0)), sub1)> { 992 // Prefer this to s_memtime because it has lower and more predictable latency. 993 let AddedComplexity = 1; 994} 995} // let OtherPredicates = [HasShaderCyclesRegister] 996 997//===----------------------------------------------------------------------===// 998// GFX10. 999//===----------------------------------------------------------------------===// 1000 1001class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName, 1002 int subtarget, RegisterWithSubRegs sgpr_null> : 1003 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 1004 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 1005 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 1006 let Inst{25-18} = op; 1007 let Inst{31-26} = 0x3d; 1008 // There are SMEM instructions that do not employ any of the offset 1009 // fields, in which case we need them to remain undefined. 1010 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?)); 1011 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 1012 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 1013} 1014 1015class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> 1016 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10, 1017 SGPR_NULL_gfxpre11> { 1018 let AssemblerPredicate = isGFX10Only; 1019 let DecoderNamespace = "GFX10"; 1020 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 1021 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 1022} 1023 1024multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, 1025 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 1026 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 1027 def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { 1028 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 1029 } 1030 def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { 1031 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 1032 } 1033 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10<op, !cast<SM_Load_Pseudo>(ps#_SGPR_IMM)> { 1034 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, 1035 smem_offset_mod:$offset, CPol:$cpol); 1036 } 1037} 1038 1039class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 1040 bits<7> sdata; 1041 1042 let sdst = ?; 1043 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1044} 1045 1046multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 1047 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 1048 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 1049 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 1050 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 1051 } 1052 1053 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 1054 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 1055 } 1056 1057 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Store_Pseudo>(ps#_SGPR_IMM)> { 1058 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, 1059 SReg_32:$soffset, smem_offset_mod:$offset, CPol:$cpol); 1060 } 1061} 1062 1063defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 1064defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 1065defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 1066defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 1067defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 1068 1069let SubtargetPredicate = HasScalarFlatScratchInsts in { 1070defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 1071defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 1072defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 1073} // End SubtargetPredicate = HasScalarFlatScratchInsts 1074 1075defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 1076defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 1077defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 1078defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 1079defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 1080 1081let SubtargetPredicate = HasScalarStores in { 1082defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 1083defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 1084defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 1085let OtherPredicates = [HasScalarFlatScratchInsts] in { 1086defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 1087defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 1088defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 1089} // End OtherPredicates = [HasScalarFlatScratchInsts] 1090defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 1091defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 1092defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 1093} // End SubtargetPredicate = HasScalarStores 1094 1095def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 1096def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 1097def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 1098def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 1099def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 1100 1101let SubtargetPredicate = HasScalarStores in { 1102def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 1103} // End SubtargetPredicate = HasScalarStores 1104 1105multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 1106 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1107 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1108 def _SGPR_IMM_gfx10 1109 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1110} 1111 1112defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 1113defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 1114 1115class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 1116 : SMEM_Real_gfx10 <op, ps>, 1117 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 1118 1119 bits<7> sdata; 1120 1121 let Constraints = ps.Constraints; 1122 let DisableEncoding = ps.DisableEncoding; 1123 1124 let cpol{CPolBit.GLC} = ps.glc; 1125 1126 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1127 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 1128} 1129 1130multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 1131 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1132 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1133 def _SGPR_IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 1134 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1135 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1136 def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 1137} 1138 1139let SubtargetPredicate = HasScalarAtomics in { 1140 1141defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1142defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1143defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1144defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1145defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1146defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1147defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1148defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1149defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1150defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1151defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1152defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1153defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1154 1155defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1156defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1157defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1158defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1159defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1160defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1161defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1162defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1163defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1164defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1165defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1166defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1167defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1168 1169defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1170defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1171defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1172defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1173defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1174defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1175defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1176defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1177defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1178defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1179defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1180defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1181defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1182 1183defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1184defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1185defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1186defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1187defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1188defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1189defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1190defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1191defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1192defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1193defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1194defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1195defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1196 1197multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1198 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1199 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1200 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1201} 1202 1203defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1204defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1205 1206} // End SubtargetPredicate = HasScalarAtomics 1207 1208def SMInfoTable : GenericTable { 1209 let FilterClass = "SM_Real"; 1210 let CppTypeName = "SMInfo"; 1211 let Fields = ["Opcode", "is_buffer"]; 1212 1213 let PrimaryKey = ["Opcode"]; 1214 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1215} 1216 1217//===----------------------------------------------------------------------===// 1218// GFX11. 1219//===----------------------------------------------------------------------===// 1220 1221class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1222 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11, 1223 SGPR_NULL_gfx11plus> { 1224 let AssemblerPredicate = isGFX11Plus; 1225 let DecoderNamespace = "GFX11"; 1226 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1227 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0); 1228} 1229 1230class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName, dag offsets> : 1231 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName> { 1232 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps).BaseClass; 1233 let InOperandList = !con((ins BaseClass:$sbase), offsets, (ins CPol:$cpol)); 1234} 1235 1236multiclass SM_Real_Loads_gfx11<bits<8> op, string ps, string opName> { 1237 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName, (ins smem_offset:$offset)>; 1238 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName, (ins SReg_32:$soffset)>; 1239 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11< 1240 op, ps#"_SGPR_IMM", opName, (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 1241 def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>, 1242 Requires<[isGFX11Plus]>; 1243} 1244 1245defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD", "s_load_b32">; 1246defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2", "s_load_b64">; 1247defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4", "s_load_b128">; 1248defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8", "s_load_b256">; 1249defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16", "s_load_b512">; 1250 1251defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD", "s_buffer_load_b32">; 1252defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2", "s_buffer_load_b64">; 1253defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4", "s_buffer_load_b128">; 1254defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8", "s_buffer_load_b256">; 1255defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16", "s_buffer_load_b512">; 1256 1257def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>; 1258def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>; 1259 1260class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> { 1261 // encoding 1262 bits<7> sdata; 1263 1264 let sdst = ?; 1265 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1266} 1267 1268multiclass SM_Real_Probe_gfx11<bits<8> op, string ps> { 1269 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1270 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 1271 def _SGPR_IMM_gfx11 1272 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 1273} 1274 1275defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22, "S_ATC_PROBE">; 1276defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23, "S_ATC_PROBE_BUFFER">; 1277