1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14class SMEMOffset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21def smem_offset : SMEMOffset; 22 23def smem_offset_mod : SMEMOffset { 24 let PrintMethod = "printSMEMOffsetMod"; 25} 26 27//===----------------------------------------------------------------------===// 28// Scalar Memory classes 29//===----------------------------------------------------------------------===// 30 31class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 32 InstSI <outs, ins, "", pattern>, 33 SIMCInstr<opName, SIEncodingFamily.NONE> { 34 let isPseudo = 1; 35 let isCodeGenOnly = 1; 36 37 let LGKM_CNT = 1; 38 let SMRD = 1; 39 let mayStore = 0; 40 let mayLoad = 1; 41 let hasSideEffects = 0; 42 let UseNamedOperandTable = 1; 43 let SchedRW = [WriteSMEM]; 44 45 string Mnemonic = opName; 46 string AsmOperands = asmOps; 47 48 bits<1> has_sbase = 1; 49 bits<1> has_sdst = 1; 50 bit has_glc = 0; 51 bit has_dlc = 0; 52 bit has_offset = 0; 53 bit has_soffset = 0; 54 bit is_buffer = 0; 55} 56 57class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic> 58 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> { 59 60 let isPseudo = 0; 61 let isCodeGenOnly = 0; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 // copy relevant pseudo op flags 66 let LGKM_CNT = ps.LGKM_CNT; 67 let SMRD = ps.SMRD; 68 let mayStore = ps.mayStore; 69 let mayLoad = ps.mayLoad; 70 let hasSideEffects = ps.hasSideEffects; 71 let UseNamedOperandTable = ps.UseNamedOperandTable; 72 let SchedRW = ps.SchedRW; 73 let SubtargetPredicate = ps.SubtargetPredicate; 74 let AsmMatchConverter = ps.AsmMatchConverter; 75 let IsAtomicRet = ps.IsAtomicRet; 76 let IsAtomicNoRet = ps.IsAtomicNoRet; 77 78 let TSFlags = ps.TSFlags; 79 80 bit is_buffer = ps.is_buffer; 81 82 // encoding 83 bits<7> sbase; 84 bits<7> sdst; 85 bits<32> offset; 86 bits<8> soffset; 87 bits<5> cpol; 88} 89 90class SM_Probe_Pseudo <string opName, dag ins, bit isImm> 91 : SM_Pseudo<opName, (outs), ins, 92 " $sdata, $sbase, " # !if(isImm, "$offset", "$soffset")> { 93 let mayLoad = 0; 94 let mayStore = 0; 95 let has_glc = 0; 96 let LGKM_CNT = 0; 97 let ScalarStore = 0; 98 let hasSideEffects = 1; 99 let has_offset = isImm; 100 let has_soffset = !not(isImm); 101 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 102} 103 104class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 105 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 106 RegisterClass BaseClass; 107 let mayLoad = 1; 108 let mayStore = 0; 109 let has_glc = 1; 110 let has_dlc = 1; 111} 112 113class SM_Store_Pseudo <string opName, RegisterClass baseClass, 114 RegisterClass srcClass, dag ins, string asmOps> 115 : SM_Pseudo<opName, (outs), ins, asmOps, []> { 116 RegisterClass BaseClass = baseClass; 117 RegisterClass SrcClass = srcClass; 118 let mayLoad = 0; 119 let mayStore = 1; 120 let has_glc = 1; 121 let has_dlc = 1; 122 let ScalarStore = 1; 123} 124 125class SM_Discard_Pseudo <string opName, dag ins, bit isImm> 126 : SM_Pseudo<opName, (outs), ins, 127 " $sbase, " # !if(isImm, "$offset", "$soffset")> { 128 let mayLoad = 0; 129 let mayStore = 0; 130 let has_glc = 0; 131 let has_sdst = 0; 132 let ScalarStore = 0; 133 let hasSideEffects = 1; 134 let has_offset = isImm; 135 let has_soffset = !not(isImm); 136 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 137} 138 139multiclass SM_Pseudo_Loads<string opName, 140 RegisterClass baseClass, 141 RegisterClass dstClass> { 142 def _IMM : SM_Load_Pseudo <opName, 143 (outs dstClass:$sdst), 144 (ins baseClass:$sbase, i32imm:$offset, CPol:$cpol), 145 " $sdst, $sbase, $offset$cpol", []> { 146 let has_offset = 1; 147 let BaseClass = baseClass; 148 let PseudoInstr = opName # "_IMM"; 149 let has_glc = 1; 150 let has_dlc = 1; 151 } 152 153 def _SGPR : SM_Load_Pseudo <opName, 154 (outs dstClass:$sdst), 155 (ins baseClass:$sbase, SReg_32:$soffset, CPol:$cpol), 156 " $sdst, $sbase, $soffset$cpol", []> { 157 let has_soffset = 1; 158 let BaseClass = baseClass; 159 let PseudoInstr = opName # "_SGPR"; 160 let has_glc = 1; 161 let has_dlc = 1; 162 } 163 164 def _SGPR_IMM : SM_Load_Pseudo <opName, 165 (outs dstClass:$sdst), 166 (ins baseClass:$sbase, SReg_32:$soffset, 167 i32imm:$offset, CPol:$cpol), 168 " $sdst, $sbase, $soffset$offset$cpol", []> { 169 let has_offset = 1; 170 let has_soffset = 1; 171 let BaseClass = baseClass; 172 let PseudoInstr = opName # "_SGPR_IMM"; 173 let has_glc = 1; 174 let has_dlc = 1; 175 } 176} 177 178multiclass SM_Pseudo_Stores<string opName, 179 RegisterClass baseClass, 180 RegisterClass srcClass> { 181 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 182 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol), 183 " $sdata, $sbase, $offset$cpol"> { 184 let has_offset = 1; 185 let PseudoInstr = opName # "_IMM"; 186 } 187 188 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, 189 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPol:$cpol), 190 " $sdata, $sbase, $soffset$cpol"> { 191 let has_soffset = 1; 192 let PseudoInstr = opName # "_SGPR"; 193 } 194 195 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 196 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, i32imm:$offset, 197 CPol:$cpol), 198 " $sdata, $sbase, $soffset$offset$cpol"> { 199 let has_offset = 1; 200 let has_soffset = 1; 201 let PseudoInstr = opName # "_SGPR_IMM"; 202 } 203} 204 205multiclass SM_Pseudo_Discards<string opName> { 206 def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>; 207 def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$soffset), 0>; 208} 209 210class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 211 opName, (outs SReg_64_XEXEC:$sdst), (ins), 212 " $sdst", [(set i64:$sdst, (node))]> { 213 let hasSideEffects = 1; 214 215 let mayStore = 0; 216 let mayLoad = 0; 217 let has_sbase = 0; 218} 219 220class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 221 opName, (outs), (ins), "", [(node)]> { 222 let hasSideEffects = 1; 223 let mayLoad = 0; 224 let mayStore = 0; 225 let has_sdst = 0; 226 let has_sbase = 0; 227} 228 229multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 230 def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>; 231 def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$soffset), 0>; 232} 233 234class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 235 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 236 " $sdst", [(set i32:$sdst, (node))]> { 237 let hasSideEffects = 1; 238 let mayStore = 0; 239 let mayLoad = 0; 240 let has_sbase = 0; 241} 242 243//===----------------------------------------------------------------------===// 244// Scalar Atomic Memory Classes 245//===----------------------------------------------------------------------===// 246 247class SM_Atomic_Pseudo <string opName, 248 dag outs, dag ins, string asmOps, bit isRet> 249 : SM_Pseudo<opName, outs, ins, asmOps, []> { 250 251 bit glc = isRet; 252 253 let mayLoad = 1; 254 let mayStore = 1; 255 let has_glc = 1; 256 let has_dlc = 1; 257 let has_soffset = 1; 258 259 // Should these be set? 260 let ScalarStore = 1; 261 let hasSideEffects = 1; 262 let maybeAtomic = 1; 263 264 let IsAtomicNoRet = !not(isRet); 265 let IsAtomicRet = isRet; 266 267 let AsmMatchConverter = "cvtSMEMAtomic"; 268} 269 270class SM_Pseudo_Atomic<string opName, 271 RegisterClass baseClass, 272 RegisterClass dataClass, 273 bit isImm, 274 bit isRet, 275 string opNameWithSuffix = opName # !if(isImm, 276 !if(isRet, "_IMM_RTN", "_IMM"), 277 !if(isRet, "_SGPR_RTN", "_SGPR")), 278 Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> : 279 SM_Atomic_Pseudo<opName, 280 !if(isRet, (outs dataClass:$sdst), (outs)), 281 !if(isImm, 282 (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, CPolTy:$cpol), 283 (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPolTy:$cpol)), 284 !if(isRet, " $sdst", " $sdata") # ", $sbase, " # 285 !if(isImm, "$offset", "$soffset") # "$cpol", 286 isRet>, 287 AtomicNoRet <opNameWithSuffix, isRet> { 288 let has_offset = isImm; 289 let has_soffset = !not(isImm); 290 let PseudoInstr = opNameWithSuffix; 291 292 let Constraints = !if(isRet, "$sdst = $sdata", ""); 293 let DisableEncoding = !if(isRet, "$sdata", ""); 294} 295 296multiclass SM_Pseudo_Atomics<string opName, 297 RegisterClass baseClass, 298 RegisterClass dataClass> { 299 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>; 300 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>; 301 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>; 302 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>; 303} 304 305//===----------------------------------------------------------------------===// 306// Scalar Memory Instructions 307//===----------------------------------------------------------------------===// 308 309// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 310// SMRD instructions, because the SReg_32_XM0 register class does not include M0 311// and writing to M0 from an SMRD instruction will hang the GPU. 312 313// XXX - SMEM instructions do not allow exec for data operand, but 314// does sdst for SMRD on SI/CI? 315defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 316defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 317defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 318defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 319defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 320 321let is_buffer = 1 in { 322defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 323 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 324>; 325 326// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 327// SI/CI, bit disallowed for SMEM on VI. 328defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 329 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 330>; 331 332defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 333 "s_buffer_load_dwordx4", SReg_128, SReg_128 334>; 335 336defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 337 "s_buffer_load_dwordx8", SReg_128, SReg_256 338>; 339 340defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 341 "s_buffer_load_dwordx16", SReg_128, SReg_512 342>; 343} 344 345let SubtargetPredicate = HasScalarStores in { 346defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 347defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 348defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 349 350let is_buffer = 1 in { 351defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 352 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 353>; 354 355defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 356 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 357>; 358 359defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 360 "s_buffer_store_dwordx4", SReg_128, SReg_128 361>; 362} 363} // End SubtargetPredicate = HasScalarStores 364 365let SubtargetPredicate = HasSMemTimeInst in 366def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 367def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 368 369let SubtargetPredicate = isGFX7GFX8GFX9 in { 370def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 371} // let SubtargetPredicate = isGFX7GFX8GFX9 372 373let SubtargetPredicate = isGFX8Plus in { 374let OtherPredicates = [HasScalarStores] in { 375def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 376def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 377} // End OtherPredicates = [HasScalarStores] 378 379defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 380let is_buffer = 1 in { 381defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 382} 383} // SubtargetPredicate = isGFX8Plus 384 385let SubtargetPredicate = HasSMemRealTime in 386def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 387 388let SubtargetPredicate = isGFX10Plus in 389def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 390let SubtargetPredicate = HasGetWaveIdInst in 391def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 392 393 394let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 395defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 396defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 397defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 398 399defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 400defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 401defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 402} // SubtargetPredicate = HasScalarFlatScratchInsts 403 404let SubtargetPredicate = HasScalarAtomics in { 405 406let is_buffer = 1 in { 407defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 408defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 409defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 410defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 411defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 412defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 413defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 414defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 415defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 416defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 417defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 418defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 419defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 420 421defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 422defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 423defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 424defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 425defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 426defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 427defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 428defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 429defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 430defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 431defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 432defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 433defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 434} 435 436defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 437defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 438defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 439defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 440defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 441defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 442defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 443defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 444defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 445defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 446defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 447defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 448defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 449 450defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 451defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 452defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 453defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 454defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 455defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 456defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 457defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 458defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 459defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 460defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 461defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 462defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 463 464} // let SubtargetPredicate = HasScalarAtomics 465 466let SubtargetPredicate = HasScalarAtomics in { 467defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 468defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 469} 470 471//===----------------------------------------------------------------------===// 472// Targets 473//===----------------------------------------------------------------------===// 474 475//===----------------------------------------------------------------------===// 476// SI 477//===----------------------------------------------------------------------===// 478 479class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 480 : SM_Real<ps> 481 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 482 , Enc32 { 483 484 let AssemblerPredicate = isGFX6GFX7; 485 let DecoderNamespace = "GFX6GFX7"; 486 487 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 488 let Inst{8} = ps.has_offset; 489 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 490 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 491 let Inst{26-22} = op; 492 let Inst{31-27} = 0x18; //encoding 493} 494 495multiclass SM_Real_Loads_si<bits<5> op, string ps, 496 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 497 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 498 499 def _IMM_si : SMRD_Real_si <op, immPs> { 500 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 501 } 502 503 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 504 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 505 } 506 507} 508 509defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 510defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 511defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 512defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 513defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 514defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 515defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 516defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 517defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 518defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 519 520def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 521def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 522 523 524//===----------------------------------------------------------------------===// 525// VI and GFX9. 526//===----------------------------------------------------------------------===// 527 528class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 529 : SM_Real<ps> 530 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 531 , Enc64 { 532 field bit IsGFX9Specific = false; 533 let AssemblerPredicate = !if(IsGFX9Specific, isGFX9Only, isGFX8GFX9); 534 let DecoderNamespace = "GFX8"; 535 536 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 537 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 538 539 // Note that for GFX9 instructions with immediate offsets, soffset_en 540 // must be defined, whereas in GFX8 it's undefined in all cases, 541 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite 542 // documentation suggesting otherwise. 543 field bit SOffsetEn = !if(IsGFX9Specific, 544 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)), 545 ?); 546 let Inst{14} = SOffsetEn; 547 548 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 549 550 // imm 551 // TODO: Shall not be defined if the instruction has no offset nor 552 // soffset. 553 let Inst{17} = ps.has_offset; 554 555 let Inst{25-18} = op; 556 let Inst{31-26} = 0x30; //encoding 557 558 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 559 // Offset value is corrected accordingly when offset is encoded/decoded. 560 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics. 561 field bits<21> Offset; 562 let Offset{6-0} = !if(ps.has_offset, offset{6-0}, 563 !if(ps.has_soffset, soffset{6-0}, ?)); 564 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?); 565 let Inst{52-32} = Offset; 566 567 // soffset 568 let Inst{63-57} = !if(!and(IsGFX9Specific, ps.has_soffset), soffset{6-0}, ?); 569} 570 571class SMEM_Real_Load_vi<bits<8> op, string ps, dag offsets> 572 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)> { 573 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps).BaseClass; 574 let InOperandList = !con((ins BaseClass:$sbase), offsets, (ins CPol:$cpol)); 575} 576 577multiclass SM_Real_Loads_vi<bits<8> op, string ps> { 578 def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM", (ins smem_offset:$offset)>; 579 def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR", (ins SReg_32:$soffset)>; 580 let IsGFX9Specific = true in { 581 // The alternative GFX9 SGPR encoding using soffset to encode the 582 // offset register. Not available in assembler and goes to the GFX9 583 // encoding family to avoid conflicts with the primary SGPR variant. 584 let SOffsetEn = 1, Offset = ?, Subtarget = SIEncodingFamily.GFX9, 585 AsmVariantName = "NonParsable" in 586 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR", 587 (ins SReg_32:$soffset)>; 588 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi < 589 op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 590 } 591} 592 593class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 594 // encoding 595 bits<7> sdata; 596 597 let sdst = ?; 598 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 599} 600 601class SMEM_Real_Store_vi <bits<8> op, string ps, dag offsets> 602 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)> { 603 RegisterClass SrcClass = !cast<SM_Store_Pseudo>(ps).SrcClass; 604 RegisterClass BaseClass = !cast<SM_Store_Pseudo>(ps).BaseClass; 605 let InOperandList = !con((ins SrcClass:$sdata, BaseClass:$sbase), 606 offsets, (ins CPol:$cpol)); 607} 608 609multiclass SM_Real_Stores_vi<bits<8> op, string ps> { 610 def _IMM_vi : SMEM_Real_Store_vi <op, ps#_IMM, (ins smem_offset:$offset)>; 611 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#_SGPR, (ins SReg_32:$soffset)>; 612 let IsGFX9Specific = true in { 613 // The alternative GFX9 SGPR encoding using soffset to encode the 614 // offset register. Not available in assembler and goes to the GFX9 615 // encoding family to avoid conflicts with the primary SGPR variant. 616 let SOffsetEn = 1, Offset = ?, Subtarget = SIEncodingFamily.GFX9, 617 AsmVariantName = "NonParsable" in 618 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR", 619 (ins SReg_32:$soffset)>; 620 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi < 621 op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 622 } 623} 624 625multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 626 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 627 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 628} 629 630defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 631defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 632defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 633defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 634defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 635defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 636defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 637defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 638defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 639defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 640 641defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 642defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 643defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 644 645defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 646defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 647defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 648 649// These instructions use same encoding 650def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 651def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 652def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 653def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 654def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 655def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 656 657defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 658defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 659defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 660 661defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 662defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 663defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 664 665defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 666defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 667 668//===----------------------------------------------------------------------===// 669// GFX9 670//===----------------------------------------------------------------------===// 671 672class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 673 : SMEM_Real_vi <op, ps>, 674 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 675 676 bits<7> sdata; 677 678 let Constraints = ps.Constraints; 679 let DisableEncoding = ps.DisableEncoding; 680 681 let cpol{CPolBit.GLC} = ps.glc; 682 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 683} 684 685multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 686 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 687 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 688 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 689 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 690} 691 692defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 693defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 694defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 695defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 696defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 697defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 698defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 699defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 700defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 701defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 702defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 703defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 704defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 705 706defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 707defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 708defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 709defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 710defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 711defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 712defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 713defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 714defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 715defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 716defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 717defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 718defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 719 720defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 721defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 722defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 723defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 724defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 725defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 726defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 727defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 728defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 729defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 730defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 731defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 732defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 733 734defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 735defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 736defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 737defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 738defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 739defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 740defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 741defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 742defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 743defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 744defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 745defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 746defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 747 748multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 749 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 750 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 751} 752 753defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 754defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 755 756//===----------------------------------------------------------------------===// 757// CI 758//===----------------------------------------------------------------------===// 759 760def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 761 NamedMatchClass<"SMRDLiteralOffset">> { 762 let OperandType = "OPERAND_IMMEDIATE"; 763} 764 765class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 766 SM_Real<ps>, 767 Enc64 { 768 769 let AssemblerPredicate = isGFX7Only; 770 let DecoderNamespace = "GFX7"; 771 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 772 773 let Inst{7-0} = 0xff; 774 let Inst{8} = 0; 775 let Inst{14-9} = sbase{6-1}; 776 let Inst{21-15} = sdst{6-0}; 777 let Inst{26-22} = op; 778 let Inst{31-27} = 0x18; //encoding 779 let Inst{63-32} = offset{31-0}; 780} 781 782def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 783def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 784def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 785def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 786def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 787def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 788def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 789def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 790def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 791def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 792 793class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 794 : SM_Real<ps> 795 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 796 , Enc32 { 797 798 let AssemblerPredicate = isGFX7Only; 799 let DecoderNamespace = "GFX7"; 800 801 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 802 let Inst{8} = ps.has_offset; 803 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 804 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 805 let Inst{26-22} = op; 806 let Inst{31-27} = 0x18; //encoding 807} 808 809def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 810 811//===----------------------------------------------------------------------===// 812// Scalar Memory Patterns 813//===----------------------------------------------------------------------===// 814 815def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 816 let GISelPredicateCode = [{ 817 if (!MI.hasOneMemOperand()) 818 return false; 819 if (!isInstrUniform(MI)) 820 return false; 821 822 // FIXME: We should probably be caching this. 823 SmallVector<GEPInfo, 4> AddrInfo; 824 getAddrModeInfo(MI, MRI, AddrInfo); 825 826 if (hasVgprParts(AddrInfo)) 827 return false; 828 return true; 829 }]; 830} 831 832def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 833def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 834def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 835def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 836def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 837 838multiclass SMRD_Pattern <string Instr, ValueType vt> { 839 840 // 1. IMM offset 841 def : GCNPat < 842 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 843 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 844 >; 845 846 // 2. 32-bit IMM offset on CI 847 def : GCNPat < 848 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 849 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 850 let OtherPredicates = [isGFX7Only]; 851 } 852 853 // 3. SGPR offset 854 def : GCNPat < 855 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 856 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0)) 857 >; 858 859 // 4. No offset 860 def : GCNPat < 861 (vt (smrd_load (i64 SReg_64:$sbase))), 862 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0)) 863 >; 864} 865 866multiclass SMLoad_Pattern <string Instr, ValueType vt> { 867 // 1. Offset as an immediate 868 def : GCNPat < 869 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 870 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 871 let AddedComplexity = 2; 872 } 873 874 // 2. 32-bit IMM offset on CI 875 def : GCNPat < 876 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 877 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 878 (extract_cpol $cachepolicy))> { 879 let OtherPredicates = [isGFX7Only]; 880 let AddedComplexity = 1; 881 } 882 883 // 3. Offset loaded in an 32bit SGPR 884 def : GCNPat < 885 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 886 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_cpol $cachepolicy))) 887 >; 888} 889 890// Global and constant loads can be selected to either MUBUF or SMRD 891// instructions, but SMRD instructions are faster so we want the instruction 892// selector to prefer those. 893let AddedComplexity = 100 in { 894 895foreach vt = Reg32Types.types in { 896defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 897} 898 899foreach vt = SReg_64.RegTypes in { 900defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 901} 902 903foreach vt = SReg_128.RegTypes in { 904defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 905} 906 907foreach vt = SReg_256.RegTypes in { 908defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 909} 910 911foreach vt = SReg_512.RegTypes in { 912defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 913} 914 915defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 916defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 917defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 918defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 919defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 920 921defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 922defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 923defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 924defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 925defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 926} // End let AddedComplexity = 100 927 928let OtherPredicates = [HasSMemTimeInst] in { 929def : GCNPat < 930 (i64 (readcyclecounter)), 931 (S_MEMTIME) 932>; 933} // let OtherPredicates = [HasSMemTimeInst] 934 935let OtherPredicates = [HasShaderCyclesRegister] in { 936def : GCNPat < 937 (i64 (readcyclecounter)), 938 (REG_SEQUENCE SReg_64, 939 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 940 (S_MOV_B32 (i32 0)), sub1)> { 941 // Prefer this to s_memtime because it has lower and more predictable latency. 942 let AddedComplexity = 1; 943} 944} // let OtherPredicates = [HasShaderCyclesRegister] 945 946//===----------------------------------------------------------------------===// 947// GFX10. 948//===----------------------------------------------------------------------===// 949 950class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName, 951 int subtarget, RegisterWithSubRegs sgpr_null> : 952 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 953 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 954 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 955 let Inst{25-18} = op; 956 let Inst{31-26} = 0x3d; 957 // There are SMEM instructions that do not employ any of the offset 958 // fields, in which case we need them to remain undefined. 959 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?)); 960 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 961 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 962} 963 964class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> 965 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10, 966 SGPR_NULL_gfxpre11> { 967 let AssemblerPredicate = isGFX10Only; 968 let DecoderNamespace = "GFX10"; 969 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 970 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 971} 972 973multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, 974 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 975 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 976 def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { 977 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 978 } 979 def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { 980 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 981 } 982 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10<op, !cast<SM_Load_Pseudo>(ps#_SGPR_IMM)> { 983 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, 984 smem_offset_mod:$offset, CPol:$cpol); 985 } 986} 987 988class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 989 bits<7> sdata; 990 991 let sdst = ?; 992 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 993} 994 995multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 996 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 997 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 998 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 999 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 1000 } 1001 1002 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 1003 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 1004 } 1005 1006 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Store_Pseudo>(ps#_SGPR_IMM)> { 1007 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, 1008 SReg_32:$soffset, smem_offset_mod:$offset, CPol:$cpol); 1009 } 1010} 1011 1012defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 1013defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 1014defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 1015defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 1016defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 1017 1018let SubtargetPredicate = HasScalarFlatScratchInsts in { 1019defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 1020defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 1021defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 1022} // End SubtargetPredicate = HasScalarFlatScratchInsts 1023 1024defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 1025defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 1026defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 1027defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 1028defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 1029 1030let SubtargetPredicate = HasScalarStores in { 1031defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 1032defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 1033defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 1034let OtherPredicates = [HasScalarFlatScratchInsts] in { 1035defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 1036defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 1037defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 1038} // End OtherPredicates = [HasScalarFlatScratchInsts] 1039defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 1040defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 1041defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 1042} // End SubtargetPredicate = HasScalarStores 1043 1044def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 1045def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 1046def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 1047def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 1048def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 1049 1050let SubtargetPredicate = HasScalarStores in { 1051def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 1052} // End SubtargetPredicate = HasScalarStores 1053 1054multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 1055 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1056 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1057} 1058 1059defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 1060defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 1061 1062class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 1063 : SMEM_Real_gfx10 <op, ps>, 1064 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 1065 1066 bits<7> sdata; 1067 1068 let Constraints = ps.Constraints; 1069 let DisableEncoding = ps.DisableEncoding; 1070 1071 let cpol{CPolBit.GLC} = ps.glc; 1072 1073 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1074 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 1075} 1076 1077multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 1078 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1079 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1080 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1081 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1082} 1083 1084let SubtargetPredicate = HasScalarAtomics in { 1085 1086defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1087defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1088defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1089defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1090defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1091defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1092defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1093defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1094defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1095defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1096defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1097defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1098defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1099 1100defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1101defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1102defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1103defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1104defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1105defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1106defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1107defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1108defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1109defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1110defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1111defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1112defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1113 1114defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1115defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1116defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1117defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1118defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1119defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1120defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1121defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1122defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1123defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1124defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1125defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1126defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1127 1128defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1129defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1130defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1131defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1132defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1133defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1134defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1135defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1136defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1137defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1138defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1139defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1140defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1141 1142multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1143 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1144 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1145} 1146 1147defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1148defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1149 1150} // End SubtargetPredicate = HasScalarAtomics 1151 1152def SMInfoTable : GenericTable { 1153 let FilterClass = "SM_Real"; 1154 let CppTypeName = "SMInfo"; 1155 let Fields = ["Opcode", "is_buffer"]; 1156 1157 let PrimaryKey = ["Opcode"]; 1158 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1159} 1160 1161//===----------------------------------------------------------------------===// 1162// GFX11. 1163//===----------------------------------------------------------------------===// 1164 1165class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1166 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11, 1167 SGPR_NULL_gfx11plus> { 1168 let AssemblerPredicate = isGFX11Plus; 1169 let DecoderNamespace = "GFX11"; 1170 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1171 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0); 1172} 1173 1174class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName, dag offsets> : 1175 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName> { 1176 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps).BaseClass; 1177 let InOperandList = !con((ins BaseClass:$sbase), offsets, (ins CPol:$cpol)); 1178} 1179 1180multiclass SM_Real_Loads_gfx11<bits<8> op, string ps, string opName> { 1181 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName, (ins smem_offset:$offset)>; 1182 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName, (ins SReg_32:$soffset)>; 1183 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11< 1184 op, ps#"_SGPR_IMM", opName, (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 1185 def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>, 1186 Requires<[isGFX11Plus]>; 1187} 1188 1189defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD", "s_load_b32">; 1190defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2", "s_load_b64">; 1191defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4", "s_load_b128">; 1192defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8", "s_load_b256">; 1193defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16", "s_load_b512">; 1194 1195defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD", "s_buffer_load_b32">; 1196defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2", "s_buffer_load_b64">; 1197defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4", "s_buffer_load_b128">; 1198defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8", "s_buffer_load_b256">; 1199defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16", "s_buffer_load_b512">; 1200 1201def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>; 1202def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>; 1203 1204class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> { 1205 // encoding 1206 bits<7> sdata; 1207 1208 let sdst = ?; 1209 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1210} 1211 1212multiclass SM_Real_Probe_gfx11<bits<8> op, string ps> { 1213 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1214 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 1215} 1216 1217defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22, "S_ATC_PROBE">; 1218defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23, "S_ATC_PROBE_BUFFER">; 1219