1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14def smem_offset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21//===----------------------------------------------------------------------===// 22// Scalar Memory classes 23//===----------------------------------------------------------------------===// 24 25class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 26 InstSI <outs, ins, "", pattern>, 27 SIMCInstr<opName, SIEncodingFamily.NONE> { 28 let isPseudo = 1; 29 let isCodeGenOnly = 1; 30 31 let LGKM_CNT = 1; 32 let SMRD = 1; 33 let mayStore = 0; 34 let mayLoad = 1; 35 let hasSideEffects = 0; 36 let UseNamedOperandTable = 1; 37 let SchedRW = [WriteSMEM]; 38 39 string Mnemonic = opName; 40 string AsmOperands = asmOps; 41 42 bits<1> has_sbase = 1; 43 bits<1> has_sdst = 1; 44 bit has_glc = 0; 45 bit has_dlc = 0; 46 bits<1> has_offset = 1; 47 bits<1> offset_is_imm = 0; 48 bit is_buffer = 0; 49} 50 51class SM_Real <SM_Pseudo ps> 52 : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 53 54 let isPseudo = 0; 55 let isCodeGenOnly = 0; 56 57 Instruction Opcode = !cast<Instruction>(NAME); 58 59 // copy relevant pseudo op flags 60 let SubtargetPredicate = ps.SubtargetPredicate; 61 let AsmMatchConverter = ps.AsmMatchConverter; 62 let UseNamedOperandTable = ps.UseNamedOperandTable; 63 let SMRD = ps.SMRD; 64 65 let TSFlags = ps.TSFlags; 66 67 bit is_buffer = ps.is_buffer; 68 69 // encoding 70 bits<7> sbase; 71 bits<7> sdst; 72 bits<32> offset; 73 bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0); 74} 75 76class SM_Probe_Pseudo <string opName, dag ins, bit isImm> 77 : SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> { 78 let mayLoad = 0; 79 let mayStore = 0; 80 let has_glc = 0; 81 let LGKM_CNT = 0; 82 let ScalarStore = 0; 83 let hasSideEffects = 1; 84 let offset_is_imm = isImm; 85 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 86} 87 88class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 89 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 90 RegisterClass BaseClass; 91 let mayLoad = 1; 92 let mayStore = 0; 93 let has_glc = 1; 94 let has_dlc = 1; 95} 96 97class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []> 98 : SM_Pseudo<opName, (outs), ins, asmOps, pattern> { 99 RegisterClass BaseClass; 100 RegisterClass SrcClass; 101 let mayLoad = 0; 102 let mayStore = 1; 103 let has_glc = 1; 104 let has_dlc = 1; 105 let ScalarStore = 1; 106} 107 108class SM_Discard_Pseudo <string opName, dag ins, bit isImm> 109 : SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> { 110 let mayLoad = 0; 111 let mayStore = 0; 112 let has_glc = 0; 113 let has_sdst = 0; 114 let ScalarStore = 0; 115 let hasSideEffects = 1; 116 let offset_is_imm = isImm; 117 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 118} 119 120multiclass SM_Pseudo_Loads<string opName, 121 RegisterClass baseClass, 122 RegisterClass dstClass> { 123 def _IMM : SM_Load_Pseudo <opName, 124 (outs dstClass:$sdst), 125 (ins baseClass:$sbase, i32imm:$offset, i1imm:$glc, i1imm:$dlc), 126 " $sdst, $sbase, $offset$glc$dlc", []> { 127 let offset_is_imm = 1; 128 let BaseClass = baseClass; 129 let PseudoInstr = opName # "_IMM"; 130 let has_glc = 1; 131 let has_dlc = 1; 132 } 133 134 def _SGPR : SM_Load_Pseudo <opName, 135 (outs dstClass:$sdst), 136 (ins baseClass:$sbase, SReg_32:$soff, i1imm:$glc, i1imm:$dlc), 137 " $sdst, $sbase, $offset$glc$dlc", []> { 138 let BaseClass = baseClass; 139 let PseudoInstr = opName # "_SGPR"; 140 let has_glc = 1; 141 let has_dlc = 1; 142 } 143} 144 145multiclass SM_Pseudo_Stores<string opName, 146 RegisterClass baseClass, 147 RegisterClass srcClass> { 148 def _IMM : SM_Store_Pseudo <opName, 149 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, i1imm:$glc, i1imm:$dlc), 150 " $sdata, $sbase, $offset$glc$dlc", []> { 151 let offset_is_imm = 1; 152 let BaseClass = baseClass; 153 let SrcClass = srcClass; 154 let PseudoInstr = opName # "_IMM"; 155 } 156 157 def _SGPR : SM_Store_Pseudo <opName, 158 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, i1imm:$glc, i1imm:$dlc), 159 " $sdata, $sbase, $offset$glc$dlc", []> { 160 let BaseClass = baseClass; 161 let SrcClass = srcClass; 162 let PseudoInstr = opName # "_SGPR"; 163 } 164} 165 166multiclass SM_Pseudo_Discards<string opName> { 167 def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>; 168 def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>; 169} 170 171class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 172 opName, (outs SReg_64_XEXEC:$sdst), (ins), 173 " $sdst", [(set i64:$sdst, (node))]> { 174 let hasSideEffects = 1; 175 176 // FIXME: This should be definitively mayStore = 0. TableGen 177 // brokenly tries to infer these based on the intrinsic properties 178 // corresponding to the IR attributes. The target intrinsics are 179 // considered as writing to memory for IR dependency purposes, but 180 // those can be modeled with hasSideEffects here. These also end up 181 // inferring differently for llvm.readcyclecounter and the amdgcn 182 // intrinsics. 183 let mayStore = ?; 184 let mayLoad = 1; 185 let has_sbase = 0; 186 let has_offset = 0; 187} 188 189class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 190 opName, (outs), (ins), "", [(node)]> { 191 let hasSideEffects = 1; 192 let mayStore = 0; 193 let has_sdst = 0; 194 let has_sbase = 0; 195 let has_offset = 0; 196} 197 198multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 199 def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>; 200 def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>; 201} 202 203class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 204 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 205 " $sdst", [(set i32:$sdst, (node))]> { 206 let hasSideEffects = 1; 207 let mayStore = 0; 208 let mayLoad = 1; 209 let has_sbase = 0; 210 let has_offset = 0; 211} 212 213//===----------------------------------------------------------------------===// 214// Scalar Atomic Memory Classes 215//===----------------------------------------------------------------------===// 216 217class SM_Atomic_Pseudo <string opName, 218 dag outs, dag ins, string asmOps, bit isRet> 219 : SM_Pseudo<opName, outs, ins, asmOps, []> { 220 221 bit glc = isRet; 222 223 let mayLoad = 1; 224 let mayStore = 1; 225 let has_glc = 1; 226 let has_dlc = 1; 227 228 // Should these be set? 229 let ScalarStore = 1; 230 let hasSideEffects = 1; 231 let maybeAtomic = 1; 232 233 let IsAtomicNoRet = !not(isRet); 234 let IsAtomicRet = isRet; 235} 236 237class SM_Pseudo_Atomic<string opName, 238 RegisterClass baseClass, 239 RegisterClass dataClass, 240 bit isImm, 241 bit isRet, 242 string opNameWithSuffix = opName # !if(isImm, 243 !if(isRet, "_IMM_RTN", "_IMM"), 244 !if(isRet, "_SGPR_RTN", "_SGPR"))> : 245 SM_Atomic_Pseudo<opName, 246 !if(isRet, (outs dataClass:$sdst), (outs)), 247 !if(isImm, 248 (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, DLC:$dlc), 249 (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, DLC:$dlc)), 250 !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", "") # "$dlc", 251 isRet>, 252 AtomicNoRet <opNameWithSuffix, isRet> { 253 let offset_is_imm = isImm; 254 let PseudoInstr = opNameWithSuffix; 255 256 let Constraints = !if(isRet, "$sdst = $sdata", ""); 257 let DisableEncoding = !if(isRet, "$sdata", ""); 258} 259 260multiclass SM_Pseudo_Atomics<string opName, 261 RegisterClass baseClass, 262 RegisterClass dataClass> { 263 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>; 264 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>; 265 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>; 266 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>; 267} 268 269//===----------------------------------------------------------------------===// 270// Scalar Memory Instructions 271//===----------------------------------------------------------------------===// 272 273// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 274// SMRD instructions, because the SReg_32_XM0 register class does not include M0 275// and writing to M0 from an SMRD instruction will hang the GPU. 276 277// XXX - SMEM instructions do not allow exec for data operand, but 278// does sdst for SMRD on SI/CI? 279defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 280defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 281defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 282defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 283defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 284 285let is_buffer = 1 in { 286defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 287 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 288>; 289 290// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 291// SI/CI, bit disallowed for SMEM on VI. 292defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 293 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 294>; 295 296defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 297 "s_buffer_load_dwordx4", SReg_128, SReg_128 298>; 299 300defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 301 "s_buffer_load_dwordx8", SReg_128, SReg_256 302>; 303 304defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 305 "s_buffer_load_dwordx16", SReg_128, SReg_512 306>; 307} 308 309let SubtargetPredicate = HasScalarStores in { 310defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 311defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 312defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 313 314let is_buffer = 1 in { 315defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 316 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 317>; 318 319defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 320 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 321>; 322 323defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 324 "s_buffer_store_dwordx4", SReg_128, SReg_128 325>; 326} 327} // End SubtargetPredicate = HasScalarStores 328 329let SubtargetPredicate = HasSMemTimeInst in 330def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 331def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 332 333let SubtargetPredicate = isGFX7GFX8GFX9 in { 334def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 335} // let SubtargetPredicate = isGFX7GFX8GFX9 336 337let SubtargetPredicate = isGFX8Plus in { 338let OtherPredicates = [HasScalarStores] in { 339def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 340def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 341} // End OtherPredicates = [HasScalarStores] 342 343defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 344let is_buffer = 1 in { 345defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 346} 347} // SubtargetPredicate = isGFX8Plus 348 349let SubtargetPredicate = HasSMemRealTime in 350def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 351 352let SubtargetPredicate = isGFX10Plus in 353def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 354let SubtargetPredicate = HasGetWaveIdInst in 355def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 356 357 358let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 359defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 360defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 361defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 362 363defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 364defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 365defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 366} // SubtargetPredicate = HasScalarFlatScratchInsts 367 368let SubtargetPredicate = HasScalarAtomics in { 369 370let is_buffer = 1 in { 371defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 372defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 373defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 374defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 375defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 376defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 377defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 378defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 379defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 380defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 381defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 382defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 383defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 384 385defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 386defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 387defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 388defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 389defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 390defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 391defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 392defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 393defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 394defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 395defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 396defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 397defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 398} 399 400defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 401defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 402defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 403defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 404defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 405defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 406defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 407defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 408defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 409defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 410defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 411defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 412defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 413 414defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 415defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 416defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 417defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 418defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 419defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 420defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 421defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 422defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 423defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 424defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 425defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 426defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 427 428} // let SubtargetPredicate = HasScalarAtomics 429 430let SubtargetPredicate = HasScalarAtomics in { 431defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 432defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 433} 434 435//===----------------------------------------------------------------------===// 436// Targets 437//===----------------------------------------------------------------------===// 438 439//===----------------------------------------------------------------------===// 440// SI 441//===----------------------------------------------------------------------===// 442 443class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 444 : SM_Real<ps> 445 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 446 , Enc32 { 447 448 let AssemblerPredicate = isGFX6GFX7; 449 let DecoderNamespace = "GFX6GFX7"; 450 451 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 452 let Inst{8} = imm; 453 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 454 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 455 let Inst{26-22} = op; 456 let Inst{31-27} = 0x18; //encoding 457} 458 459// FIXME: Assembler should reject trying to use glc on SMRD 460// instructions on SI. 461multiclass SM_Real_Loads_si<bits<5> op, string ps, 462 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 463 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 464 465 def _IMM_si : SMRD_Real_si <op, immPs> { 466 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, GLC:$glc, DLC:$dlc); 467 } 468 469 // FIXME: The operand name $offset is inconsistent with $soff used 470 // in the pseudo 471 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 472 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 473 } 474 475} 476 477defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 478defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 479defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 480defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 481defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 482defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 483defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 484defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 485defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 486defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 487 488def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 489def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 490 491 492//===----------------------------------------------------------------------===// 493// VI 494//===----------------------------------------------------------------------===// 495 496class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 497 : SM_Real<ps> 498 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 499 , Enc64 { 500 bit glc; 501 502 let AssemblerPredicate = isGFX8GFX9; 503 let DecoderNamespace = "GFX8"; 504 505 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 506 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 507 508 let Inst{16} = !if(ps.has_glc, glc, ?); 509 let Inst{17} = imm; 510 let Inst{25-18} = op; 511 let Inst{31-26} = 0x30; //encoding 512 513 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 514 // Offset value is corrected accordingly when offset is encoded/decoded. 515 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, ?); 516} 517 518multiclass SM_Real_Loads_vi<bits<8> op, string ps, 519 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 520 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 521 def _IMM_vi : SMEM_Real_vi <op, immPs> { 522 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 523 } 524 def _SGPR_vi : SMEM_Real_vi <op, sgprPs> { 525 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 526 } 527} 528 529class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 530 // encoding 531 bits<7> sdata; 532 533 let sdst = ?; 534 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 535} 536 537multiclass SM_Real_Stores_vi<bits<8> op, string ps, 538 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 539 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 540 // FIXME: The operand name $offset is inconsistent with $soff used 541 // in the pseudo 542 def _IMM_vi : SMEM_Real_Store_vi <op, immPs> { 543 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 544 } 545 546 def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> { 547 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 548 } 549} 550 551multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 552 def _IMM_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 553 def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 554} 555 556defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 557defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 558defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 559defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 560defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 561defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 562defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 563defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 564defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 565defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 566 567defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 568defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 569defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 570 571defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 572defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 573defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 574 575// These instructions use same encoding 576def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 577def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 578def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 579def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 580def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 581def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 582 583defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 584defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 585defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 586 587defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 588defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 589defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 590 591defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 592defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 593 594//===----------------------------------------------------------------------===// 595// GFX9 596//===----------------------------------------------------------------------===// 597 598class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 599 : SMEM_Real_vi <op, ps>, 600 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 601 602 bits<7> sdata; 603 604 let Constraints = ps.Constraints; 605 let DisableEncoding = ps.DisableEncoding; 606 607 let glc = ps.glc; 608 let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0}); 609} 610 611multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 612 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 613 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 614 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 615 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 616} 617 618defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 619defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 620defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 621defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 622defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 623defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 624defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 625defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 626defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 627defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 628defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 629defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 630defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 631 632defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 633defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 634defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 635defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 636defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 637defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 638defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 639defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 640defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 641defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 642defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 643defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 644defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 645 646defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 647defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 648defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 649defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 650defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 651defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 652defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 653defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 654defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 655defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 656defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 657defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 658defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 659 660defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 661defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 662defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 663defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 664defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 665defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 666defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 667defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 668defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 669defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 670defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 671defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 672defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 673 674multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 675 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 676 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 677} 678 679defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 680defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 681 682//===----------------------------------------------------------------------===// 683// CI 684//===----------------------------------------------------------------------===// 685 686def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 687 NamedMatchClass<"SMRDLiteralOffset">> { 688 let OperandType = "OPERAND_IMMEDIATE"; 689} 690 691class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 692 SM_Real<ps>, 693 Enc64 { 694 695 let AssemblerPredicate = isGFX7Only; 696 let DecoderNamespace = "GFX7"; 697 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc, DLC:$dlc); 698 699 let LGKM_CNT = ps.LGKM_CNT; 700 let mayLoad = ps.mayLoad; 701 let mayStore = ps.mayStore; 702 let hasSideEffects = ps.hasSideEffects; 703 let SchedRW = ps.SchedRW; 704 705 let Inst{7-0} = 0xff; 706 let Inst{8} = 0; 707 let Inst{14-9} = sbase{6-1}; 708 let Inst{21-15} = sdst{6-0}; 709 let Inst{26-22} = op; 710 let Inst{31-27} = 0x18; //encoding 711 let Inst{63-32} = offset{31-0}; 712} 713 714def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 715def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 716def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 717def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 718def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 719def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 720def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 721def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 722def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 723def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 724 725class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 726 : SM_Real<ps> 727 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 728 , Enc32 { 729 730 let AssemblerPredicate = isGFX7Only; 731 let DecoderNamespace = "GFX7"; 732 733 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 734 let Inst{8} = imm; 735 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 736 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 737 let Inst{26-22} = op; 738 let Inst{31-27} = 0x18; //encoding 739} 740 741def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 742 743//===----------------------------------------------------------------------===// 744// Scalar Memory Patterns 745//===----------------------------------------------------------------------===// 746 747def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 748 let GISelPredicateCode = [{ 749 if (!MI.hasOneMemOperand()) 750 return false; 751 if (!isInstrUniform(MI)) 752 return false; 753 754 // FIXME: We should probably be caching this. 755 SmallVector<GEPInfo, 4> AddrInfo; 756 getAddrModeInfo(MI, MRI, AddrInfo); 757 758 if (hasVgprParts(AddrInfo)) 759 return false; 760 return true; 761 }]; 762} 763 764def SMRDImm : ComplexPattern<i64, 2, "SelectSMRDImm">; 765def SMRDImm32 : ComplexPattern<i64, 2, "SelectSMRDImm32">; 766def SMRDSgpr : ComplexPattern<i64, 2, "SelectSMRDSgpr">; 767def SMRDBufferImm : ComplexPattern<i32, 1, "SelectSMRDBufferImm">; 768def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">; 769 770multiclass SMRD_Pattern <string Instr, ValueType vt> { 771 772 // 1. IMM offset 773 def : GCNPat < 774 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 775 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0, 0)) 776 >; 777 778 // 2. 32-bit IMM offset on CI 779 def : GCNPat < 780 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 781 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0, 0))> { 782 let OtherPredicates = [isGFX7Only]; 783 } 784 785 // 3. SGPR offset 786 def : GCNPat < 787 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 788 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0, 0)) 789 >; 790 791 // 4. No offset 792 def : GCNPat < 793 (vt (smrd_load (i64 SReg_64:$sbase))), 794 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0, 0)) 795 >; 796} 797 798multiclass SMLoad_Pattern <string Instr, ValueType vt> { 799 // 1. Offset as an immediate 800 def : GCNPat < 801 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 802 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_glc $cachepolicy), 803 (extract_dlc $cachepolicy)))> { 804 let AddedComplexity = 2; 805 } 806 807 // 2. 32-bit IMM offset on CI 808 def : GCNPat < 809 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 810 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 811 (extract_glc $cachepolicy), (extract_dlc $cachepolicy))> { 812 let OtherPredicates = [isGFX7Only]; 813 let AddedComplexity = 1; 814 } 815 816 // 3. Offset loaded in an 32bit SGPR 817 def : GCNPat < 818 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 819 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_glc $cachepolicy), 820 (extract_dlc $cachepolicy))) 821 >; 822} 823 824// Global and constant loads can be selected to either MUBUF or SMRD 825// instructions, but SMRD instructions are faster so we want the instruction 826// selector to prefer those. 827let AddedComplexity = 100 in { 828 829foreach vt = Reg32Types.types in { 830defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 831} 832 833foreach vt = SReg_64.RegTypes in { 834defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 835} 836 837foreach vt = SReg_128.RegTypes in { 838defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 839} 840 841foreach vt = SReg_256.RegTypes in { 842defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 843} 844 845foreach vt = SReg_512.RegTypes in { 846defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 847} 848 849defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 850defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 851defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 852defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 853defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 854 855defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 856defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 857defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 858defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 859defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 860} // End let AddedComplexity = 100 861 862let OtherPredicates = [HasSMemTimeInst] in { 863def : GCNPat < 864 (i64 (readcyclecounter)), 865 (S_MEMTIME) 866>; 867} // let OtherPredicates = [HasSMemTimeInst] 868 869let OtherPredicates = [HasNoSMemTimeInst] in { 870def : GCNPat < 871 (i64 (readcyclecounter)), 872 (REG_SEQUENCE SReg_64, 873 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 874 (S_MOV_B32 (i32 0)), sub1) 875>; 876} // let OtherPredicates = [HasNoSMemTimeInst] 877 878//===----------------------------------------------------------------------===// 879// GFX10. 880//===----------------------------------------------------------------------===// 881 882class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> : 883 SM_Real<ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10>, Enc64 { 884 bit glc; 885 bit dlc; 886 887 let AssemblerPredicate = isGFX10Plus; 888 let DecoderNamespace = "GFX10"; 889 890 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 891 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 892 let Inst{14} = !if(ps.has_dlc, dlc, ?); 893 let Inst{16} = !if(ps.has_glc, glc, ?); 894 let Inst{25-18} = op; 895 let Inst{31-26} = 0x3d; 896 let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?); 897 let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding), 898 !if(ps.has_offset, offset{6-0}, ?)); 899} 900 901multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, 902 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 903 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 904 def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { 905 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 906 } 907 def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { 908 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 909 } 910} 911 912class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 913 bits<7> sdata; 914 915 let sdst = ?; 916 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 917} 918 919multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 920 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 921 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 922 // FIXME: The operand name $offset is inconsistent with $soff used 923 // in the pseudo 924 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 925 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 926 } 927 928 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 929 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 930 } 931} 932 933defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 934defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 935defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 936defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 937defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 938 939let SubtargetPredicate = HasScalarFlatScratchInsts in { 940defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 941defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 942defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 943} // End SubtargetPredicate = HasScalarFlatScratchInsts 944 945defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 946defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 947defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 948defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 949defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 950 951let SubtargetPredicate = HasScalarStores in { 952defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 953defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 954defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 955let OtherPredicates = [HasScalarFlatScratchInsts] in { 956defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 957defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 958defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 959} // End OtherPredicates = [HasScalarFlatScratchInsts] 960defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 961defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 962defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 963} // End SubtargetPredicate = HasScalarStores 964 965def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 966def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 967def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 968def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 969def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 970 971let SubtargetPredicate = HasScalarStores in { 972def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 973} // End SubtargetPredicate = HasScalarStores 974 975multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 976 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 977 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 978} 979 980defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 981defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 982 983class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 984 : SMEM_Real_gfx10 <op, ps>, 985 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 986 987 bits<7> sdata; 988 bit dlc; 989 990 let Constraints = ps.Constraints; 991 let DisableEncoding = ps.DisableEncoding; 992 993 let glc = ps.glc; 994 995 let Inst{14} = !if(ps.has_dlc, dlc, 0); 996 let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0}); 997} 998 999multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 1000 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1001 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1002 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1003 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1004} 1005 1006let SubtargetPredicate = HasScalarAtomics in { 1007 1008defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1009defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1010defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1011defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1012defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1013defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1014defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1015defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1016defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1017defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1018defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1019defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1020defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1021 1022defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1023defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1024defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1025defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1026defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1027defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1028defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1029defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1030defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1031defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1032defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1033defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1034defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1035 1036defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1037defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1038defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1039defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1040defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1041defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1042defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1043defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1044defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1045defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1046defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1047defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1048defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1049 1050defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1051defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1052defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1053defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1054defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1055defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1056defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1057defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1058defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1059defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1060defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1061defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1062defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1063 1064multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1065 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1066 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1067} 1068 1069defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1070defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1071 1072} // End SubtargetPredicate = HasScalarAtomics 1073 1074def SMInfoTable : GenericTable { 1075 let FilterClass = "SM_Real"; 1076 let CppTypeName = "SMInfo"; 1077 let Fields = ["Opcode", "is_buffer"]; 1078 1079 let PrimaryKey = ["Opcode"]; 1080 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1081} 1082