1//===---- SMInstructions.td - Scalar Memory Instruction Defintions --------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 11 NamedMatchClass<"SMRDOffset8">> { 12 let OperandType = "OPERAND_IMMEDIATE"; 13} 14 15def smrd_offset_20 : NamedOperandU32<"SMRDOffset20", 16 NamedMatchClass<"SMRDOffset20">> { 17 let OperandType = "OPERAND_IMMEDIATE"; 18} 19 20//===----------------------------------------------------------------------===// 21// Scalar Memory classes 22//===----------------------------------------------------------------------===// 23 24class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 25 InstSI <outs, ins, "", pattern>, 26 SIMCInstr<opName, SIEncodingFamily.NONE> { 27 let isPseudo = 1; 28 let isCodeGenOnly = 1; 29 30 let LGKM_CNT = 1; 31 let SMRD = 1; 32 let mayStore = 0; 33 let mayLoad = 1; 34 let hasSideEffects = 0; 35 let UseNamedOperandTable = 1; 36 let SchedRW = [WriteSMEM]; 37 let SubtargetPredicate = isGCN; 38 39 string Mnemonic = opName; 40 string AsmOperands = asmOps; 41 42 bits<1> has_sbase = 1; 43 bits<1> has_sdst = 1; 44 bit has_glc = 0; 45 bits<1> has_offset = 1; 46 bits<1> offset_is_imm = 0; 47} 48 49class SM_Real <SM_Pseudo ps> 50 : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 51 52 let isPseudo = 0; 53 let isCodeGenOnly = 0; 54 55 // copy relevant pseudo op flags 56 let SubtargetPredicate = ps.SubtargetPredicate; 57 let AsmMatchConverter = ps.AsmMatchConverter; 58 59 // encoding 60 bits<7> sbase; 61 bits<7> sdst; 62 bits<32> offset; 63 bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0); 64} 65 66class SM_Probe_Pseudo <string opName, dag ins, bit isImm> 67 : SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> { 68 let mayLoad = 0; 69 let mayStore = 0; 70 let has_glc = 0; 71 let LGKM_CNT = 0; 72 let ScalarStore = 0; 73 let hasSideEffects = 1; 74 let offset_is_imm = isImm; 75 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 76} 77 78class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 79 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 80 RegisterClass BaseClass; 81 let mayLoad = 1; 82 let mayStore = 0; 83 let has_glc = 1; 84} 85 86class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []> 87 : SM_Pseudo<opName, (outs), ins, asmOps, pattern> { 88 RegisterClass BaseClass; 89 RegisterClass SrcClass; 90 let mayLoad = 0; 91 let mayStore = 1; 92 let has_glc = 1; 93 let ScalarStore = 1; 94} 95 96class SM_Discard_Pseudo <string opName, dag ins, bit isImm> 97 : SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> { 98 let mayLoad = 0; 99 let mayStore = 0; 100 let has_glc = 0; 101 let has_sdst = 0; 102 let ScalarStore = 0; 103 let hasSideEffects = 1; 104 let offset_is_imm = isImm; 105 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 106} 107 108multiclass SM_Pseudo_Loads<string opName, 109 RegisterClass baseClass, 110 RegisterClass dstClass> { 111 def _IMM : SM_Load_Pseudo <opName, 112 (outs dstClass:$sdst), 113 (ins baseClass:$sbase, i32imm:$offset, i1imm:$glc), 114 " $sdst, $sbase, $offset$glc", []> { 115 let offset_is_imm = 1; 116 let BaseClass = baseClass; 117 let PseudoInstr = opName # "_IMM"; 118 let has_glc = 1; 119 } 120 121 def _SGPR : SM_Load_Pseudo <opName, 122 (outs dstClass:$sdst), 123 (ins baseClass:$sbase, SReg_32:$soff, i1imm:$glc), 124 " $sdst, $sbase, $offset$glc", []> { 125 let BaseClass = baseClass; 126 let PseudoInstr = opName # "_SGPR"; 127 let has_glc = 1; 128 } 129} 130 131multiclass SM_Pseudo_Stores<string opName, 132 RegisterClass baseClass, 133 RegisterClass srcClass> { 134 def _IMM : SM_Store_Pseudo <opName, 135 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, i1imm:$glc), 136 " $sdata, $sbase, $offset$glc", []> { 137 let offset_is_imm = 1; 138 let BaseClass = baseClass; 139 let SrcClass = srcClass; 140 let PseudoInstr = opName # "_IMM"; 141 } 142 143 def _SGPR : SM_Store_Pseudo <opName, 144 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, i1imm:$glc), 145 " $sdata, $sbase, $offset$glc", []> { 146 let BaseClass = baseClass; 147 let SrcClass = srcClass; 148 let PseudoInstr = opName # "_SGPR"; 149 } 150} 151 152multiclass SM_Pseudo_Discards<string opName> { 153 def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smrd_offset_20:$offset), 1>; 154 def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>; 155} 156 157class SM_Time_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 158 opName, (outs SReg_64_XEXEC:$sdst), (ins), 159 " $sdst", [(set i64:$sdst, (node))]> { 160 let hasSideEffects = 1; 161 let mayStore = 0; 162 let mayLoad = 1; 163 let has_sbase = 0; 164 let has_offset = 0; 165} 166 167class SM_Inval_Pseudo <string opName, SDPatternOperator node> : SM_Pseudo< 168 opName, (outs), (ins), "", [(node)]> { 169 let hasSideEffects = 1; 170 let mayStore = 1; 171 let has_sdst = 0; 172 let has_sbase = 0; 173 let has_offset = 0; 174} 175 176multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 177 def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smrd_offset_20:$offset), 1>; 178 def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>; 179} 180 181//===----------------------------------------------------------------------===// 182// Scalar Atomic Memory Classes 183//===----------------------------------------------------------------------===// 184 185class SM_Atomic_Pseudo <string opName, 186 dag outs, dag ins, string asmOps, bit isRet> 187 : SM_Pseudo<opName, outs, ins, asmOps, []> { 188 189 bit glc = isRet; 190 191 let mayLoad = 1; 192 let mayStore = 1; 193 let has_glc = 1; 194 195 // Should these be set? 196 let ScalarStore = 1; 197 let hasSideEffects = 1; 198 let maybeAtomic = 1; 199} 200 201class SM_Pseudo_Atomic<string opName, 202 RegisterClass baseClass, 203 RegisterClass dataClass, 204 bit isImm, 205 bit isRet> : 206 SM_Atomic_Pseudo<opName, 207 !if(isRet, (outs dataClass:$sdst), (outs)), 208 !if(isImm, 209 (ins dataClass:$sdata, baseClass:$sbase, smrd_offset_20:$offset), 210 (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset)), 211 !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", ""), 212 isRet> { 213 let offset_is_imm = isImm; 214 let PseudoInstr = opName # !if(isImm, 215 !if(isRet, "_IMM_RTN", "_IMM"), 216 !if(isRet, "_SGPR_RTN", "_SGPR")); 217 218 let Constraints = !if(isRet, "$sdst = $sdata", ""); 219 let DisableEncoding = !if(isRet, "$sdata", ""); 220} 221 222multiclass SM_Pseudo_Atomics<string opName, 223 RegisterClass baseClass, 224 RegisterClass dataClass> { 225 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>; 226 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>; 227 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>; 228 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>; 229} 230 231//===----------------------------------------------------------------------===// 232// Scalar Memory Instructions 233//===----------------------------------------------------------------------===// 234 235// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 236// SMRD instructions, because the SReg_32_XM0 register class does not include M0 237// and writing to M0 from an SMRD instruction will hang the GPU. 238 239// XXX - SMEM instructions do not allow exec for data operand, but 240// does sdst for SMRD on SI/CI? 241defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 242defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 243defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 244defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 245defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 246 247defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 248 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 249>; 250 251// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 252// SI/CI, bit disallowed for SMEM on VI. 253defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 254 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 255>; 256 257defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 258 "s_buffer_load_dwordx4", SReg_128, SReg_128 259>; 260 261defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 262 "s_buffer_load_dwordx8", SReg_128, SReg_256 263>; 264 265defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 266 "s_buffer_load_dwordx16", SReg_128, SReg_512 267>; 268 269defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 270defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 271defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 272 273defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 274 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 275>; 276 277defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 278 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 279>; 280 281defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 282 "s_buffer_store_dwordx4", SReg_128, SReg_128 283>; 284 285 286def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 287def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 288 289let SubtargetPredicate = isCIVI in { 290def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 291} // let SubtargetPredicate = isCIVI 292 293let SubtargetPredicate = isVI in { 294def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 295def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 296def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 297 298defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 299defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 300} // SubtargetPredicate = isVI 301 302let SubtargetPredicate = HasFlatScratchInsts, Uses = [FLAT_SCR] in { 303defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 304defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 305defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 306 307defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 308defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 309defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 310} // SubtargetPredicate = HasFlatScratchInsts 311 312let SubtargetPredicate = HasScalarAtomics in { 313 314defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 315defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 316defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 317defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 318defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 319defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 320defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 321defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 322defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 323defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 324defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 325defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 326defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 327 328defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 329defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 330defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 331defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 332defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 333defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 334defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 335defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 336defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 337defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 338defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 339defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 340defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 341 342defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 343defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 344defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 345defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 346defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 347defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 348defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 349defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 350defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 351defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 352defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 353defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 354defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 355 356defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 357defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 358defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 359defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 360defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 361defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 362defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 363defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 364defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 365defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 366defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 367defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 368defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 369 370} // let SubtargetPredicate = HasScalarAtomics 371 372let SubtargetPredicate = isGFX9 in { 373defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 374defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 375} 376 377//===----------------------------------------------------------------------===// 378// Targets 379//===----------------------------------------------------------------------===// 380 381//===----------------------------------------------------------------------===// 382// SI 383//===----------------------------------------------------------------------===// 384 385class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 386 : SM_Real<ps> 387 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 388 , Enc32 { 389 390 let AssemblerPredicates = [isSICI]; 391 let DecoderNamespace = "SICI"; 392 393 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 394 let Inst{8} = imm; 395 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 396 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 397 let Inst{26-22} = op; 398 let Inst{31-27} = 0x18; //encoding 399} 400 401// FIXME: Assembler should reject trying to use glc on SMRD 402// instructions on SI. 403multiclass SM_Real_Loads_si<bits<5> op, string ps, 404 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 405 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 406 407 def _IMM_si : SMRD_Real_si <op, immPs> { 408 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, GLC:$glc); 409 } 410 411 // FIXME: The operand name $offset is inconsistent with $soff used 412 // in the pseudo 413 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 414 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc); 415 } 416 417} 418 419defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 420defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 421defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 422defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 423defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 424defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 425defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 426defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 427defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 428defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 429 430def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 431def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 432 433 434//===----------------------------------------------------------------------===// 435// VI 436//===----------------------------------------------------------------------===// 437 438class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 439 : SM_Real<ps> 440 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 441 , Enc64 { 442 bit glc; 443 444 let AssemblerPredicates = [isVI]; 445 let DecoderNamespace = "VI"; 446 447 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 448 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 449 450 let Inst{16} = !if(ps.has_glc, glc, ?); 451 let Inst{17} = imm; 452 let Inst{25-18} = op; 453 let Inst{31-26} = 0x30; //encoding 454 let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?); 455} 456 457multiclass SM_Real_Loads_vi<bits<8> op, string ps, 458 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 459 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 460 def _IMM_vi : SMEM_Real_vi <op, immPs> { 461 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc); 462 } 463 def _SGPR_vi : SMEM_Real_vi <op, sgprPs> { 464 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc); 465 } 466} 467 468class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 469 // encoding 470 bits<7> sdata; 471 472 let sdst = ?; 473 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 474} 475 476multiclass SM_Real_Stores_vi<bits<8> op, string ps, 477 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 478 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 479 // FIXME: The operand name $offset is inconsistent with $soff used 480 // in the pseudo 481 def _IMM_vi : SMEM_Real_Store_vi <op, immPs> { 482 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc); 483 } 484 485 def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> { 486 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc); 487 } 488} 489 490multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 491 def _IMM_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 492 def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 493} 494 495defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 496defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 497defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 498defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 499defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 500defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 501defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 502defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 503defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 504defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 505 506defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 507defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 508defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 509 510defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 511defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 512defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 513 514// These instructions use same encoding 515def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 516def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 517def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 518def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 519def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 520def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 521 522defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 523defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 524defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 525 526defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 527defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 528defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 529 530defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 531defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 532 533//===----------------------------------------------------------------------===// 534// GFX9 535//===----------------------------------------------------------------------===// 536 537class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 538 : SMEM_Real_vi <op, ps> { 539 540 bits<7> sdata; 541 542 let Constraints = ps.Constraints; 543 let DisableEncoding = ps.DisableEncoding; 544 545 let glc = ps.glc; 546 let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0}); 547} 548 549multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 550 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 551 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 552 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 553 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 554} 555 556defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 557defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 558defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 559defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 560defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 561defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 562defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 563defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 564defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 565defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 566defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 567defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 568defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 569 570defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 571defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 572defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 573defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 574defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 575defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 576defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 577defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 578defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 579defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 580defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 581defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 582defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 583 584defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 585defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 586defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 587defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 588defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 589defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 590defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 591defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 592defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 593defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 594defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 595defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 596defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 597 598defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 599defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 600defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 601defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 602defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 603defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 604defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 605defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 606defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 607defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 608defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 609defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 610defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 611 612multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 613 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 614 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 615} 616 617defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 618defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 619 620//===----------------------------------------------------------------------===// 621// CI 622//===----------------------------------------------------------------------===// 623 624def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 625 NamedMatchClass<"SMRDLiteralOffset">> { 626 let OperandType = "OPERAND_IMMEDIATE"; 627} 628 629class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 630 SM_Real<ps>, 631 Enc64 { 632 633 let AssemblerPredicates = [isCIOnly]; 634 let DecoderNamespace = "CI"; 635 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc); 636 637 let LGKM_CNT = ps.LGKM_CNT; 638 let SMRD = ps.SMRD; 639 let mayLoad = ps.mayLoad; 640 let mayStore = ps.mayStore; 641 let hasSideEffects = ps.hasSideEffects; 642 let SchedRW = ps.SchedRW; 643 let UseNamedOperandTable = ps.UseNamedOperandTable; 644 645 let Inst{7-0} = 0xff; 646 let Inst{8} = 0; 647 let Inst{14-9} = sbase{6-1}; 648 let Inst{21-15} = sdst{6-0}; 649 let Inst{26-22} = op; 650 let Inst{31-27} = 0x18; //encoding 651 let Inst{63-32} = offset{31-0}; 652} 653 654def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 655def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 656def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 657def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 658def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 659def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 660def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 661def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 662def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 663def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 664 665class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 666 : SM_Real<ps> 667 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 668 , Enc32 { 669 670 let AssemblerPredicates = [isCIOnly]; 671 let DecoderNamespace = "CI"; 672 673 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 674 let Inst{8} = imm; 675 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 676 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 677 let Inst{26-22} = op; 678 let Inst{31-27} = 0x18; //encoding 679} 680 681def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 682 683//===----------------------------------------------------------------------===// 684// Scalar Memory Patterns 685//===----------------------------------------------------------------------===// 686 687def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]>; 688 689def SMRDImm : ComplexPattern<i64, 2, "SelectSMRDImm">; 690def SMRDImm32 : ComplexPattern<i64, 2, "SelectSMRDImm32">; 691def SMRDSgpr : ComplexPattern<i64, 2, "SelectSMRDSgpr">; 692def SMRDBufferImm : ComplexPattern<i32, 1, "SelectSMRDBufferImm">; 693def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">; 694 695multiclass SMRD_Pattern <string Instr, ValueType vt> { 696 697 // 1. IMM offset 698 def : GCNPat < 699 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 700 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 701 >; 702 703 // 2. 32-bit IMM offset on CI 704 def : GCNPat < 705 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 706 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 707 let OtherPredicates = [isCIOnly]; 708 } 709 710 // 3. SGPR offset 711 def : GCNPat < 712 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 713 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0)) 714 >; 715} 716 717multiclass SMLoad_Pattern <string Instr, ValueType vt> { 718 // 1. Offset as an immediate 719 def : GCNPat < 720 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc), 721 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, (as_i1imm $glc))) 722 >; 723 724 // 2. 32-bit IMM offset on CI 725 def : GCNPat < 726 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc)), 727 (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, (as_i1imm $glc))> { 728 let OtherPredicates = [isCIOnly]; 729 } 730 731 // 3. Offset loaded in an 32bit SGPR 732 def : GCNPat < 733 (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc), 734 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, (as_i1imm $glc))) 735 >; 736} 737 738// Global and constant loads can be selected to either MUBUF or SMRD 739// instructions, but SMRD instructions are faster so we want the instruction 740// selector to prefer those. 741let AddedComplexity = 100 in { 742 743defm : SMRD_Pattern <"S_LOAD_DWORD", i32>; 744defm : SMRD_Pattern <"S_LOAD_DWORDX2", v2i32>; 745defm : SMRD_Pattern <"S_LOAD_DWORDX4", v4i32>; 746defm : SMRD_Pattern <"S_LOAD_DWORDX8", v8i32>; 747defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>; 748 749defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 750defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 751defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 752defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 753defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 754} // End let AddedComplexity = 100 755 756let OtherPredicates = [isSICI] in { 757def : GCNPat < 758 (i64 (readcyclecounter)), 759 (S_MEMTIME) 760>; 761} 762 763let OtherPredicates = [isVI] in { 764 765def : GCNPat < 766 (i64 (readcyclecounter)), 767 (S_MEMREALTIME) 768>; 769 770} // let OtherPredicates = [isVI] 771