1/* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23/* To compile this assembly code: 24 * 25 * Navi1x: 26 * cpp -DASIC_FAMILY=CHIP_NAVI10 cwsr_trap_handler_gfx10.asm -P -o nv1x.sp3 27 * sp3 nv1x.sp3 -hex nv1x.hex 28 * 29 * gfx10: 30 * cpp -DASIC_FAMILY=CHIP_SIENNA_CICHLID cwsr_trap_handler_gfx10.asm -P -o gfx10.sp3 31 * sp3 gfx10.sp3 -hex gfx10.hex 32 * 33 * gfx11: 34 * cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3 35 * sp3 gfx11.sp3 -hex gfx11.hex 36 * 37 * gfx12: 38 * cpp -DASIC_FAMILY=CHIP_GFX12 cwsr_trap_handler_gfx10.asm -P -o gfx12.sp3 39 * sp3 gfx12.sp3 -hex gfx12.hex 40 */ 41 42#define CHIP_NAVI10 26 43#define CHIP_SIENNA_CICHLID 30 44#define CHIP_PLUM_BONITO 36 45#define CHIP_GFX12 37 46 47#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID) 48#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID) 49#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO) 50#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO) 51#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO && ASIC_FAMILY < CHIP_GFX12) 52#define SAVE_AFTER_XNACK_ERROR (HAVE_XNACK && !NO_SQC_STORE) // workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger 53#define SINGLE_STEP_MISSED_WORKAROUND 1 //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised 54 55#if ASIC_FAMILY < CHIP_GFX12 56#define S_COHERENCE glc:1 57#define V_COHERENCE slc:1 glc:1 58#else 59#define S_COHERENCE scope:SCOPE_SYS 60#define V_COHERENCE scope:SCOPE_SYS 61 62#define HW_REG_SHADER_FLAT_SCRATCH_LO HW_REG_WAVE_SCRATCH_BASE_LO 63#define HW_REG_SHADER_FLAT_SCRATCH_HI HW_REG_WAVE_SCRATCH_BASE_HI 64#define HW_REG_GPR_ALLOC HW_REG_WAVE_GPR_ALLOC 65#define HW_REG_LDS_ALLOC HW_REG_WAVE_LDS_ALLOC 66#define HW_REG_MODE HW_REG_WAVE_MODE 67#endif 68 69#if ASIC_FAMILY < CHIP_GFX12 70var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006 71var SQ_WAVE_STATUS_HALT_MASK = 0x2000 72var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x20000 73var SQ_WAVE_STATUS_TRAP_EN_SHIFT = 6 74var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11 75var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1 76var S_STATUS_HWREG = HW_REG_STATUS 77var S_STATUS_ALWAYS_CLEAR_MASK = SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK 78var S_STATUS_HALT_MASK = SQ_WAVE_STATUS_HALT_MASK 79var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000 80var S_SAVE_PC_HI_HT_MASK = 0x01000000 81#else 82var SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK = 0xC00 83var SQ_WAVE_STATE_PRIV_HALT_MASK = 0x4000 84var SQ_WAVE_STATE_PRIV_POISON_ERR_MASK = 0x8000 85var SQ_WAVE_STATUS_WAVE64_SHIFT = 29 86var SQ_WAVE_STATUS_WAVE64_SIZE = 1 87var S_STATUS_HWREG = HW_REG_WAVE_STATE_PRIV 88var S_STATUS_ALWAYS_CLEAR_MASK = SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK 89var S_STATUS_HALT_MASK = SQ_WAVE_STATE_PRIV_HALT_MASK 90var S_SAVE_PC_HI_TRAP_ID_MASK = 0xF0000000 91#endif 92 93var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12 94var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9 95var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8 96var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24 97var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4 98 99#if ASIC_FAMILY < CHIP_PLUM_BONITO 100var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8 101#else 102var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12 103#endif 104 105#if ASIC_FAMILY < CHIP_GFX12 106var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400 107var SQ_WAVE_TRAPSTS_EXCP_MASK = 0x1FF 108var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10 109var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK = 0x80 110var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT = 7 111var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100 112var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8 113var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x800 114var SQ_WAVE_TRAPSTS_EXCP_HI_MASK = 0x7000 115#if ASIC_FAMILY >= CHIP_PLUM_BONITO 116var SQ_WAVE_TRAPSTS_WAVE_START_MASK = 0x20000 117var SQ_WAVE_TRAPSTS_WAVE_END_MASK = 0x40000 118var SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK = 0x100000 119#endif 120var SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK = 0x10000000 121 122var SQ_WAVE_MODE_EXCP_EN_SHIFT = 12 123var SQ_WAVE_MODE_EXCP_EN_ADDR_WATCH_SHIFT = 19 124 125var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15 126var SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT = 25 127var SQ_WAVE_IB_STS_REPLAY_W64H_MASK = 0x02000000 128var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK = 0x003F8000 129 130var SQ_WAVE_MODE_DEBUG_EN_MASK = 0x800 131 132#if ASIC_FAMILY < CHIP_PLUM_BONITO 133var S_TRAPSTS_NON_MASKABLE_EXCP_MASK = SQ_WAVE_TRAPSTS_MEM_VIOL_MASK|SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK 134#else 135var S_TRAPSTS_NON_MASKABLE_EXCP_MASK = SQ_WAVE_TRAPSTS_MEM_VIOL_MASK |\ 136 SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK |\ 137 SQ_WAVE_TRAPSTS_WAVE_START_MASK |\ 138 SQ_WAVE_TRAPSTS_WAVE_END_MASK |\ 139 SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK 140#endif 141var S_TRAPSTS_HWREG = HW_REG_TRAPSTS 142var S_TRAPSTS_SAVE_CONTEXT_MASK = SQ_WAVE_TRAPSTS_SAVECTX_MASK 143var S_TRAPSTS_SAVE_CONTEXT_SHIFT = SQ_WAVE_TRAPSTS_SAVECTX_SHIFT 144#else 145var SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK = 0xF 146var SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK = 0x10 147var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT = 5 148var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK = 0x20 149var SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK = 0x40 150var SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK = 0x80 151var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_MASK = 0x100 152var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_MASK = 0x200 153var SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_MASK = 0x800 154var SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK = 0x80 155var SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST_MASK = 0x200 156 157var S_TRAPSTS_HWREG = HW_REG_WAVE_EXCP_FLAG_PRIV 158var S_TRAPSTS_SAVE_CONTEXT_MASK = SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK 159var S_TRAPSTS_SAVE_CONTEXT_SHIFT = SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT 160var S_TRAPSTS_NON_MASKABLE_EXCP_MASK = SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK |\ 161 SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK |\ 162 SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK |\ 163 SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_MASK |\ 164 SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_MASK |\ 165 SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_MASK 166var BARRIER_STATE_SIGNAL_OFFSET = 16 167var BARRIER_STATE_VALID_OFFSET = 0 168#endif 169 170// bits [31:24] unused by SPI debug data 171var TTMP11_SAVE_REPLAY_W64H_SHIFT = 31 172var TTMP11_SAVE_REPLAY_W64H_MASK = 0x80000000 173var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT = 24 174var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK = 0x7F000000 175var TTMP11_DEBUG_TRAP_ENABLED_SHIFT = 23 176var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000 177 178// SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] 179// when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE 180var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000 181var S_SAVE_BUF_RSRC_WORD3_MISC = 0x10807FAC 182var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 183var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26 184 185var S_SAVE_PC_HI_FIRST_WAVE_MASK = 0x80000000 186var S_SAVE_PC_HI_FIRST_WAVE_SHIFT = 31 187 188var s_sgpr_save_num = 108 189 190var s_save_spi_init_lo = exec_lo 191var s_save_spi_init_hi = exec_hi 192var s_save_pc_lo = ttmp0 193var s_save_pc_hi = ttmp1 194var s_save_exec_lo = ttmp2 195var s_save_exec_hi = ttmp3 196var s_save_status = ttmp12 197var s_save_trapsts = ttmp15 198var s_save_xnack_mask = s_save_trapsts 199var s_wave_size = ttmp7 200var s_save_buf_rsrc0 = ttmp8 201var s_save_buf_rsrc1 = ttmp9 202var s_save_buf_rsrc2 = ttmp10 203var s_save_buf_rsrc3 = ttmp11 204var s_save_mem_offset = ttmp4 205var s_save_alloc_size = s_save_trapsts 206var s_save_tmp = ttmp14 207var s_save_m0 = ttmp5 208var s_save_ttmps_lo = s_save_tmp 209var s_save_ttmps_hi = s_save_trapsts 210 211var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE 212var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC 213 214var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 215var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26 216var S_WAVE_SIZE = 25 217 218var s_restore_spi_init_lo = exec_lo 219var s_restore_spi_init_hi = exec_hi 220var s_restore_mem_offset = ttmp12 221var s_restore_alloc_size = ttmp3 222var s_restore_tmp = ttmp2 223var s_restore_mem_offset_save = s_restore_tmp 224var s_restore_m0 = s_restore_alloc_size 225var s_restore_mode = ttmp7 226var s_restore_flat_scratch = s_restore_tmp 227var s_restore_pc_lo = ttmp0 228var s_restore_pc_hi = ttmp1 229var s_restore_exec_lo = ttmp4 230var s_restore_exec_hi = ttmp5 231var s_restore_status = ttmp14 232var s_restore_trapsts = ttmp15 233var s_restore_xnack_mask = ttmp13 234var s_restore_buf_rsrc0 = ttmp8 235var s_restore_buf_rsrc1 = ttmp9 236var s_restore_buf_rsrc2 = ttmp10 237var s_restore_buf_rsrc3 = ttmp11 238var s_restore_size = ttmp6 239var s_restore_ttmps_lo = s_restore_tmp 240var s_restore_ttmps_hi = s_restore_alloc_size 241var s_restore_spi_init_hi_save = s_restore_exec_hi 242 243shader main 244 asic(DEFAULT) 245 type(CS) 246 wave_size(32) 247 248 s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save 249 250L_JUMP_TO_RESTORE: 251 s_branch L_RESTORE 252 253L_SKIP_RESTORE: 254 s_getreg_b32 s_save_status, hwreg(S_STATUS_HWREG) //save STATUS since we will change SCC 255 256 // Clear SPI_PRIO: do not save with elevated priority. 257 // Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd. 258 s_andn2_b32 s_save_status, s_save_status, S_STATUS_ALWAYS_CLEAR_MASK 259 260 s_getreg_b32 s_save_trapsts, hwreg(S_TRAPSTS_HWREG) 261 262#if SW_SA_TRAP 263 // If ttmp1[30] is set then issue s_barrier to unblock dependent waves. 264 s_bitcmp1_b32 s_save_pc_hi, 30 265 s_cbranch_scc0 L_TRAP_NO_BARRIER 266 s_barrier 267 268L_TRAP_NO_BARRIER: 269 // If ttmp1[31] is set then trap may occur early. 270 // Spin wait until SAVECTX exception is raised. 271 s_bitcmp1_b32 s_save_pc_hi, 31 272 s_cbranch_scc1 L_CHECK_SAVE 273#endif 274 275 s_and_b32 ttmp2, s_save_status, S_STATUS_HALT_MASK 276 s_cbranch_scc0 L_NOT_HALTED 277 278L_HALTED: 279 // Host trap may occur while wave is halted. 280#if ASIC_FAMILY < CHIP_GFX12 281 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK 282#else 283 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK 284#endif 285 s_cbranch_scc1 L_FETCH_2ND_TRAP 286 287L_CHECK_SAVE: 288 s_and_b32 ttmp2, s_save_trapsts, S_TRAPSTS_SAVE_CONTEXT_MASK 289 s_cbranch_scc1 L_SAVE 290 291 // Wave is halted but neither host trap nor SAVECTX is raised. 292 // Caused by instruction fetch memory violation. 293 // Spin wait until context saved to prevent interrupt storm. 294 s_sleep 0x10 295 s_getreg_b32 s_save_trapsts, hwreg(S_TRAPSTS_HWREG) 296 s_branch L_CHECK_SAVE 297 298L_NOT_HALTED: 299 // Let second-level handle non-SAVECTX exception or trap. 300 // Any concurrent SAVECTX will be handled upon re-entry once halted. 301 302 // Check non-maskable exceptions. memory_violation, illegal_instruction 303 // and xnack_error exceptions always cause the wave to enter the trap 304 // handler. 305 s_and_b32 ttmp2, s_save_trapsts, S_TRAPSTS_NON_MASKABLE_EXCP_MASK 306 s_cbranch_scc1 L_FETCH_2ND_TRAP 307 308 // Check for maskable exceptions in trapsts.excp and trapsts.excp_hi. 309 // Maskable exceptions only cause the wave to enter the trap handler if 310 // their respective bit in mode.excp_en is set. 311#if ASIC_FAMILY < CHIP_GFX12 312 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK 313 s_cbranch_scc0 L_CHECK_TRAP_ID 314 315 s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK 316 s_cbranch_scc0 L_NOT_ADDR_WATCH 317 s_bitset1_b32 ttmp2, SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT // Check all addr_watch[123] exceptions against excp_en.addr_watch 318 319L_NOT_ADDR_WATCH: 320 s_getreg_b32 ttmp3, hwreg(HW_REG_MODE) 321 s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT 322 s_and_b32 ttmp2, ttmp2, ttmp3 323 s_cbranch_scc1 L_FETCH_2ND_TRAP 324#else 325 s_getreg_b32 ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_USER) 326 s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK 327 s_cbranch_scc0 L_NOT_ADDR_WATCH 328 s_or_b32 ttmp2, ttmp2, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK 329 330L_NOT_ADDR_WATCH: 331 s_getreg_b32 ttmp3, hwreg(HW_REG_WAVE_TRAP_CTRL) 332 s_and_b32 ttmp2, ttmp3, ttmp2 333 s_cbranch_scc1 L_FETCH_2ND_TRAP 334#endif 335 336L_CHECK_TRAP_ID: 337 // Check trap_id != 0 338 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK 339 s_cbranch_scc1 L_FETCH_2ND_TRAP 340 341#if SINGLE_STEP_MISSED_WORKAROUND 342 // Prioritize single step exception over context save. 343 // Second-level trap will halt wave and RFE, re-entering for SAVECTX. 344#if ASIC_FAMILY < CHIP_GFX12 345 s_getreg_b32 ttmp2, hwreg(HW_REG_MODE) 346 s_and_b32 ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK 347#else 348 // WAVE_TRAP_CTRL is already in ttmp3. 349 s_and_b32 ttmp3, ttmp3, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST_MASK 350#endif 351 s_cbranch_scc1 L_FETCH_2ND_TRAP 352#endif 353 354 s_and_b32 ttmp2, s_save_trapsts, S_TRAPSTS_SAVE_CONTEXT_MASK 355 s_cbranch_scc1 L_SAVE 356 357L_FETCH_2ND_TRAP: 358#if HAVE_XNACK 359 save_and_clear_ib_sts(ttmp14, ttmp15) 360#endif 361 362 // Read second-level TBA/TMA from first-level TMA and jump if available. 363 // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data) 364 // ttmp12 holds SQ_WAVE_STATUS 365#if HAVE_SENDMSG_RTN 366 s_sendmsg_rtn_b64 [ttmp14, ttmp15], sendmsg(MSG_RTN_GET_TMA) 367 s_waitcnt lgkmcnt(0) 368#else 369 s_getreg_b32 ttmp14, hwreg(HW_REG_SHADER_TMA_LO) 370 s_getreg_b32 ttmp15, hwreg(HW_REG_SHADER_TMA_HI) 371#endif 372 s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 373 374 s_bitcmp1_b32 ttmp15, 0xF 375 s_cbranch_scc0 L_NO_SIGN_EXTEND_TMA 376 s_or_b32 ttmp15, ttmp15, 0xFFFF0000 377L_NO_SIGN_EXTEND_TMA: 378 379 s_load_dword ttmp2, [ttmp14, ttmp15], 0x10 S_COHERENCE // debug trap enabled flag 380 s_waitcnt lgkmcnt(0) 381 s_lshl_b32 ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT 382 s_andn2_b32 ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK 383 s_or_b32 ttmp11, ttmp11, ttmp2 384 385 s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 S_COHERENCE // second-level TBA 386 s_waitcnt lgkmcnt(0) 387 s_load_dwordx2 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 S_COHERENCE // second-level TMA 388 s_waitcnt lgkmcnt(0) 389 390 s_and_b64 [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3] 391 s_cbranch_scc0 L_NO_NEXT_TRAP // second-level trap handler not been set 392 s_setpc_b64 [ttmp2, ttmp3] // jump to second-level trap handler 393 394L_NO_NEXT_TRAP: 395 // If not caused by trap then halt wave to prevent re-entry. 396 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK 397 s_cbranch_scc1 L_TRAP_CASE 398 399 // Host trap will not cause trap re-entry. 400#if ASIC_FAMILY < CHIP_GFX12 401 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK 402#else 403 s_getreg_b32 ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV) 404 s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK 405#endif 406 s_cbranch_scc1 L_EXIT_TRAP 407 s_or_b32 s_save_status, s_save_status, S_STATUS_HALT_MASK 408 409 // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set. 410 // Rewind the PC to prevent this from occurring. 411 s_sub_u32 ttmp0, ttmp0, 0x8 412 s_subb_u32 ttmp1, ttmp1, 0x0 413 414 s_branch L_EXIT_TRAP 415 416L_TRAP_CASE: 417 // Advance past trap instruction to prevent re-entry. 418 s_add_u32 ttmp0, ttmp0, 0x4 419 s_addc_u32 ttmp1, ttmp1, 0x0 420 421L_EXIT_TRAP: 422 s_and_b32 ttmp1, ttmp1, 0xFFFF 423 424#if HAVE_XNACK 425 restore_ib_sts(ttmp14, ttmp15) 426#endif 427 428 // Restore SQ_WAVE_STATUS. 429 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 430 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 431 s_setreg_b32 hwreg(S_STATUS_HWREG), s_save_status 432 433 s_rfe_b64 [ttmp0, ttmp1] 434 435L_SAVE: 436 s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32] 437 s_mov_b32 s_save_tmp, 0 438 s_setreg_b32 hwreg(S_TRAPSTS_HWREG, S_TRAPSTS_SAVE_CONTEXT_SHIFT, 1), s_save_tmp //clear saveCtx bit 439 440#if HAVE_XNACK 441 save_and_clear_ib_sts(s_save_tmp, s_save_trapsts) 442#endif 443 444 /* inform SPI the readiness and wait for SPI's go signal */ 445 s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI 446 s_mov_b32 s_save_exec_hi, exec_hi 447 s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive 448 449#if HAVE_SENDMSG_RTN 450 s_sendmsg_rtn_b64 [exec_lo, exec_hi], sendmsg(MSG_RTN_SAVE_WAVE) 451#else 452 s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC 453#endif 454 455#if ASIC_FAMILY < CHIP_SIENNA_CICHLID 456L_SLEEP: 457 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause 458 // SQ hang, since the 7,8th wave could not get arbit to exec inst, while 459 // other waves are stuck into the sleep-loop and waiting for wrexec!=0 460 s_sleep 0x2 461 s_cbranch_execz L_SLEEP 462#else 463 s_waitcnt lgkmcnt(0) 464#endif 465 466 // Save first_wave flag so we can clear high bits of save address. 467 s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK 468 s_lshl_b32 s_save_tmp, s_save_tmp, (S_SAVE_PC_HI_FIRST_WAVE_SHIFT - S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT) 469 s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp 470 471#if NO_SQC_STORE 472#if ASIC_FAMILY <= CHIP_SIENNA_CICHLID 473 // gfx10: If there was a VALU exception, the exception state must be 474 // cleared before executing the VALU instructions below. 475 v_clrexcp 476#endif 477 478 // Trap temporaries must be saved via VGPR but all VGPRs are in use. 479 // There is no ttmp space to hold the resource constant for VGPR save. 480 // Save v0 by itself since it requires only two SGPRs. 481 s_mov_b32 s_save_ttmps_lo, exec_lo 482 s_and_b32 s_save_ttmps_hi, exec_hi, 0xFFFF 483 s_mov_b32 exec_lo, 0xFFFFFFFF 484 s_mov_b32 exec_hi, 0xFFFFFFFF 485 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] V_COHERENCE 486 v_mov_b32 v0, 0x0 487 s_mov_b32 exec_lo, s_save_ttmps_lo 488 s_mov_b32 exec_hi, s_save_ttmps_hi 489#endif 490 491 // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic 492 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40 493 get_wave_size2(s_save_ttmps_hi) 494 get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi) 495 get_svgpr_size_bytes(s_save_ttmps_hi) 496 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi 497 s_and_b32 s_save_ttmps_hi, s_save_spi_init_hi, 0xFFFF 498 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, get_sgpr_size_bytes() 499 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo 500 s_addc_u32 s_save_ttmps_hi, s_save_ttmps_hi, 0x0 501 502#if NO_SQC_STORE 503 v_writelane_b32 v0, ttmp4, 0x4 504 v_writelane_b32 v0, ttmp5, 0x5 505 v_writelane_b32 v0, ttmp6, 0x6 506 v_writelane_b32 v0, ttmp7, 0x7 507 v_writelane_b32 v0, ttmp8, 0x8 508 v_writelane_b32 v0, ttmp9, 0x9 509 v_writelane_b32 v0, ttmp10, 0xA 510 v_writelane_b32 v0, ttmp11, 0xB 511 v_writelane_b32 v0, ttmp13, 0xD 512 v_writelane_b32 v0, exec_lo, 0xE 513 v_writelane_b32 v0, exec_hi, 0xF 514 515 s_mov_b32 exec_lo, 0x3FFF 516 s_mov_b32 exec_hi, 0x0 517 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] inst_offset:0x40 V_COHERENCE 518 v_readlane_b32 ttmp14, v0, 0xE 519 v_readlane_b32 ttmp15, v0, 0xF 520 s_mov_b32 exec_lo, ttmp14 521 s_mov_b32 exec_hi, ttmp15 522#else 523 s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 S_COHERENCE 524 s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 S_COHERENCE 525 s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 S_COHERENCE 526#endif 527 528 /* setup Resource Contants */ 529 s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo 530 s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi 531 s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE 532 s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited 533 s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC 534 535 s_mov_b32 s_save_m0, m0 536 537 /* global mem offset */ 538 s_mov_b32 s_save_mem_offset, 0x0 539 get_wave_size2(s_wave_size) 540 541#if HAVE_XNACK 542 // Save and clear vector XNACK state late to free up SGPRs. 543 s_getreg_b32 s_save_xnack_mask, hwreg(HW_REG_SHADER_XNACK_MASK) 544 s_setreg_imm32_b32 hwreg(HW_REG_SHADER_XNACK_MASK), 0x0 545#endif 546 547 /* save first 4 VGPRs, needed for SGPR save */ 548 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on 549 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE 550 s_and_b32 m0, m0, 1 551 s_cmp_eq_u32 m0, 1 552 s_cbranch_scc1 L_ENABLE_SAVE_4VGPR_EXEC_HI 553 s_mov_b32 exec_hi, 0x00000000 554 s_branch L_SAVE_4VGPR_WAVE32 555L_ENABLE_SAVE_4VGPR_EXEC_HI: 556 s_mov_b32 exec_hi, 0xFFFFFFFF 557 s_branch L_SAVE_4VGPR_WAVE64 558L_SAVE_4VGPR_WAVE32: 559 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 560 561 // VGPR Allocated in 4-GPR granularity 562 563#if SAVE_AFTER_XNACK_ERROR 564 check_if_tcp_store_ok() 565 s_cbranch_scc1 L_SAVE_FIRST_VGPRS32_WITH_TCP 566 567 write_vgprs_to_mem_with_sqc_w32(v0, 4, s_save_buf_rsrc0, s_save_mem_offset) 568 s_branch L_SAVE_HWREG 569 570L_SAVE_FIRST_VGPRS32_WITH_TCP: 571#endif 572 573#if !NO_SQC_STORE 574 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 575#endif 576 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128 577 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128*2 578 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128*3 579 s_branch L_SAVE_HWREG 580 581L_SAVE_4VGPR_WAVE64: 582 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 583 584 // VGPR Allocated in 4-GPR granularity 585 586#if SAVE_AFTER_XNACK_ERROR 587 check_if_tcp_store_ok() 588 s_cbranch_scc1 L_SAVE_FIRST_VGPRS64_WITH_TCP 589 590 write_vgprs_to_mem_with_sqc_w64(v0, 4, s_save_buf_rsrc0, s_save_mem_offset) 591 s_branch L_SAVE_HWREG 592 593L_SAVE_FIRST_VGPRS64_WITH_TCP: 594#endif 595 596#if !NO_SQC_STORE 597 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 598#endif 599 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256 600 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256*2 601 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256*3 602 603 /* save HW registers */ 604 605L_SAVE_HWREG: 606 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR) 607 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) 608 get_svgpr_size_bytes(s_save_tmp) 609 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp 610 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes() 611 612 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 613 614#if NO_SQC_STORE 615 v_mov_b32 v0, 0x0 //Offset[31:0] from buffer resource 616 v_mov_b32 v1, 0x0 //Offset[63:32] from buffer resource 617 v_mov_b32 v2, 0x0 //Set of SGPRs for TCP store 618 s_mov_b32 m0, 0x0 //Next lane of v2 to write to 619#endif 620 621#if ASIC_FAMILY >= CHIP_GFX12 622 // Ensure no further changes to barrier or LDS state. 623 s_barrier_signal -2 624 s_barrier_wait -2 625#endif 626 627 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 628 write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset) 629 s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK 630 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset) 631 write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset) 632 write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset) 633 write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset) 634 635 s_getreg_b32 s_save_tmp, hwreg(S_TRAPSTS_HWREG) 636 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset) 637 638 // Not used on Sienna_Cichlid but keep layout same for debugger. 639 write_hwreg_to_mem(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset) 640 641 s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) 642 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 643 644 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO) 645 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 646 647 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI) 648 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 649 650#if ASIC_FAMILY >= CHIP_GFX12 651 s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_EXCP_FLAG_USER) 652 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 653 654 s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_TRAP_CTRL) 655 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) 656 657 s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATUS) 658 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset) 659 660 s_get_barrier_state s_save_tmp, -1 661 s_wait_kmcnt (0) 662 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset) 663#endif 664 665#if NO_SQC_STORE 666 // Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this. 667 s_mov_b32 exec_lo, 0xFFFF 668 s_mov_b32 exec_hi, 0x0 669 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 670 671 // Write SGPRs with 32 VGPR lanes. This works in wave32 and wave64 mode. 672 s_mov_b32 exec_lo, 0xFFFFFFFF 673#endif 674 675 /* save SGPRs */ 676 // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save... 677 678 // SGPR SR memory offset : size(VGPR)+size(SVGPR) 679 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) 680 get_svgpr_size_bytes(s_save_tmp) 681 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp 682 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 683 684#if NO_SQC_STORE 685 s_mov_b32 ttmp13, 0x0 //next VGPR lane to copy SGPR into 686#else 687 // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0 688 s_mov_b32 s_save_xnack_mask, s_save_buf_rsrc0 689 s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset 690 s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0 691#endif 692 693 s_mov_b32 m0, 0x0 //SGPR initial index value =0 694 s_nop 0x0 //Manually inserted wait states 695L_SAVE_SGPR_LOOP: 696 // SGPR is allocated in 16 SGPR granularity 697 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0] 698 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0] 699 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0] 700 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0] 701 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0] 702 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0] 703 s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0] 704 s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0] 705 706 write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) 707 708#if NO_SQC_STORE 709 s_cmp_eq_u32 ttmp13, 0x20 //have 32 VGPR lanes filled? 710 s_cbranch_scc0 L_SAVE_SGPR_SKIP_TCP_STORE 711 712 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 713 s_add_u32 s_save_mem_offset, s_save_mem_offset, 0x80 714 s_mov_b32 ttmp13, 0x0 715 v_mov_b32 v2, 0x0 716L_SAVE_SGPR_SKIP_TCP_STORE: 717#endif 718 719 s_add_u32 m0, m0, 16 //next sgpr index 720 s_cmp_lt_u32 m0, 96 //scc = (m0 < first 96 SGPR) ? 1 : 0 721 s_cbranch_scc1 L_SAVE_SGPR_LOOP //first 96 SGPR save is complete? 722 723 //save the rest 12 SGPR 724 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0] 725 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0] 726 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0] 727 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0] 728 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0] 729 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0] 730 write_12sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) 731 732#if NO_SQC_STORE 733 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 734#else 735 // restore s_save_buf_rsrc0,1 736 s_mov_b32 s_save_buf_rsrc0, s_save_xnack_mask 737#endif 738 739 /* save LDS */ 740 741L_SAVE_LDS: 742 // Change EXEC to all threads... 743 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on 744 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE 745 s_and_b32 m0, m0, 1 746 s_cmp_eq_u32 m0, 1 747 s_cbranch_scc1 L_ENABLE_SAVE_LDS_EXEC_HI 748 s_mov_b32 exec_hi, 0x00000000 749 s_branch L_SAVE_LDS_NORMAL 750L_ENABLE_SAVE_LDS_EXEC_HI: 751 s_mov_b32 exec_hi, 0xFFFFFFFF 752L_SAVE_LDS_NORMAL: 753 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 754 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero? 755 s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE 756 757#if ASIC_FAMILY < CHIP_GFX12 758 s_barrier //LDS is used? wait for other waves in the same TG 759#endif 760 s_and_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK 761 s_cbranch_scc0 L_SAVE_LDS_DONE 762 763 // first wave do LDS save; 764 765 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw 766 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes 767 s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes 768 769 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG) 770 // 771 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) 772 get_svgpr_size_bytes(s_save_tmp) 773 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp 774 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes() 775 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes() 776 777 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 778 779 //load 0~63*4(byte address) to vgpr v0 780 v_mbcnt_lo_u32_b32 v0, -1, 0 781 v_mbcnt_hi_u32_b32 v0, -1, v0 782 v_mul_u32_u24 v0, 4, v0 783 784 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE 785 s_and_b32 m0, m0, 1 786 s_cmp_eq_u32 m0, 1 787 s_mov_b32 m0, 0x0 788 s_cbranch_scc1 L_SAVE_LDS_W64 789 790L_SAVE_LDS_W32: 791#if SAVE_AFTER_XNACK_ERROR 792 check_if_tcp_store_ok() 793 s_cbranch_scc1 L_SAVE_LDS_WITH_TCP_W32 794 795L_SAVE_LDS_LOOP_SQC_W32: 796 ds_read_b32 v1, v0 797 s_waitcnt 0 798 799 write_vgprs_to_mem_with_sqc_w32(v1, 1, s_save_buf_rsrc0, s_save_mem_offset) 800 801 s_add_u32 m0, m0, 128 //every buffer_store_lds does 128 bytes 802 v_add_nc_u32 v0, v0, 128 //mem offset increased by 128 bytes 803 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0 804 s_cbranch_scc1 L_SAVE_LDS_LOOP_SQC_W32 //LDS save is complete? 805 806 s_branch L_SAVE_LDS_DONE 807 808L_SAVE_LDS_WITH_TCP_W32: 809#endif 810 811 s_mov_b32 s3, 128 812 s_nop 0 813 s_nop 0 814 s_nop 0 815L_SAVE_LDS_LOOP_W32: 816 ds_read_b32 v1, v0 817 s_waitcnt 0 818 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 819 820 s_add_u32 m0, m0, s3 //every buffer_store_lds does 128 bytes 821 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 822 v_add_nc_u32 v0, v0, 128 //mem offset increased by 128 bytes 823 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0 824 s_cbranch_scc1 L_SAVE_LDS_LOOP_W32 //LDS save is complete? 825 826 s_branch L_SAVE_LDS_DONE 827 828L_SAVE_LDS_W64: 829#if SAVE_AFTER_XNACK_ERROR 830 check_if_tcp_store_ok() 831 s_cbranch_scc1 L_SAVE_LDS_WITH_TCP_W64 832 833L_SAVE_LDS_LOOP_SQC_W64: 834 ds_read_b32 v1, v0 835 s_waitcnt 0 836 837 write_vgprs_to_mem_with_sqc_w64(v1, 1, s_save_buf_rsrc0, s_save_mem_offset) 838 839 s_add_u32 m0, m0, 256 //every buffer_store_lds does 256 bytes 840 v_add_nc_u32 v0, v0, 256 //mem offset increased by 256 bytes 841 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0 842 s_cbranch_scc1 L_SAVE_LDS_LOOP_SQC_W64 //LDS save is complete? 843 844 s_branch L_SAVE_LDS_DONE 845 846L_SAVE_LDS_WITH_TCP_W64: 847#endif 848 849 s_mov_b32 s3, 256 850 s_nop 0 851 s_nop 0 852 s_nop 0 853L_SAVE_LDS_LOOP_W64: 854 ds_read_b32 v1, v0 855 s_waitcnt 0 856 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 857 858 s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes 859 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 860 v_add_nc_u32 v0, v0, 256 //mem offset increased by 256 bytes 861 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0 862 s_cbranch_scc1 L_SAVE_LDS_LOOP_W64 //LDS save is complete? 863 864L_SAVE_LDS_DONE: 865 /* save VGPRs - set the Rest VGPRs */ 866L_SAVE_VGPR: 867 // VGPR SR memory offset: 0 868 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on 869 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE 870 s_and_b32 m0, m0, 1 871 s_cmp_eq_u32 m0, 1 872 s_cbranch_scc1 L_ENABLE_SAVE_VGPR_EXEC_HI 873 s_mov_b32 s_save_mem_offset, (0+128*4) // for the rest VGPRs 874 s_mov_b32 exec_hi, 0x00000000 875 s_branch L_SAVE_VGPR_NORMAL 876L_ENABLE_SAVE_VGPR_EXEC_HI: 877 s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs 878 s_mov_b32 exec_hi, 0xFFFFFFFF 879L_SAVE_VGPR_NORMAL: 880 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) 881 s_add_u32 s_save_alloc_size, s_save_alloc_size, 1 882 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value) 883 //determine it is wave32 or wave64 884 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE 885 s_and_b32 m0, m0, 1 886 s_cmp_eq_u32 m0, 1 887 s_cbranch_scc1 L_SAVE_VGPR_WAVE64 888 889 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 890 891 // VGPR Allocated in 4-GPR granularity 892 893 // VGPR store using dw burst 894 s_mov_b32 m0, 0x4 //VGPR initial index value =4 895 s_cmp_lt_u32 m0, s_save_alloc_size 896 s_cbranch_scc0 L_SAVE_VGPR_END 897 898#if SAVE_AFTER_XNACK_ERROR 899 check_if_tcp_store_ok() 900 s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP 901 902L_SAVE_VGPR_LOOP_SQC_W32: 903 v_movrels_b32 v0, v0 //v0 = v[0+m0] 904 v_movrels_b32 v1, v1 //v1 = v[1+m0] 905 v_movrels_b32 v2, v2 //v2 = v[2+m0] 906 v_movrels_b32 v3, v3 //v3 = v[3+m0] 907 908 write_vgprs_to_mem_with_sqc_w32(v0, 4, s_save_buf_rsrc0, s_save_mem_offset) 909 910 s_add_u32 m0, m0, 4 911 s_cmp_lt_u32 m0, s_save_alloc_size 912 s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC_W32 913 914 s_branch L_SAVE_VGPR_END 915#endif 916 917L_SAVE_VGPR_W32_LOOP: 918 v_movrels_b32 v0, v0 //v0 = v[0+m0] 919 v_movrels_b32 v1, v1 //v1 = v[1+m0] 920 v_movrels_b32 v2, v2 //v2 = v[2+m0] 921 v_movrels_b32 v3, v3 //v3 = v[3+m0] 922 923 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 924 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128 925 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128*2 926 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:128*3 927 928 s_add_u32 m0, m0, 4 //next vgpr index 929 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128*4 //every buffer_store_dword does 128 bytes 930 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0 931 s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP //VGPR save is complete? 932 933 s_branch L_SAVE_VGPR_END 934 935L_SAVE_VGPR_WAVE64: 936 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 937 938 // VGPR store using dw burst 939 s_mov_b32 m0, 0x4 //VGPR initial index value =4 940 s_cmp_lt_u32 m0, s_save_alloc_size 941 s_cbranch_scc0 L_SAVE_SHARED_VGPR 942 943#if SAVE_AFTER_XNACK_ERROR 944 check_if_tcp_store_ok() 945 s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP 946 947L_SAVE_VGPR_LOOP_SQC_W64: 948 v_movrels_b32 v0, v0 //v0 = v[0+m0] 949 v_movrels_b32 v1, v1 //v1 = v[1+m0] 950 v_movrels_b32 v2, v2 //v2 = v[2+m0] 951 v_movrels_b32 v3, v3 //v3 = v[3+m0] 952 953 write_vgprs_to_mem_with_sqc_w64(v0, 4, s_save_buf_rsrc0, s_save_mem_offset) 954 955 s_add_u32 m0, m0, 4 956 s_cmp_lt_u32 m0, s_save_alloc_size 957 s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC_W64 958 959 s_branch L_SAVE_VGPR_END 960#endif 961 962L_SAVE_VGPR_W64_LOOP: 963 v_movrels_b32 v0, v0 //v0 = v[0+m0] 964 v_movrels_b32 v1, v1 //v1 = v[1+m0] 965 v_movrels_b32 v2, v2 //v2 = v[2+m0] 966 v_movrels_b32 v3, v3 //v3 = v[3+m0] 967 968 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 969 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256 970 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256*2 971 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE offset:256*3 972 973 s_add_u32 m0, m0, 4 //next vgpr index 974 s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes 975 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0 976 s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP //VGPR save is complete? 977 978L_SAVE_SHARED_VGPR: 979 //Below part will be the save shared vgpr part (new for gfx10) 980 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) 981 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero? 982 s_cbranch_scc0 L_SAVE_VGPR_END //no shared_vgpr used? jump to L_SAVE_LDS 983 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value) 984 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count. 985 //save shared_vgpr will start from the index of m0 986 s_add_u32 s_save_alloc_size, s_save_alloc_size, m0 987 s_mov_b32 exec_lo, 0xFFFFFFFF 988 s_mov_b32 exec_hi, 0x00000000 989 990#if SAVE_AFTER_XNACK_ERROR 991 check_if_tcp_store_ok() 992 s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP 993 994L_SAVE_SHARED_VGPR_WAVE64_LOOP_SQC: 995 v_movrels_b32 v0, v0 996 997 write_vgprs_to_mem_with_sqc_w64(v0, 1, s_save_buf_rsrc0, s_save_mem_offset) 998 999 s_add_u32 m0, m0, 1 1000 s_cmp_lt_u32 m0, s_save_alloc_size 1001 s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP_SQC 1002 1003 s_branch L_SAVE_VGPR_END 1004#endif 1005 1006L_SAVE_SHARED_VGPR_WAVE64_LOOP: 1007 v_movrels_b32 v0, v0 //v0 = v[0+m0] 1008 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset V_COHERENCE 1009 s_add_u32 m0, m0, 1 //next vgpr index 1010 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128 1011 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0 1012 s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP //SHARED_VGPR save is complete? 1013 1014L_SAVE_VGPR_END: 1015 s_branch L_END_PGM 1016 1017L_RESTORE: 1018 /* Setup Resource Contants */ 1019 s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo 1020 s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi 1021 s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE 1022 s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) 1023 s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC 1024 1025#if ASIC_FAMILY >= CHIP_GFX12 1026 // Save s_restore_spi_init_hi for later use. 1027 s_mov_b32 s_restore_spi_init_hi_save, s_restore_spi_init_hi 1028#endif 1029 1030 //determine it is wave32 or wave64 1031 get_wave_size2(s_restore_size) 1032 1033 s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK 1034 s_cbranch_scc0 L_RESTORE_VGPR 1035 1036 /* restore LDS */ 1037L_RESTORE_LDS: 1038 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on 1039 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE 1040 s_and_b32 m0, m0, 1 1041 s_cmp_eq_u32 m0, 1 1042 s_cbranch_scc1 L_ENABLE_RESTORE_LDS_EXEC_HI 1043 s_mov_b32 exec_hi, 0x00000000 1044 s_branch L_RESTORE_LDS_NORMAL 1045L_ENABLE_RESTORE_LDS_EXEC_HI: 1046 s_mov_b32 exec_hi, 0xFFFFFFFF 1047L_RESTORE_LDS_NORMAL: 1048 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 1049 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero? 1050 s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR 1051 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw 1052 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes 1053 s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes 1054 1055 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG) 1056 // 1057 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) 1058 get_svgpr_size_bytes(s_restore_tmp) 1059 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp 1060 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() 1061 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes() 1062 1063 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 1064 1065 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE 1066 s_and_b32 m0, m0, 1 1067 s_cmp_eq_u32 m0, 1 1068 s_mov_b32 m0, 0x0 1069 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 1070 1071L_RESTORE_LDS_LOOP_W32: 1072#if HAVE_BUFFER_LDS_LOAD 1073 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW 1074#else 1075 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset 1076 s_waitcnt vmcnt(0) 1077 ds_store_addtid_b32 v0 1078#endif 1079 s_add_u32 m0, m0, 128 // 128 DW 1080 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //mem offset increased by 128DW 1081 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0 1082 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W32 //LDS restore is complete? 1083 s_branch L_RESTORE_VGPR 1084 1085L_RESTORE_LDS_LOOP_W64: 1086#if HAVE_BUFFER_LDS_LOAD 1087 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW 1088#else 1089 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset 1090 s_waitcnt vmcnt(0) 1091 ds_store_addtid_b32 v0 1092#endif 1093 s_add_u32 m0, m0, 256 // 256 DW 1094 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //mem offset increased by 256DW 1095 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0 1096 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 //LDS restore is complete? 1097 1098 /* restore VGPRs */ 1099L_RESTORE_VGPR: 1100 // VGPR SR memory offset : 0 1101 s_mov_b32 s_restore_mem_offset, 0x0 1102 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on 1103 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE 1104 s_and_b32 m0, m0, 1 1105 s_cmp_eq_u32 m0, 1 1106 s_cbranch_scc1 L_ENABLE_RESTORE_VGPR_EXEC_HI 1107 s_mov_b32 exec_hi, 0x00000000 1108 s_branch L_RESTORE_VGPR_NORMAL 1109L_ENABLE_RESTORE_VGPR_EXEC_HI: 1110 s_mov_b32 exec_hi, 0xFFFFFFFF 1111L_RESTORE_VGPR_NORMAL: 1112 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) 1113 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1 1114 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value) 1115 //determine it is wave32 or wave64 1116 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE 1117 s_and_b32 m0, m0, 1 1118 s_cmp_eq_u32 m0, 1 1119 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64 1120 1121 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 1122 1123 // VGPR load using dw burst 1124 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last 1125 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4 1126 s_mov_b32 m0, 4 //VGPR initial index value = 4 1127 s_cmp_lt_u32 m0, s_restore_alloc_size 1128 s_cbranch_scc0 L_RESTORE_SGPR 1129 1130L_RESTORE_VGPR_WAVE32_LOOP: 1131 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE 1132 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:128 1133 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:128*2 1134 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:128*3 1135 s_waitcnt vmcnt(0) 1136 v_movreld_b32 v0, v0 //v[0+m0] = v0 1137 v_movreld_b32 v1, v1 1138 v_movreld_b32 v2, v2 1139 v_movreld_b32 v3, v3 1140 s_add_u32 m0, m0, 4 //next vgpr index 1141 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4 //every buffer_load_dword does 128 bytes 1142 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0 1143 s_cbranch_scc1 L_RESTORE_VGPR_WAVE32_LOOP //VGPR restore (except v0) is complete? 1144 1145 /* VGPR restore on v0 */ 1146 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE 1147 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:128 1148 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:128*2 1149 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:128*3 1150 s_waitcnt vmcnt(0) 1151 1152 s_branch L_RESTORE_SGPR 1153 1154L_RESTORE_VGPR_WAVE64: 1155 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 1156 1157 // VGPR load using dw burst 1158 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v4, v0 will be the last 1159 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 1160 s_mov_b32 m0, 4 //VGPR initial index value = 4 1161 s_cmp_lt_u32 m0, s_restore_alloc_size 1162 s_cbranch_scc0 L_RESTORE_SHARED_VGPR 1163 1164L_RESTORE_VGPR_WAVE64_LOOP: 1165 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE 1166 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:256 1167 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:256*2 1168 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE offset:256*3 1169 s_waitcnt vmcnt(0) 1170 v_movreld_b32 v0, v0 //v[0+m0] = v0 1171 v_movreld_b32 v1, v1 1172 v_movreld_b32 v2, v2 1173 v_movreld_b32 v3, v3 1174 s_add_u32 m0, m0, 4 //next vgpr index 1175 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes 1176 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0 1177 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete? 1178 1179L_RESTORE_SHARED_VGPR: 1180 //Below part will be the restore shared vgpr part (new for gfx10) 1181 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size 1182 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero? 1183 s_cbranch_scc0 L_RESTORE_V0 //no shared_vgpr used? 1184 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value) 1185 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count. 1186 //restore shared_vgpr will start from the index of m0 1187 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, m0 1188 s_mov_b32 exec_lo, 0xFFFFFFFF 1189 s_mov_b32 exec_hi, 0x00000000 1190L_RESTORE_SHARED_VGPR_WAVE64_LOOP: 1191 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset V_COHERENCE 1192 s_waitcnt vmcnt(0) 1193 v_movreld_b32 v0, v0 //v[0+m0] = v0 1194 s_add_u32 m0, m0, 1 //next vgpr index 1195 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 1196 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0 1197 s_cbranch_scc1 L_RESTORE_SHARED_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete? 1198 1199 s_mov_b32 exec_hi, 0xFFFFFFFF //restore back exec_hi before restoring V0!! 1200 1201 /* VGPR restore on v0 */ 1202L_RESTORE_V0: 1203 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE 1204 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:256 1205 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:256*2 1206 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save V_COHERENCE offset:256*3 1207 s_waitcnt vmcnt(0) 1208 1209 /* restore SGPRs */ 1210 //will be 2+8+16*6 1211 // SGPR SR memory offset : size(VGPR)+size(SVGPR) 1212L_RESTORE_SGPR: 1213 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) 1214 get_svgpr_size_bytes(s_restore_tmp) 1215 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp 1216 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() 1217 s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 20*4 //s108~s127 is not saved 1218 1219 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 1220 1221 s_mov_b32 m0, s_sgpr_save_num 1222 1223 read_4sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) 1224 s_waitcnt lgkmcnt(0) 1225 1226 s_sub_u32 m0, m0, 4 // Restore from S[0] to S[104] 1227 s_nop 0 // hazard SALU M0=> S_MOVREL 1228 1229 s_movreld_b64 s0, s0 //s[0+m0] = s0 1230 s_movreld_b64 s2, s2 1231 1232 read_8sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) 1233 s_waitcnt lgkmcnt(0) 1234 1235 s_sub_u32 m0, m0, 8 // Restore from S[0] to S[96] 1236 s_nop 0 // hazard SALU M0=> S_MOVREL 1237 1238 s_movreld_b64 s0, s0 //s[0+m0] = s0 1239 s_movreld_b64 s2, s2 1240 s_movreld_b64 s4, s4 1241 s_movreld_b64 s6, s6 1242 1243 L_RESTORE_SGPR_LOOP: 1244 read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) 1245 s_waitcnt lgkmcnt(0) 1246 1247 s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0] 1248 s_nop 0 // hazard SALU M0=> S_MOVREL 1249 1250 s_movreld_b64 s0, s0 //s[0+m0] = s0 1251 s_movreld_b64 s2, s2 1252 s_movreld_b64 s4, s4 1253 s_movreld_b64 s6, s6 1254 s_movreld_b64 s8, s8 1255 s_movreld_b64 s10, s10 1256 s_movreld_b64 s12, s12 1257 s_movreld_b64 s14, s14 1258 1259 s_cmp_eq_u32 m0, 0 //scc = (m0 < s_sgpr_save_num) ? 1 : 0 1260 s_cbranch_scc0 L_RESTORE_SGPR_LOOP 1261 1262 // s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception. 1263 // Clear DEBUG_EN before and restore MODE after the barrier. 1264 s_setreg_imm32_b32 hwreg(HW_REG_MODE), 0 1265#if ASIC_FAMILY < CHIP_GFX12 1266 s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG 1267#endif 1268 1269 /* restore HW registers */ 1270L_RESTORE_HWREG: 1271 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR) 1272 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) 1273 get_svgpr_size_bytes(s_restore_tmp) 1274 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp 1275 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() 1276 1277 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes 1278 1279#if ASIC_FAMILY >= CHIP_GFX12 1280 // Restore s_restore_spi_init_hi before the saved value gets clobbered. 1281 s_mov_b32 s_restore_spi_init_hi, s_restore_spi_init_hi_save 1282#endif 1283 1284 read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset) 1285 read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset) 1286 read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset) 1287 read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset) 1288 read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset) 1289 read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset) 1290 read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset) 1291 read_hwreg_from_mem(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset) 1292 read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset) 1293 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset) 1294 s_waitcnt lgkmcnt(0) 1295 1296 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s_restore_flat_scratch 1297 1298 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset) 1299 s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS 1300 1301 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch 1302 1303#if ASIC_FAMILY >= CHIP_GFX12 1304 read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) 1305 s_waitcnt lgkmcnt(0) 1306 s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_USER), s_restore_tmp 1307 1308 read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) 1309 s_waitcnt lgkmcnt(0) 1310 s_setreg_b32 hwreg(HW_REG_WAVE_TRAP_CTRL), s_restore_tmp 1311 1312 // Only the first wave needs to restore the workgroup barrier. 1313 s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK 1314 s_cbranch_scc0 L_SKIP_BARRIER_RESTORE 1315 1316 // Skip over WAVE_STATUS, since there is no state to restore from it 1317 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 4 1318 1319 read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) 1320 s_waitcnt lgkmcnt(0) 1321 1322 s_bitcmp1_b32 s_restore_tmp, BARRIER_STATE_VALID_OFFSET 1323 s_cbranch_scc0 L_SKIP_BARRIER_RESTORE 1324 1325 // extract the saved signal count from s_restore_tmp 1326 s_lshr_b32 s_restore_tmp, s_restore_tmp, BARRIER_STATE_SIGNAL_OFFSET 1327 1328 // We need to call s_barrier_signal repeatedly to restore the signal 1329 // count of the work group barrier. The member count is already 1330 // initialized with the number of waves in the work group. 1331L_BARRIER_RESTORE_LOOP: 1332 s_and_b32 s_restore_tmp, s_restore_tmp, s_restore_tmp 1333 s_cbranch_scc0 L_SKIP_BARRIER_RESTORE 1334 s_barrier_signal -1 1335 s_add_i32 s_restore_tmp, s_restore_tmp, -1 1336 s_branch L_BARRIER_RESTORE_LOOP 1337 1338L_SKIP_BARRIER_RESTORE: 1339 // Make barrier and LDS state visible to all waves in the group. 1340 s_barrier_signal -2 1341 s_barrier_wait -2 1342#endif 1343 1344 s_mov_b32 m0, s_restore_m0 1345 s_mov_b32 exec_lo, s_restore_exec_lo 1346 s_mov_b32 exec_hi, s_restore_exec_hi 1347 1348#if HAVE_XNACK 1349 s_setreg_b32 hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask 1350#endif 1351 1352 s_setreg_b32 hwreg(S_TRAPSTS_HWREG), s_restore_trapsts 1353 s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode 1354 1355 // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic 1356 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40 1357 get_vgpr_size_bytes(s_restore_ttmps_lo, s_restore_size) 1358 get_svgpr_size_bytes(s_restore_ttmps_hi) 1359 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi 1360 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, get_sgpr_size_bytes() 1361 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0 1362 s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0 1363 s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF 1364 s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 S_COHERENCE 1365 s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 S_COHERENCE 1366 s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 S_COHERENCE 1367 s_waitcnt lgkmcnt(0) 1368 1369#if HAVE_XNACK 1370 restore_ib_sts(s_restore_tmp, s_restore_m0) 1371#endif 1372 1373 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS 1374 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 1375 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 1376 1377#if SW_SA_TRAP 1378 // If traps are enabled then return to the shader with PRIV=0. 1379 // Otherwise retain PRIV=1 for subsequent context save requests. 1380 s_getreg_b32 s_restore_tmp, hwreg(HW_REG_STATUS) 1381 s_bitcmp1_b32 s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT 1382 s_cbranch_scc1 L_RETURN_WITHOUT_PRIV 1383 1384 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu 1385 s_setpc_b64 [s_restore_pc_lo, s_restore_pc_hi] 1386L_RETURN_WITHOUT_PRIV: 1387#endif 1388 1389 s_setreg_b32 hwreg(S_STATUS_HWREG), s_restore_status // SCC is included, which is changed by previous salu 1390 s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution 1391 1392L_END_PGM: 1393 s_endpgm_saved 1394end 1395 1396function write_hwreg_to_mem(s, s_rsrc, s_mem_offset) 1397#if NO_SQC_STORE 1398 // Copy into VGPR for later TCP store. 1399 v_writelane_b32 v2, s, m0 1400 s_add_u32 m0, m0, 0x1 1401#else 1402 s_mov_b32 exec_lo, m0 1403 s_mov_b32 m0, s_mem_offset 1404 s_buffer_store_dword s, s_rsrc, m0 S_COHERENCE 1405 s_add_u32 s_mem_offset, s_mem_offset, 4 1406 s_mov_b32 m0, exec_lo 1407#endif 1408end 1409 1410 1411function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset) 1412#if NO_SQC_STORE 1413 // Copy into VGPR for later TCP store. 1414 for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++ 1415 v_writelane_b32 v2, s[sgpr_idx], ttmp13 1416 s_add_u32 ttmp13, ttmp13, 0x1 1417 end 1418#else 1419 s_buffer_store_dwordx4 s[0], s_rsrc, 0 S_COHERENCE 1420 s_buffer_store_dwordx4 s[4], s_rsrc, 16 S_COHERENCE 1421 s_buffer_store_dwordx4 s[8], s_rsrc, 32 S_COHERENCE 1422 s_buffer_store_dwordx4 s[12], s_rsrc, 48 S_COHERENCE 1423 s_add_u32 s_rsrc[0], s_rsrc[0], 4*16 1424 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0 1425#endif 1426end 1427 1428function write_12sgpr_to_mem(s, s_rsrc, s_mem_offset) 1429#if NO_SQC_STORE 1430 // Copy into VGPR for later TCP store. 1431 for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++ 1432 v_writelane_b32 v2, s[sgpr_idx], ttmp13 1433 s_add_u32 ttmp13, ttmp13, 0x1 1434 end 1435#else 1436 s_buffer_store_dwordx4 s[0], s_rsrc, 0 S_COHERENCE 1437 s_buffer_store_dwordx4 s[4], s_rsrc, 16 S_COHERENCE 1438 s_buffer_store_dwordx4 s[8], s_rsrc, 32 S_COHERENCE 1439 s_add_u32 s_rsrc[0], s_rsrc[0], 4*12 1440 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0 1441#endif 1442end 1443 1444function read_hwreg_from_mem(s, s_rsrc, s_mem_offset) 1445 s_buffer_load_dword s, s_rsrc, s_mem_offset S_COHERENCE 1446 s_add_u32 s_mem_offset, s_mem_offset, 4 1447end 1448 1449function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset) 1450 s_sub_u32 s_mem_offset, s_mem_offset, 4*16 1451 s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset S_COHERENCE 1452end 1453 1454function read_8sgpr_from_mem(s, s_rsrc, s_mem_offset) 1455 s_sub_u32 s_mem_offset, s_mem_offset, 4*8 1456 s_buffer_load_dwordx8 s, s_rsrc, s_mem_offset S_COHERENCE 1457end 1458 1459function read_4sgpr_from_mem(s, s_rsrc, s_mem_offset) 1460 s_sub_u32 s_mem_offset, s_mem_offset, 4*4 1461 s_buffer_load_dwordx4 s, s_rsrc, s_mem_offset S_COHERENCE 1462end 1463 1464#if SAVE_AFTER_XNACK_ERROR 1465function check_if_tcp_store_ok 1466 // If TRAPSTS.XNACK_ERROR=1 then TCP stores will fail. 1467 s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS) 1468 s_andn2_b32 s_save_tmp, SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK, s_save_tmp 1469 1470L_TCP_STORE_CHECK_DONE: 1471end 1472 1473function write_vgpr_to_mem_with_sqc(vgpr, n_lanes, s_rsrc, s_mem_offset) 1474 s_mov_b32 s4, 0 1475 1476L_WRITE_VGPR_LANE_LOOP: 1477 for var lane = 0; lane < 4; ++lane 1478 v_readlane_b32 s[lane], vgpr, s4 1479 s_add_u32 s4, s4, 1 1480 end 1481 1482 s_buffer_store_dwordx4 s[0:3], s_rsrc, s_mem_offset glc:1 1483 1484 s_add_u32 s_mem_offset, s_mem_offset, 0x10 1485 s_cmp_eq_u32 s4, n_lanes 1486 s_cbranch_scc0 L_WRITE_VGPR_LANE_LOOP 1487end 1488 1489function write_vgprs_to_mem_with_sqc_w32(vgpr0, n_vgprs, s_rsrc, s_mem_offset) 1490 for var vgpr = 0; vgpr < n_vgprs; ++vgpr 1491 write_vgpr_to_mem_with_sqc(vgpr0[vgpr], 32, s_rsrc, s_mem_offset) 1492 end 1493end 1494 1495function write_vgprs_to_mem_with_sqc_w64(vgpr0, n_vgprs, s_rsrc, s_mem_offset) 1496 for var vgpr = 0; vgpr < n_vgprs; ++vgpr 1497 write_vgpr_to_mem_with_sqc(vgpr0[vgpr], 64, s_rsrc, s_mem_offset) 1498 end 1499end 1500#endif 1501 1502function get_lds_size_bytes(s_lds_size_byte) 1503 s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 1504 s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW 1505end 1506 1507function get_vgpr_size_bytes(s_vgpr_size_byte, s_size) 1508 s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) 1509 s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 1 1510 s_bitcmp1_b32 s_size, S_WAVE_SIZE 1511 s_cbranch_scc1 L_ENABLE_SHIFT_W64 1512 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+7) //Number of VGPRs = (vgpr_size + 1) * 4 * 32 * 4 (non-zero value) 1513 s_branch L_SHIFT_DONE 1514L_ENABLE_SHIFT_W64: 1515 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value) 1516L_SHIFT_DONE: 1517end 1518 1519function get_svgpr_size_bytes(s_svgpr_size_byte) 1520 s_getreg_b32 s_svgpr_size_byte, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) 1521 s_lshl_b32 s_svgpr_size_byte, s_svgpr_size_byte, (3+7) 1522end 1523 1524function get_sgpr_size_bytes 1525 return 512 1526end 1527 1528function get_hwreg_size_bytes 1529 return 128 1530end 1531 1532function get_wave_size2(s_reg) 1533#if ASIC_FAMILY < CHIP_GFX12 1534 s_getreg_b32 s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE) 1535#else 1536 s_getreg_b32 s_reg, hwreg(HW_REG_WAVE_STATUS,SQ_WAVE_STATUS_WAVE64_SHIFT,SQ_WAVE_STATUS_WAVE64_SIZE) 1537#endif 1538 s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE 1539end 1540 1541#if HAVE_XNACK 1542function save_and_clear_ib_sts(tmp1, tmp2) 1543 // Preserve and clear scalar XNACK state before issuing scalar loads. 1544 // Save IB_STS.REPLAY_W64H[25], RCNT[21:16], FIRST_REPLAY[15] into 1545 // unused space ttmp11[31:24]. 1546 s_andn2_b32 ttmp11, ttmp11, (TTMP11_SAVE_REPLAY_W64H_MASK | TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK) 1547 s_getreg_b32 tmp1, hwreg(HW_REG_IB_STS) 1548 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK 1549 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT) 1550 s_or_b32 ttmp11, ttmp11, tmp2 1551 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK 1552 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) 1553 s_or_b32 ttmp11, ttmp11, tmp2 1554 s_andn2_b32 tmp1, tmp1, (SQ_WAVE_IB_STS_REPLAY_W64H_MASK | SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK) 1555 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1 1556end 1557 1558function restore_ib_sts(tmp1, tmp2) 1559 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) 1560 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK 1561 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT) 1562 s_and_b32 tmp1, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK 1563 s_or_b32 tmp1, tmp1, tmp2 1564 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1 1565end 1566#endif 1567