Lines Matching refs:PAL

5 …lloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9-PAL %s
7 …nable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1010-PAL %s
8 …nable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1030-PAL %s
9 …ca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-PAL %s
71 ; GFX9-PAL-LABEL: zero_init_kernel:
72 ; GFX9-PAL:       ; %bb.0:
73 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
74 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
75 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
76 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
77 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
78 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
79 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
80 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
81 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
82 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
83 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
84 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
85 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
86 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
87 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
88 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
89 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:64
90 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
91 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
92 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
93 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
94 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
95 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
96 ; GFX9-PAL-NEXT:    s_endpgm
112 ; GFX1010-PAL-LABEL: zero_init_kernel:
113 ; GFX1010-PAL:       ; %bb.0:
114 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
115 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
116 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
117 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
118 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
119 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
120 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
121 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
122 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
123 ; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
124 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
125 ; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
126 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
127 ; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
128 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
129 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
130 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
131 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
132 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:64
133 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
134 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
135 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
136 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
137 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
138 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
139 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
140 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
141 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
142 ; GFX1010-PAL-NEXT:    s_endpgm
144 ; GFX1030-PAL-LABEL: zero_init_kernel:
145 ; GFX1030-PAL:       ; %bb.0:
146 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
147 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
148 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
149 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
150 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
151 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
152 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
153 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
154 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
155 ; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
156 ; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
157 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
158 ; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
159 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
160 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
161 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
162 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
163 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
164 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
165 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
166 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
167 ; GFX1030-PAL-NEXT:    s_endpgm
169 ; GFX11-PAL-LABEL: zero_init_kernel:
170 ; GFX11-PAL:       ; %bb.0:
171 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
172 ; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
173 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
174 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
175 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
176 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
177 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
178 ; GFX11-PAL-NEXT:    s_clause 0x3
179 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:64
180 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:48
181 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
182 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:16
183 ; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
184 ; GFX11-PAL-NEXT:    s_endpgm
248 ; GFX9-PAL-LABEL: zero_init_foo:
249 ; GFX9-PAL:       ; %bb.0:
250 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
251 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
252 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
253 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
254 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
255 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
256 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
257 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
258 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
259 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
260 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
261 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
262 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
263 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
264 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
282 ; GFX10-PAL-LABEL: zero_init_foo:
283 ; GFX10-PAL:       ; %bb.0:
284 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
285 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
286 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
287 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
288 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
289 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
290 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
291 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
292 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
293 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
294 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
295 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
296 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
297 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
298 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
299 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
301 ; GFX11-PAL-LABEL: zero_init_foo:
302 ; GFX11-PAL:       ; %bb.0:
303 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
304 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
305 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
306 ; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
307 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
308 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
309 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
310 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
311 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
312 ; GFX11-PAL-NEXT:    s_clause 0x3
313 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:48
314 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:32
315 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:16
316 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32
317 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
318 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
395 ; GFX9-PAL-LABEL: store_load_sindex_kernel:
396 ; GFX9-PAL:       ; %bb.0: ; %bb
397 ; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
398 ; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
399 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
400 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
401 ; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
402 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
403 ; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
404 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
405 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
406 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
407 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
408 ; GFX9-PAL-NEXT:    s_add_i32 s1, s1, 4
409 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
410 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
411 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
412 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
413 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
414 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
415 ; GFX9-PAL-NEXT:    s_endpgm
433 ; GFX10-PAL-LABEL: store_load_sindex_kernel:
434 ; GFX10-PAL:       ; %bb.0: ; %bb
435 ; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
436 ; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
437 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
438 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
439 ; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
440 ; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
441 ; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
442 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
443 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
444 ; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
445 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
446 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
447 ; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
448 ; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
449 ; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
450 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
451 ; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
452 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
453 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
454 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
455 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
456 ; GFX10-PAL-NEXT:    s_endpgm
458 ; GFX11-PAL-LABEL: store_load_sindex_kernel:
459 ; GFX11-PAL:       ; %bb.0: ; %bb
460 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
461 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
462 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
463 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
464 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
465 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
466 ; GFX11-PAL-NEXT:    s_add_i32 s0, s0, 4
467 ; GFX11-PAL-NEXT:    s_add_i32 s1, s1, 4
468 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
469 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
470 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
471 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
472 ; GFX11-PAL-NEXT:    s_endpgm
550 ; GFX9-PAL-LABEL: store_load_sindex_foo:
551 ; GFX9-PAL:       ; %bb.0: ; %bb
552 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
553 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
554 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
555 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
556 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
557 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
558 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
559 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
560 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
561 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
562 ; GFX9-PAL-NEXT:    s_add_i32 s1, s1, 4
563 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
564 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
565 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
566 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
567 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
568 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
569 ; GFX9-PAL-NEXT:    s_endpgm
585 ; GFX10-PAL-LABEL: store_load_sindex_foo:
586 ; GFX10-PAL:       ; %bb.0: ; %bb
587 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
588 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
589 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
590 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
591 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
592 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
593 ; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
594 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
595 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
596 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
597 ; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
598 ; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
599 ; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
600 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
601 ; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
602 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
603 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
604 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
605 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
606 ; GFX10-PAL-NEXT:    s_endpgm
608 ; GFX11-PAL-LABEL: store_load_sindex_foo:
609 ; GFX11-PAL:       ; %bb.0: ; %bb
610 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
611 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
612 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
613 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
614 ; GFX11-PAL-NEXT:    s_add_i32 s0, s0, 4
615 ; GFX11-PAL-NEXT:    s_add_i32 s1, s1, 4
616 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
617 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
618 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
619 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
620 ; GFX11-PAL-NEXT:    s_endpgm
689 ; GFX9-PAL-LABEL: store_load_vindex_kernel:
690 ; GFX9-PAL:       ; %bb.0: ; %bb
691 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
692 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
693 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
694 ; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
695 ; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 4, v0
696 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
697 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 4, v0
698 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
699 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
700 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
701 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
702 ; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
703 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
704 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
705 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
706 ; GFX9-PAL-NEXT:    s_endpgm
719 ; GFX10-PAL-LABEL: store_load_vindex_kernel:
720 ; GFX10-PAL:       ; %bb.0: ; %bb
721 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
722 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
723 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
724 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
725 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
726 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
727 ; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
728 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
729 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
730 ; GFX10-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
731 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
732 ; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v1, 4, v0
733 ; GFX10-PAL-NEXT:    v_sub_nc_u32_e32 v0, 4, v0
734 ; GFX10-PAL-NEXT:    scratch_store_dword v1, v2, off
735 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
736 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
737 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
738 ; GFX10-PAL-NEXT:    s_endpgm
740 ; GFX11-PAL-LABEL: store_load_vindex_kernel:
741 ; GFX11-PAL:       ; %bb.0: ; %bb
742 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
743 ; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
744 ; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 4, v0
745 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:4 dlc
746 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
747 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
748 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
749 ; GFX11-PAL-NEXT:    s_endpgm
818 ; GFX9-PAL-LABEL: store_load_vindex_foo:
819 ; GFX9-PAL:       ; %bb.0: ; %bb
820 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
821 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s32
822 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
823 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
824 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
825 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
826 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
827 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
828 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
829 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
830 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
845 ; GFX10-PAL-LABEL: store_load_vindex_foo:
846 ; GFX10-PAL:       ; %bb.0: ; %bb
847 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
848 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
849 ; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
850 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, s32
851 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
852 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, s32
853 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
854 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
855 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
856 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
857 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
859 ; GFX11-PAL-LABEL: store_load_vindex_foo:
860 ; GFX11-PAL:       ; %bb.0: ; %bb
861 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
862 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
863 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
864 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
865 ; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
866 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
867 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, s32 dlc
868 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
869 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 glc dlc
870 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
871 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
924 ; GFX9-PAL-LABEL: private_ptr_foo:
925 ; GFX9-PAL:       ; %bb.0:
926 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
927 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
928 ; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
929 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
930 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
940 ; GFX10-PAL-LABEL: private_ptr_foo:
941 ; GFX10-PAL:       ; %bb.0:
942 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
943 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
944 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
945 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
946 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
947 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
949 ; GFX11-PAL-LABEL: private_ptr_foo:
950 ; GFX11-PAL:       ; %bb.0:
951 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
952 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
953 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
954 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:4
955 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
956 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
1036 ; GFX9-PAL-LABEL: zero_init_small_offset_kernel:
1037 ; GFX9-PAL:       ; %bb.0:
1038 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1039 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1040 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1041 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1042 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
1043 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1044 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1045 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1046 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1047 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1048 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1049 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
1050 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1051 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
1052 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
1053 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
1054 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
1055 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
1056 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1057 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:272
1058 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1059 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:288
1060 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1061 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:304
1062 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1063 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:320
1064 ; GFX9-PAL-NEXT:    s_endpgm
1082 ; GFX1010-PAL-LABEL: zero_init_small_offset_kernel:
1083 ; GFX1010-PAL:       ; %bb.0:
1084 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1085 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1086 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1087 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1088 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1089 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1090 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1091 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1092 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1093 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1094 ; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
1095 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1096 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1097 ; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
1098 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1099 ; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
1100 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
1101 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
1102 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
1103 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
1104 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1105 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:272
1106 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1107 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1108 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:288
1109 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1110 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1111 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:304
1112 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1113 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1114 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:320
1115 ; GFX1010-PAL-NEXT:    s_endpgm
1117 ; GFX1030-PAL-LABEL: zero_init_small_offset_kernel:
1118 ; GFX1030-PAL:       ; %bb.0:
1119 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1120 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1121 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1122 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1123 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1124 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1125 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1126 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1127 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1128 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1129 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1130 ; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
1131 ; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
1132 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1133 ; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
1134 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
1135 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
1136 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
1137 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
1138 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
1139 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
1140 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
1141 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
1142 ; GFX1030-PAL-NEXT:    s_endpgm
1144 ; GFX11-PAL-LABEL: zero_init_small_offset_kernel:
1145 ; GFX11-PAL:       ; %bb.0:
1146 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1147 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1148 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
1149 ; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1150 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
1151 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
1152 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
1153 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1154 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
1155 ; GFX11-PAL-NEXT:    s_clause 0x3
1156 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:272
1157 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:288
1158 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:304
1159 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:320
1160 ; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1161 ; GFX11-PAL-NEXT:    s_endpgm
1234 ; GFX9-PAL-LABEL: zero_init_small_offset_foo:
1235 ; GFX9-PAL:       ; %bb.0:
1236 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1237 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s32 glc
1238 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1239 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
1240 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
1241 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1242 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
1243 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
1244 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
1245 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
1246 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
1247 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1248 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1249 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1250 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1251 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1252 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
1272 ; GFX10-PAL-LABEL: zero_init_small_offset_foo:
1273 ; GFX10-PAL:       ; %bb.0:
1274 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1275 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1276 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s32 glc dlc
1277 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1278 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
1279 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
1280 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
1281 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
1282 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
1283 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
1284 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
1285 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
1286 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1287 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1288 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1289 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1290 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1291 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
1293 ; GFX11-PAL-LABEL: zero_init_small_offset_foo:
1294 ; GFX11-PAL:       ; %bb.0:
1295 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1296 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1297 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s32 glc dlc
1298 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1299 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
1300 ; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1301 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
1302 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
1303 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
1304 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1305 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
1306 ; GFX11-PAL-NEXT:    s_clause 0x3
1307 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:256
1308 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:272
1309 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:288
1310 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:304
1311 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1312 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
1400 ; GFX9-PAL-LABEL: store_load_sindex_small_offset_kernel:
1401 ; GFX9-PAL:       ; %bb.0: ; %bb
1402 ; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
1403 ; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
1404 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1405 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1406 ; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1407 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1408 ; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1409 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
1410 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
1411 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1412 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1413 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
1414 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
1415 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
1416 ; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x104
1417 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1418 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
1419 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1420 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
1421 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
1422 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1423 ; GFX9-PAL-NEXT:    s_endpgm
1443 ; GFX1010-PAL-LABEL: store_load_sindex_small_offset_kernel:
1444 ; GFX1010-PAL:       ; %bb.0: ; %bb
1445 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
1446 ; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
1447 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1448 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1449 ; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1450 ; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3
1451 ; GFX1010-PAL-NEXT:    s_addc_u32 s5, s5, 0
1452 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
1453 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
1454 ; GFX1010-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1455 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1456 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1457 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1458 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
1459 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1460 ; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
1461 ; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1462 ; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1463 ; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x104
1464 ; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x104
1465 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
1466 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1467 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1468 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1469 ; GFX1010-PAL-NEXT:    s_endpgm
1471 ; GFX1030-PAL-LABEL: store_load_sindex_small_offset_kernel:
1472 ; GFX1030-PAL:       ; %bb.0: ; %bb
1473 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[4:5]
1474 ; GFX1030-PAL-NEXT:    s_mov_b32 s4, s0
1475 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1476 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1477 ; GFX1030-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1478 ; GFX1030-PAL-NEXT:    s_add_u32 s4, s4, s3
1479 ; GFX1030-PAL-NEXT:    s_addc_u32 s5, s5, 0
1480 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
1481 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
1482 ; GFX1030-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1483 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1484 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1485 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
1486 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1487 ; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
1488 ; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1489 ; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1490 ; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x104
1491 ; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x104
1492 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
1493 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1494 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1495 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1496 ; GFX1030-PAL-NEXT:    s_endpgm
1498 ; GFX11-PAL-LABEL: store_load_sindex_small_offset_kernel:
1499 ; GFX11-PAL:       ; %bb.0: ; %bb
1500 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
1501 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1502 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1503 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
1504 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1505 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
1506 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1507 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1508 ; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x104
1509 ; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x104
1510 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
1511 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1512 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1513 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1514 ; GFX11-PAL-NEXT:    s_endpgm
1587 ; GFX9-PAL-LABEL: store_load_sindex_small_offset_foo:
1588 ; GFX9-PAL:       ; %bb.0: ; %bb
1589 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1590 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1591 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1592 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1593 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1594 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1595 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1596 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1597 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1598 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1599 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
1600 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
1601 ; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x104
1602 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
1603 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1604 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
1605 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1606 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
1607 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
1608 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1609 ; GFX9-PAL-NEXT:    s_endpgm
1627 ; GFX1010-PAL-LABEL: store_load_sindex_small_offset_foo:
1628 ; GFX1010-PAL:       ; %bb.0: ; %bb
1629 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1630 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1631 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1632 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1633 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1634 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1635 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1636 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1637 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1638 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1639 ; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
1640 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1641 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1642 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
1643 ; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1644 ; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1645 ; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x104
1646 ; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x104
1647 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
1648 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1649 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1650 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1651 ; GFX1010-PAL-NEXT:    s_endpgm
1653 ; GFX1030-PAL-LABEL: store_load_sindex_small_offset_foo:
1654 ; GFX1030-PAL:       ; %bb.0: ; %bb
1655 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1656 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1657 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1658 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1659 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1660 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1661 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1662 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1663 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1664 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1665 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1666 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
1667 ; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
1668 ; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1669 ; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1670 ; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x104
1671 ; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x104
1672 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
1673 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1674 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1675 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1676 ; GFX1030-PAL-NEXT:    s_endpgm
1678 ; GFX11-PAL-LABEL: store_load_sindex_small_offset_foo:
1679 ; GFX11-PAL:       ; %bb.0: ; %bb
1680 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1681 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1682 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
1683 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
1684 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1685 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1686 ; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x104
1687 ; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x104
1688 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
1689 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1690 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1691 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1692 ; GFX11-PAL-NEXT:    s_endpgm
1757 ; GFX9-PAL-LABEL: store_load_vindex_small_offset_kernel:
1758 ; GFX9-PAL:       ; %bb.0: ; %bb
1759 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1760 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1761 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1762 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1763 ; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1764 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
1765 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1766 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1767 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1768 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1769 ; GFX9-PAL-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
1770 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1771 ; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 0x104, v0
1772 ; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
1773 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1774 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 0x104, v0
1775 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
1776 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1777 ; GFX9-PAL-NEXT:    s_endpgm
1792 ; GFX1010-PAL-LABEL: store_load_vindex_small_offset_kernel:
1793 ; GFX1010-PAL:       ; %bb.0: ; %bb
1794 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1795 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1796 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1797 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1798 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1799 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1800 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1801 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1802 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1803 ; GFX1010-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1804 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, 15
1805 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1806 ; GFX1010-PAL-NEXT:    scratch_load_dword v3, off, vcc_lo offset:4 glc dlc
1807 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1808 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x104, v0
1809 ; GFX1010-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x104, v0
1810 ; GFX1010-PAL-NEXT:    scratch_store_dword v1, v2, off
1811 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1812 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
1813 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1814 ; GFX1010-PAL-NEXT:    s_endpgm
1816 ; GFX1030-PAL-LABEL: store_load_vindex_small_offset_kernel:
1817 ; GFX1030-PAL:       ; %bb.0: ; %bb
1818 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1819 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1820 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1821 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1822 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1823 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1824 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1825 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1826 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1827 ; GFX1030-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1828 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, 15
1829 ; GFX1030-PAL-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
1830 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1831 ; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x104, v0
1832 ; GFX1030-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x104, v0
1833 ; GFX1030-PAL-NEXT:    scratch_store_dword v1, v2, off
1834 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1835 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
1836 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1837 ; GFX1030-PAL-NEXT:    s_endpgm
1839 ; GFX11-PAL-LABEL: store_load_vindex_small_offset_kernel:
1840 ; GFX11-PAL:       ; %bb.0: ; %bb
1841 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
1842 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
1843 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1844 ; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 0x104, v0
1845 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:260 dlc
1846 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1847 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
1848 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1849 ; GFX11-PAL-NEXT:    s_endpgm
1919 ; GFX9-PAL-LABEL: store_load_vindex_small_offset_foo:
1920 ; GFX9-PAL:       ; %bb.0: ; %bb
1921 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1922 ; GFX9-PAL-NEXT:    scratch_load_dword v1, off, s32 glc
1923 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1924 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x100
1925 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, vcc_hi
1926 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
1927 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
1928 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
1929 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
1930 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1931 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
1932 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
1933 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1934 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
1951 ; GFX10-PAL-LABEL: store_load_vindex_small_offset_foo:
1952 ; GFX10-PAL:       ; %bb.0: ; %bb
1953 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1954 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1955 ; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
1956 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1957 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
1958 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
1959 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1960 ; GFX10-PAL-NEXT:    scratch_load_dword v3, off, s32 glc dlc
1961 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1962 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
1963 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
1964 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1965 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
1966 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1967 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
1969 ; GFX11-PAL-LABEL: store_load_vindex_small_offset_foo:
1970 ; GFX11-PAL:       ; %bb.0: ; %bb
1971 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1972 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1973 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
1974 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1975 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, s32 glc dlc
1976 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1977 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
1978 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, s32 offset:256 dlc
1979 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1980 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 offset:256 glc dlc
1981 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1982 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
2086 ; GFX9-PAL-LABEL: zero_init_large_offset_kernel:
2087 ; GFX9-PAL:       ; %bb.0:
2088 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2089 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2090 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2091 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2092 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
2093 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2094 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2095 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2096 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2097 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:16 glc
2098 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2099 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
2100 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2101 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
2102 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
2103 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
2104 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
2105 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
2106 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2107 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2108 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2109 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2110 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2111 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2112 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2113 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2114 ; GFX9-PAL-NEXT:    s_endpgm
2136 ; GFX1010-PAL-LABEL: zero_init_large_offset_kernel:
2137 ; GFX1010-PAL:       ; %bb.0:
2138 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2139 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2140 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2141 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2142 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2143 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2144 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2145 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2146 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2147 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2148 ; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
2149 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:16 glc dlc
2150 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2151 ; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
2152 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2153 ; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
2154 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
2155 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
2156 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
2157 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
2158 ; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2159 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2160 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2161 ; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2162 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2163 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2164 ; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2165 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2166 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2167 ; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2168 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2169 ; GFX1010-PAL-NEXT:    s_endpgm
2171 ; GFX1030-PAL-LABEL: zero_init_large_offset_kernel:
2172 ; GFX1030-PAL:       ; %bb.0:
2173 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2174 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2175 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2176 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2177 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2178 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2179 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2180 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2181 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2182 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:16 glc dlc
2183 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2184 ; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
2185 ; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2186 ; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
2187 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2188 ; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
2189 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
2190 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
2191 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
2192 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
2193 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2194 ; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2195 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2196 ; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2197 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2198 ; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2199 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2200 ; GFX1030-PAL-NEXT:    s_endpgm
2202 ; GFX11-PAL-LABEL: zero_init_large_offset_kernel:
2203 ; GFX11-PAL:       ; %bb.0:
2204 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:16 glc dlc
2205 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2206 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
2207 ; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2208 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
2209 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
2210 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
2211 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2212 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2213 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2214 ; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2215 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2216 ; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2217 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2218 ; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2219 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2220 ; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2221 ; GFX11-PAL-NEXT:    s_endpgm
2304 ; GFX9-PAL-LABEL: zero_init_large_offset_foo:
2305 ; GFX9-PAL:       ; %bb.0:
2306 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2307 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc
2308 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2309 ; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
2310 ; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
2311 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2312 ; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
2313 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
2314 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
2315 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
2316 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
2317 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2318 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2319 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2320 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2321 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2322 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2323 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2324 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2325 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2326 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
2350 ; GFX1010-PAL-LABEL: zero_init_large_offset_foo:
2351 ; GFX1010-PAL:       ; %bb.0:
2352 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2353 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2354 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
2355 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2356 ; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
2357 ; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2358 ; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
2359 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2360 ; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
2361 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
2362 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
2363 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
2364 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
2365 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2366 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2367 ; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2368 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2369 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2370 ; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2371 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2372 ; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2373 ; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2374 ; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2375 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2376 ; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
2378 ; GFX1030-PAL-LABEL: zero_init_large_offset_foo:
2379 ; GFX1030-PAL:       ; %bb.0:
2380 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2381 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2382 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
2383 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2384 ; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
2385 ; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2386 ; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
2387 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2388 ; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
2389 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
2390 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
2391 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
2392 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
2393 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2394 ; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2395 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2396 ; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2397 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2398 ; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2399 ; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2400 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2401 ; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
2403 ; GFX11-PAL-LABEL: zero_init_large_offset_foo:
2404 ; GFX11-PAL:       ; %bb.0:
2405 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2406 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2407 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s32 offset:16 glc dlc
2408 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2409 ; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
2410 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2411 ; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
2412 ; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
2413 ; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
2414 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2415 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2416 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2417 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2418 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2419 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2420 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2421 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2422 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2423 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2424 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
2495 ; GFX9-PAL-LABEL: store_load_sindex_large_offset_kernel:
2496 ; GFX9-PAL:       ; %bb.0: ; %bb
2497 ; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
2498 ; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
2499 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2500 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2501 ; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2502 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2503 ; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2504 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
2505 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
2506 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2507 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2508 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
2509 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
2510 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
2511 ; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x4004
2512 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2513 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
2514 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2515 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
2516 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
2517 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2518 ; GFX9-PAL-NEXT:    s_endpgm
2538 ; GFX1010-PAL-LABEL: store_load_sindex_large_offset_kernel:
2539 ; GFX1010-PAL:       ; %bb.0: ; %bb
2540 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
2541 ; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
2542 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2543 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2544 ; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2545 ; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3
2546 ; GFX1010-PAL-NEXT:    s_addc_u32 s5, s5, 0
2547 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
2548 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
2549 ; GFX1010-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2550 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2551 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
2552 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2553 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
2554 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2555 ; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
2556 ; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2557 ; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2558 ; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x4004
2559 ; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x4004
2560 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
2561 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2562 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2563 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2564 ; GFX1010-PAL-NEXT:    s_endpgm
2566 ; GFX1030-PAL-LABEL: store_load_sindex_large_offset_kernel:
2567 ; GFX1030-PAL:       ; %bb.0: ; %bb
2568 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[4:5]
2569 ; GFX1030-PAL-NEXT:    s_mov_b32 s4, s0
2570 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2571 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2572 ; GFX1030-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2573 ; GFX1030-PAL-NEXT:    s_add_u32 s4, s4, s3
2574 ; GFX1030-PAL-NEXT:    s_addc_u32 s5, s5, 0
2575 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
2576 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
2577 ; GFX1030-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2578 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2579 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2580 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
2581 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2582 ; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
2583 ; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2584 ; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2585 ; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x4004
2586 ; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x4004
2587 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
2588 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2589 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2590 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2591 ; GFX1030-PAL-NEXT:    s_endpgm
2593 ; GFX11-PAL-LABEL: store_load_sindex_large_offset_kernel:
2594 ; GFX11-PAL:       ; %bb.0: ; %bb
2595 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
2596 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2597 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2598 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
2599 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2600 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
2601 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2602 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2603 ; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x4004
2604 ; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x4004
2605 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
2606 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2607 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2608 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2609 ; GFX11-PAL-NEXT:    s_endpgm
2682 ; GFX9-PAL-LABEL: store_load_sindex_large_offset_foo:
2683 ; GFX9-PAL:       ; %bb.0: ; %bb
2684 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2685 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2686 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2687 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2688 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2689 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2690 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2691 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2692 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2693 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2694 ; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
2695 ; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
2696 ; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x4004
2697 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
2698 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2699 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
2700 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2701 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
2702 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
2703 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2704 ; GFX9-PAL-NEXT:    s_endpgm
2722 ; GFX1010-PAL-LABEL: store_load_sindex_large_offset_foo:
2723 ; GFX1010-PAL:       ; %bb.0: ; %bb
2724 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2725 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2726 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2727 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2728 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2729 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2730 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2731 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2732 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2733 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2734 ; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
2735 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
2736 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2737 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
2738 ; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2739 ; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2740 ; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x4004
2741 ; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x4004
2742 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
2743 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2744 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2745 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2746 ; GFX1010-PAL-NEXT:    s_endpgm
2748 ; GFX1030-PAL-LABEL: store_load_sindex_large_offset_foo:
2749 ; GFX1030-PAL:       ; %bb.0: ; %bb
2750 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2751 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2752 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2753 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2754 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2755 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2756 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2757 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2758 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2759 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2760 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2761 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
2762 ; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
2763 ; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2764 ; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2765 ; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x4004
2766 ; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x4004
2767 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
2768 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2769 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2770 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2771 ; GFX1030-PAL-NEXT:    s_endpgm
2773 ; GFX11-PAL-LABEL: store_load_sindex_large_offset_foo:
2774 ; GFX11-PAL:       ; %bb.0: ; %bb
2775 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2776 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2777 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
2778 ; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
2779 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2780 ; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2781 ; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x4004
2782 ; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x4004
2783 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
2784 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2785 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2786 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2787 ; GFX11-PAL-NEXT:    s_endpgm
2853 ; GFX9-PAL-LABEL: store_load_vindex_large_offset_kernel:
2854 ; GFX9-PAL:       ; %bb.0: ; %bb
2855 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2856 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2857 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2858 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2859 ; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2860 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
2861 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2862 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2863 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2864 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2865 ; GFX9-PAL-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
2866 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2867 ; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 0x4004, v0
2868 ; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
2869 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2870 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 0x4004, v0
2871 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
2872 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2873 ; GFX9-PAL-NEXT:    s_endpgm
2889 ; GFX1010-PAL-LABEL: store_load_vindex_large_offset_kernel:
2890 ; GFX1010-PAL:       ; %bb.0: ; %bb
2891 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2892 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2893 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2894 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2895 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2896 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2897 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2898 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2899 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2900 ; GFX1010-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2901 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, 15
2902 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2903 ; GFX1010-PAL-NEXT:    scratch_load_dword v3, off, vcc_lo offset:4 glc dlc
2904 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2905 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v0
2906 ; GFX1010-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x4004, v0
2907 ; GFX1010-PAL-NEXT:    scratch_store_dword v1, v2, off
2908 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2909 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
2910 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2911 ; GFX1010-PAL-NEXT:    s_endpgm
2913 ; GFX1030-PAL-LABEL: store_load_vindex_large_offset_kernel:
2914 ; GFX1030-PAL:       ; %bb.0: ; %bb
2915 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2916 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2917 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2918 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2919 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2920 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2921 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2922 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2923 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2924 ; GFX1030-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2925 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, 15
2926 ; GFX1030-PAL-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
2927 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2928 ; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v0
2929 ; GFX1030-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x4004, v0
2930 ; GFX1030-PAL-NEXT:    scratch_store_dword v1, v2, off
2931 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2932 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
2933 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2934 ; GFX1030-PAL-NEXT:    s_endpgm
2936 ; GFX11-PAL-LABEL: store_load_vindex_large_offset_kernel:
2937 ; GFX11-PAL:       ; %bb.0: ; %bb
2938 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
2939 ; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4004
2940 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
2941 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2942 ; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 0x4004, v0
2943 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, vcc_lo dlc
2944 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2945 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
2946 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2947 ; GFX11-PAL-NEXT:    s_endpgm
3019 ; GFX9-PAL-LABEL: store_load_vindex_large_offset_foo:
3020 ; GFX9-PAL:       ; %bb.0: ; %bb
3021 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3022 ; GFX9-PAL-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
3023 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3024 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
3025 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, vcc_hi
3026 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
3027 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
3028 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
3029 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
3030 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3031 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3032 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
3033 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3034 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3053 ; GFX10-PAL-LABEL: store_load_vindex_large_offset_foo:
3054 ; GFX10-PAL:       ; %bb.0: ; %bb
3055 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3056 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3057 ; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
3058 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3059 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
3060 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
3061 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3062 ; GFX10-PAL-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
3063 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3064 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
3065 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
3066 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3067 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
3068 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3069 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3071 ; GFX11-PAL-LABEL: store_load_vindex_large_offset_foo:
3072 ; GFX11-PAL:       ; %bb.0: ; %bb
3073 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3074 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3075 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
3076 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
3077 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3078 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, s32 offset:4 glc dlc
3079 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3080 ; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
3081 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, vcc_lo dlc
3082 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3083 ; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3084 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, vcc_lo glc dlc
3085 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3086 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3167 ; GFX9-PAL-LABEL: store_load_large_imm_offset_kernel:
3168 ; GFX9-PAL:       ; %bb.0: ; %bb
3169 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
3170 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
3171 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3172 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 13
3173 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
3174 ; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
3175 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3176 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3177 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
3178 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
3179 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
3180 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3181 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
3182 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
3183 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3184 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3185 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3186 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3187 ; GFX9-PAL-NEXT:    s_endpgm
3202 ; GFX1010-PAL-LABEL: store_load_large_imm_offset_kernel:
3203 ; GFX1010-PAL:       ; %bb.0: ; %bb
3204 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
3205 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
3206 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3207 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3208 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3209 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
3210 ; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
3211 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
3212 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
3213 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 13
3214 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 15
3215 ; GFX1010-PAL-NEXT:    s_movk_i32 s0, 0x3800
3216 ; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
3217 ; GFX1010-PAL-NEXT:    s_add_i32 s0, s0, 4
3218 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, vcc_lo offset:4
3219 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3220 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3221 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3222 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3223 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
3224 ; GFX1010-PAL-NEXT:    s_endpgm
3226 ; GFX1030-PAL-LABEL: store_load_large_imm_offset_kernel:
3227 ; GFX1030-PAL:       ; %bb.0: ; %bb
3228 ; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
3229 ; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
3230 ; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3231 ; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3232 ; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3233 ; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
3234 ; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
3235 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
3236 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
3237 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 13
3238 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 15
3239 ; GFX1030-PAL-NEXT:    s_movk_i32 s0, 0x3800
3240 ; GFX1030-PAL-NEXT:    s_add_i32 s0, s0, 4
3241 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, off offset:4
3242 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3243 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3244 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3245 ; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3246 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
3247 ; GFX1030-PAL-NEXT:    s_endpgm
3249 ; GFX11-PAL-LABEL: store_load_large_imm_offset_kernel:
3250 ; GFX11-PAL:       ; %bb.0: ; %bb
3251 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 0x3000
3252 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
3253 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
3254 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3255 ; GFX11-PAL-NEXT:    scratch_store_b32 v1, v2, off offset:3716 dlc
3256 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3257 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, off offset:3716 glc dlc
3258 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3259 ; GFX11-PAL-NEXT:    s_endpgm
3319 ; GFX9-PAL-LABEL: store_load_large_imm_offset_foo:
3320 ; GFX9-PAL:       ; %bb.0: ; %bb
3321 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3322 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 13
3323 ; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
3324 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 4
3325 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
3326 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3327 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, vcc_hi
3328 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
3329 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3330 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3331 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3332 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3333 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3349 ; GFX10-PAL-LABEL: store_load_large_imm_offset_foo:
3350 ; GFX10-PAL:       ; %bb.0: ; %bb
3351 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3352 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3353 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 13
3354 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3355 ; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x3800
3356 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 4
3357 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, vcc_lo
3358 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
3359 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3360 ; GFX10-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3361 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3362 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3363 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3364 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3366 ; GFX11-PAL-LABEL: store_load_large_imm_offset_foo:
3367 ; GFX11-PAL:       ; %bb.0: ; %bb
3368 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3369 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3370 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 0x3000
3371 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
3372 ; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s32 offset:4 dlc
3373 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3374 ; GFX11-PAL-NEXT:    scratch_store_b32 v1, v2, s32 offset:3716 dlc
3375 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3376 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 offset:3716 glc dlc
3377 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3378 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3449 ; GFX9-PAL-LABEL: store_load_vidx_sidx_offset:
3450 ; GFX9-PAL:       ; %bb.0: ; %bb
3451 ; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
3452 ; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
3453 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
3454 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 4
3455 ; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
3456 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3457 ; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
3458 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
3459 ; GFX9-PAL-NEXT:    v_add_u32_e32 v0, s0, v0
3460 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
3461 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3462 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3463 ; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
3464 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3465 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
3466 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3467 ; GFX9-PAL-NEXT:    s_endpgm
3481 ; GFX10-PAL-LABEL: store_load_vidx_sidx_offset:
3482 ; GFX10-PAL:       ; %bb.0: ; %bb
3483 ; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
3484 ; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
3485 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
3486 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3487 ; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
3488 ; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
3489 ; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
3490 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
3491 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
3492 ; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
3493 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3494 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3495 ; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
3496 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, 4
3497 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
3498 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3499 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
3500 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3501 ; GFX10-PAL-NEXT:    s_endpgm
3503 ; GFX11-PAL-LABEL: store_load_vidx_sidx_offset:
3504 ; GFX11-PAL:       ; %bb.0: ; %bb
3505 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
3506 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
3507 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3508 ; GFX11-PAL-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
3509 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:1028 dlc
3510 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3511 ; GFX11-PAL-NEXT:    scratch_load_b32 v0, v0, off offset:1028 glc dlc
3512 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3513 ; GFX11-PAL-NEXT:    s_endpgm
3571 ; GFX9-PAL-LABEL: store_load_i64_aligned:
3572 ; GFX9-PAL:       ; %bb.0: ; %bb
3573 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3574 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3575 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
3576 ; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3577 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3578 ; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3579 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3580 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3593 ; GFX10-PAL-LABEL: store_load_i64_aligned:
3594 ; GFX10-PAL:       ; %bb.0: ; %bb
3595 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3596 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3597 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3598 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
3599 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3600 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3601 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3602 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3603 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3605 ; GFX11-PAL-LABEL: store_load_i64_aligned:
3606 ; GFX11-PAL:       ; %bb.0: ; %bb
3607 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3608 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3609 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
3610 ; GFX11-PAL-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3611 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3612 ; GFX11-PAL-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3613 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3614 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3666 ; GFX9-PAL-LABEL: store_load_i64_unaligned:
3667 ; GFX9-PAL:       ; %bb.0: ; %bb
3668 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3669 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3670 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
3671 ; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3672 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3673 ; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3674 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3675 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3688 ; GFX10-PAL-LABEL: store_load_i64_unaligned:
3689 ; GFX10-PAL:       ; %bb.0: ; %bb
3690 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3691 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3692 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3693 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
3694 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3695 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3696 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3697 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3698 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3700 ; GFX11-PAL-LABEL: store_load_i64_unaligned:
3701 ; GFX11-PAL:       ; %bb.0: ; %bb
3702 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3703 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3704 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
3705 ; GFX11-PAL-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3706 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3707 ; GFX11-PAL-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3708 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3709 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3764 ; GFX9-PAL-LABEL: store_load_v3i32_unaligned:
3765 ; GFX9-PAL:       ; %bb.0: ; %bb
3766 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3767 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
3768 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 2
3769 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
3770 ; GFX9-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3771 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3772 ; GFX9-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
3773 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3774 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3788 ; GFX10-PAL-LABEL: store_load_v3i32_unaligned:
3789 ; GFX10-PAL:       ; %bb.0: ; %bb
3790 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3791 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3792 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
3793 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
3794 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
3795 ; GFX10-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3796 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3797 ; GFX10-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
3798 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3799 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3801 ; GFX11-PAL-LABEL: store_load_v3i32_unaligned:
3802 ; GFX11-PAL:       ; %bb.0: ; %bb
3803 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3804 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3805 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 2
3806 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, 3
3807 ; GFX11-PAL-NEXT:    scratch_store_b96 v0, v[1:3], off dlc
3808 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3809 ; GFX11-PAL-NEXT:    scratch_load_b96 v[0:2], v0, off glc dlc
3810 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3811 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3869 ; GFX9-PAL-LABEL: store_load_v4i32_unaligned:
3870 ; GFX9-PAL:       ; %bb.0: ; %bb
3871 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3872 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
3873 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 2
3874 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
3875 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v4, 4
3876 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3877 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3878 ; GFX9-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
3879 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3880 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3895 ; GFX10-PAL-LABEL: store_load_v4i32_unaligned:
3896 ; GFX10-PAL:       ; %bb.0: ; %bb
3897 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3898 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3899 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
3900 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
3901 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
3902 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v4, 4
3903 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3904 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3905 ; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
3906 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3907 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3909 ; GFX11-PAL-LABEL: store_load_v4i32_unaligned:
3910 ; GFX11-PAL:       ; %bb.0: ; %bb
3911 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3912 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3913 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 2
3914 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v3, 3 :: v_dual_mov_b32 v4, 4
3915 ; GFX11-PAL-NEXT:    scratch_store_b128 v0, v[1:4], off dlc
3916 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3917 ; GFX11-PAL-NEXT:    scratch_load_b128 v[0:3], v0, off glc dlc
3918 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3919 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3972 ; GFX9-PAL-LABEL: store_load_i32_negative_unaligned:
3973 ; GFX9-PAL:       ; %bb.0: ; %bb
3974 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3975 ; GFX9-PAL-NEXT:    v_add_u32_e32 v0, -1, v0
3976 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
3977 ; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
3978 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3979 ; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
3980 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3981 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3994 ; GFX1010-PAL-LABEL: store_load_i32_negative_unaligned:
3995 ; GFX1010-PAL:       ; %bb.0: ; %bb
3996 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3997 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3998 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, -1, v0
3999 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
4000 ; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off
4001 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4002 ; GFX1010-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc dlc
4003 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
4004 ; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
4006 ; GFX1030-PAL-LABEL: store_load_i32_negative_unaligned:
4007 ; GFX1030-PAL:       ; %bb.0: ; %bb
4008 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4009 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4010 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 1
4011 ; GFX1030-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-1
4012 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4013 ; GFX1030-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-1 glc dlc
4014 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
4015 ; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
4017 ; GFX11-PAL-LABEL: store_load_i32_negative_unaligned:
4018 ; GFX11-PAL:       ; %bb.0: ; %bb
4019 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4020 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4021 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 1
4022 ; GFX11-PAL-NEXT:    scratch_store_b8 v0, v1, off offset:-1 dlc
4023 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4024 ; GFX11-PAL-NEXT:    scratch_load_u8 v0, v0, off offset:-1 glc dlc
4025 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4026 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
4069 ; GFX9-PAL-LABEL: store_load_i32_large_negative_unaligned:
4070 ; GFX9-PAL:       ; %bb.0: ; %bb
4071 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4072 ; GFX9-PAL-NEXT:    v_add_u32_e32 v0, 0xffffef7f, v0
4073 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
4074 ; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
4075 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4076 ; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
4077 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4078 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
4091 ; GFX1010-PAL-LABEL: store_load_i32_large_negative_unaligned:
4092 ; GFX1010-PAL:       ; %bb.0: ; %bb
4093 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4094 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4095 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xffffefff, v0
4096 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
4097 ; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-128
4098 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4099 ; GFX1010-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-128 glc dlc
4100 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
4101 ; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
4103 ; GFX1030-PAL-LABEL: store_load_i32_large_negative_unaligned:
4104 ; GFX1030-PAL:       ; %bb.0: ; %bb
4105 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4106 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4107 ; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xfffff000, v0
4108 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 1
4109 ; GFX1030-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-129
4110 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4111 ; GFX1030-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-129 glc dlc
4112 ; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
4113 ; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
4115 ; GFX11-PAL-LABEL: store_load_i32_large_negative_unaligned:
4116 ; GFX11-PAL:       ; %bb.0: ; %bb
4117 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4118 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4119 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_add_nc_u32 v0, 0xfffff000, v0
4120 ; GFX11-PAL-NEXT:    scratch_store_b8 v0, v1, off offset:-129 dlc
4121 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4122 ; GFX11-PAL-NEXT:    scratch_load_u8 v0, v0, off offset:-129 glc dlc
4123 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4124 ; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
4203 ; GFX9-PAL-LABEL: large_offset:
4204 ; GFX9-PAL:       ; %bb.0: ; %bb
4205 ; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
4206 ; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
4207 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
4208 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 0
4209 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, v0
4210 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, v0
4211 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, v0
4212 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
4213 ; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
4214 ; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s0
4215 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
4216 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
4217 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:3024
4218 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4219 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
4220 ; GFX9-PAL-NEXT:    scratch_load_dwordx4 v[0:3], off, vcc_hi offset:3024 glc
4221 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4222 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 16
4223 ; GFX9-PAL-NEXT:    ;;#ASMSTART
4224 ; GFX9-PAL-NEXT:    ; use v0
4225 ; GFX9-PAL-NEXT:    ;;#ASMEND
4226 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 0x810
4227 ; GFX9-PAL-NEXT:    ;;#ASMSTART
4228 ; GFX9-PAL-NEXT:    ; use v0
4229 ; GFX9-PAL-NEXT:    ;;#ASMEND
4230 ; GFX9-PAL-NEXT:    s_endpgm
4252 ; GFX10-PAL-LABEL: large_offset:
4253 ; GFX10-PAL:       ; %bb.0: ; %bb
4254 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
4255 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
4256 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
4257 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
4258 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
4259 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s0
4260 ; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
4261 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
4262 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
4263 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 0
4264 ; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x810
4265 ; GFX10-PAL-NEXT:    s_addk_i32 s0, 0x3c0
4266 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, v0
4267 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, v0
4268 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, v0
4269 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s0
4270 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4271 ; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 glc dlc
4272 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
4273 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 16
4274 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x810
4275 ; GFX10-PAL-NEXT:    ;;#ASMSTART
4276 ; GFX10-PAL-NEXT:    ; use v0
4277 ; GFX10-PAL-NEXT:    ;;#ASMEND
4278 ; GFX10-PAL-NEXT:    ;;#ASMSTART
4279 ; GFX10-PAL-NEXT:    ; use v1
4280 ; GFX10-PAL-NEXT:    ;;#ASMEND
4281 ; GFX10-PAL-NEXT:    s_endpgm
4283 ; GFX11-PAL-LABEL: large_offset:
4284 ; GFX11-PAL:       ; %bb.0: ; %bb
4285 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 0
4286 ; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
4287 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, v0
4288 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, v0
4289 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, v0
4290 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:3024 dlc
4291 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4292 ; GFX11-PAL-NEXT:    scratch_load_b128 v[0:3], off, off offset:3024 glc dlc
4293 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4294 ; GFX11-PAL-NEXT:    v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0x810
4295 ; GFX11-PAL-NEXT:    ;;#ASMSTART
4296 ; GFX11-PAL-NEXT:    ; use v0
4297 ; GFX11-PAL-NEXT:    ;;#ASMEND
4298 ; GFX11-PAL-NEXT:    ;;#ASMSTART
4299 ; GFX11-PAL-NEXT:    ; use v1
4300 ; GFX11-PAL-NEXT:    ;;#ASMEND
4301 ; GFX11-PAL-NEXT:    s_endpgm