1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1030 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
4; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX11 %s
5; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9-PAL %s
6; RUN: llc -march=amdgcn -mcpu=gfx940 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940 %s
7; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1010-PAL %s
8; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1030 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1030-PAL %s
9; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1100 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-PAL %s
10
11define amdgpu_kernel void @zero_init_kernel() {
12; GFX9-LABEL: zero_init_kernel:
13; GFX9:       ; %bb.0:
14; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
15; GFX9-NEXT:    s_mov_b32 s0, 0
16; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
17; GFX9-NEXT:    s_mov_b32 s1, s0
18; GFX9-NEXT:    s_mov_b32 s2, s0
19; GFX9-NEXT:    s_mov_b32 s3, s0
20; GFX9-NEXT:    v_mov_b32_e32 v0, s0
21; GFX9-NEXT:    v_mov_b32_e32 v1, s1
22; GFX9-NEXT:    v_mov_b32_e32 v2, s2
23; GFX9-NEXT:    v_mov_b32_e32 v3, s3
24; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
25; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:64
26; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
27; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
28; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
29; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
30; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
31; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
32; GFX9-NEXT:    s_endpgm
33;
34; GFX10-LABEL: zero_init_kernel:
35; GFX10:       ; %bb.0:
36; GFX10-NEXT:    s_add_u32 s0, s0, s3
37; GFX10-NEXT:    s_addc_u32 s1, s1, 0
38; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
39; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
40; GFX10-NEXT:    s_mov_b32 s0, 0
41; GFX10-NEXT:    s_mov_b32 s1, s0
42; GFX10-NEXT:    s_mov_b32 s2, s0
43; GFX10-NEXT:    s_mov_b32 s3, s0
44; GFX10-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-NEXT:    v_mov_b32_e32 v2, s2
47; GFX10-NEXT:    v_mov_b32_e32 v3, s3
48; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
49; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
50; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
51; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
52; GFX10-NEXT:    s_endpgm
53;
54; GFX11-LABEL: zero_init_kernel:
55; GFX11:       ; %bb.0:
56; GFX11-NEXT:    s_mov_b32 s0, 0
57; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
58; GFX11-NEXT:    s_mov_b32 s1, s0
59; GFX11-NEXT:    s_mov_b32 s2, s0
60; GFX11-NEXT:    s_mov_b32 s3, s0
61; GFX11-NEXT:    v_mov_b32_e32 v0, s0
62; GFX11-NEXT:    v_mov_b32_e32 v1, s1
63; GFX11-NEXT:    v_mov_b32_e32 v2, s2
64; GFX11-NEXT:    v_mov_b32_e32 v3, s3
65; GFX11-NEXT:    s_clause 0x3
66; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:64
67; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:48
68; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
69; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:16
70; GFX11-NEXT:    s_endpgm
71;
72; GFX9-PAL-LABEL: zero_init_kernel:
73; GFX9-PAL:       ; %bb.0:
74; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
75; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
76; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
77; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
78; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
79; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
80; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
81; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
82; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
83; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
84; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
85; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
86; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
87; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
88; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
89; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
90; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:64
91; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
92; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
93; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
94; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
95; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
96; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
97; GFX9-PAL-NEXT:    s_endpgm
98;
99; GFX940-LABEL: zero_init_kernel:
100; GFX940:       ; %bb.0:
101; GFX940-NEXT:    s_mov_b32 s0, 0
102; GFX940-NEXT:    s_mov_b32 s1, s0
103; GFX940-NEXT:    s_mov_b32 s2, s0
104; GFX940-NEXT:    s_mov_b32 s3, s0
105; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
106; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
107; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
108; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
109; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
110; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
111; GFX940-NEXT:    s_endpgm
112;
113; GFX1010-PAL-LABEL: zero_init_kernel:
114; GFX1010-PAL:       ; %bb.0:
115; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
116; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
117; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
118; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
119; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
120; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
121; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
122; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
123; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
124; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
125; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
126; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
127; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
128; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
129; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
130; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
131; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
132; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
133; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:64
134; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
135; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
136; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
137; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
138; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
139; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
140; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
141; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
142; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
143; GFX1010-PAL-NEXT:    s_endpgm
144;
145; GFX1030-PAL-LABEL: zero_init_kernel:
146; GFX1030-PAL:       ; %bb.0:
147; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
148; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
149; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
150; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
152; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
153; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
154; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
155; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
156; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
157; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
158; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
159; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
160; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
161; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
162; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
163; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
164; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
165; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
166; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
167; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
168; GFX1030-PAL-NEXT:    s_endpgm
169;
170; GFX11-PAL-LABEL: zero_init_kernel:
171; GFX11-PAL:       ; %bb.0:
172; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
173; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
174; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
175; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
176; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
177; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
178; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
179; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
180; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
181; GFX11-PAL-NEXT:    s_clause 0x3
182; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:64
183; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:48
184; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:32
185; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:16
186; GFX11-PAL-NEXT:    s_endpgm
187  %alloca = alloca [32 x i16], align 2, addrspace(5)
188  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
189  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
190  ret void
191}
192
193define void @zero_init_foo() {
194; GFX9-LABEL: zero_init_foo:
195; GFX9:       ; %bb.0:
196; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
197; GFX9-NEXT:    s_mov_b32 s0, 0
198; GFX9-NEXT:    s_mov_b32 s1, s0
199; GFX9-NEXT:    s_mov_b32 s2, s0
200; GFX9-NEXT:    s_mov_b32 s3, s0
201; GFX9-NEXT:    v_mov_b32_e32 v0, s0
202; GFX9-NEXT:    v_mov_b32_e32 v1, s1
203; GFX9-NEXT:    v_mov_b32_e32 v2, s2
204; GFX9-NEXT:    v_mov_b32_e32 v3, s3
205; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
206; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
207; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
208; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
209; GFX9-NEXT:    s_waitcnt vmcnt(0)
210; GFX9-NEXT:    s_setpc_b64 s[30:31]
211;
212; GFX10-LABEL: zero_init_foo:
213; GFX10:       ; %bb.0:
214; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
215; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
216; GFX10-NEXT:    s_mov_b32 s0, 0
217; GFX10-NEXT:    s_mov_b32 s1, s0
218; GFX10-NEXT:    s_mov_b32 s2, s0
219; GFX10-NEXT:    s_mov_b32 s3, s0
220; GFX10-NEXT:    v_mov_b32_e32 v0, s0
221; GFX10-NEXT:    v_mov_b32_e32 v1, s1
222; GFX10-NEXT:    v_mov_b32_e32 v2, s2
223; GFX10-NEXT:    v_mov_b32_e32 v3, s3
224; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
225; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
226; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
227; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
228; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
229; GFX10-NEXT:    s_setpc_b64 s[30:31]
230;
231; GFX11-LABEL: zero_init_foo:
232; GFX11:       ; %bb.0:
233; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
234; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
235; GFX11-NEXT:    s_mov_b32 s0, 0
236; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
237; GFX11-NEXT:    s_mov_b32 s1, s0
238; GFX11-NEXT:    s_mov_b32 s2, s0
239; GFX11-NEXT:    s_mov_b32 s3, s0
240; GFX11-NEXT:    v_mov_b32_e32 v0, s0
241; GFX11-NEXT:    v_mov_b32_e32 v1, s1
242; GFX11-NEXT:    v_mov_b32_e32 v2, s2
243; GFX11-NEXT:    v_mov_b32_e32 v3, s3
244; GFX11-NEXT:    s_clause 0x3
245; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:48
246; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:32
247; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:16
248; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32
249; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
250; GFX11-NEXT:    s_setpc_b64 s[30:31]
251;
252; GFX9-PAL-LABEL: zero_init_foo:
253; GFX9-PAL:       ; %bb.0:
254; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
255; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
256; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
257; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
258; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
259; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
260; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
261; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
262; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
263; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
264; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
265; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
266; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
267; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
268; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
269;
270; GFX940-LABEL: zero_init_foo:
271; GFX940:       ; %bb.0:
272; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
273; GFX940-NEXT:    s_mov_b32 s0, 0
274; GFX940-NEXT:    s_mov_b32 s1, s0
275; GFX940-NEXT:    s_mov_b32 s2, s0
276; GFX940-NEXT:    s_mov_b32 s3, s0
277; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
278; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
279; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
280; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
281; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
282; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
283; GFX940-NEXT:    s_waitcnt vmcnt(0)
284; GFX940-NEXT:    s_setpc_b64 s[30:31]
285;
286; GFX10-PAL-LABEL: zero_init_foo:
287; GFX10-PAL:       ; %bb.0:
288; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
289; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
290; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
291; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
292; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
293; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
294; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
295; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
296; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
297; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
298; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
299; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
300; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
301; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
302; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
303; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
304;
305; GFX11-PAL-LABEL: zero_init_foo:
306; GFX11-PAL:       ; %bb.0:
307; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
308; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
309; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
310; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
311; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
312; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
313; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
314; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
315; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
316; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
317; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
318; GFX11-PAL-NEXT:    s_clause 0x3
319; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:48
320; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:32
321; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:16
322; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32
323; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
324; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
325; GCN-LABEL: zero_init_foo:
326; GCN:       ; %bb.0:
327; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
328; GCN-NEXT:    s_mov_b32 s0, 0
329; GCN-NEXT:    s_mov_b32 s1, s0
330; GCN-NEXT:    s_mov_b32 s2, s0
331; GCN-NEXT:    s_mov_b32 s3, s0
332; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
333; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
334; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
335; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
336; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
337; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
338; GCN-NEXT:    s_waitcnt vmcnt(0)
339; GCN-NEXT:    s_setpc_b64 s[30:31]
340  %alloca = alloca [32 x i16], align 2, addrspace(5)
341  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
342  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
343  ret void
344}
345
346define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
347; GFX9-LABEL: store_load_sindex_kernel:
348; GFX9:       ; %bb.0: ; %bb
349; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
350; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
351; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
352; GFX9-NEXT:    v_mov_b32_e32 v0, 15
353; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
354; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
355; GFX9-NEXT:    s_and_b32 s0, s0, 15
356; GFX9-NEXT:    s_add_i32 s1, s1, 4
357; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
358; GFX9-NEXT:    scratch_store_dword off, v0, s1
359; GFX9-NEXT:    s_waitcnt vmcnt(0)
360; GFX9-NEXT:    s_add_i32 s0, s0, 4
361; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
362; GFX9-NEXT:    s_waitcnt vmcnt(0)
363; GFX9-NEXT:    s_endpgm
364;
365; GFX10-LABEL: store_load_sindex_kernel:
366; GFX10:       ; %bb.0: ; %bb
367; GFX10-NEXT:    s_add_u32 s2, s2, s5
368; GFX10-NEXT:    s_addc_u32 s3, s3, 0
369; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
370; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
371; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
372; GFX10-NEXT:    v_mov_b32_e32 v0, 15
373; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
374; GFX10-NEXT:    s_and_b32 s1, s0, 15
375; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
376; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
377; GFX10-NEXT:    s_add_i32 s0, s0, 4
378; GFX10-NEXT:    s_add_i32 s1, s1, 4
379; GFX10-NEXT:    scratch_store_dword off, v0, s0
380; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
381; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
382; GFX10-NEXT:    s_waitcnt vmcnt(0)
383; GFX10-NEXT:    s_endpgm
384;
385; GFX11-LABEL: store_load_sindex_kernel:
386; GFX11:       ; %bb.0: ; %bb
387; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x24
388; GFX11-NEXT:    v_mov_b32_e32 v0, 15
389; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX11-NEXT:    s_and_b32 s1, s0, 15
391; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
392; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
393; GFX11-NEXT:    s_add_i32 s0, s0, 4
394; GFX11-NEXT:    s_add_i32 s1, s1, 4
395; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
396; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
397; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
398; GFX11-NEXT:    s_waitcnt vmcnt(0)
399; GFX11-NEXT:    s_endpgm
400;
401; GFX9-PAL-LABEL: store_load_sindex_kernel:
402; GFX9-PAL:       ; %bb.0: ; %bb
403; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
404; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
405; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
406; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
407; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
408; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
409; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
410; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
411; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
412; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
413; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
414; GFX9-PAL-NEXT:    s_add_i32 s1, s1, 4
415; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
416; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
417; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
418; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
419; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
420; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
421; GFX9-PAL-NEXT:    s_endpgm
422;
423; GFX940-LABEL: store_load_sindex_kernel:
424; GFX940:       ; %bb.0: ; %bb
425; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
426; GFX940-NEXT:    v_mov_b32_e32 v0, 15
427; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
428; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
429; GFX940-NEXT:    s_and_b32 s0, s0, 15
430; GFX940-NEXT:    s_add_i32 s1, s1, 4
431; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
432; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
433; GFX940-NEXT:    s_waitcnt vmcnt(0)
434; GFX940-NEXT:    s_add_i32 s0, s0, 4
435; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
436; GFX940-NEXT:    s_waitcnt vmcnt(0)
437; GFX940-NEXT:    s_endpgm
438;
439; GFX10-PAL-LABEL: store_load_sindex_kernel:
440; GFX10-PAL:       ; %bb.0: ; %bb
441; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
442; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
443; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
444; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
445; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
446; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
447; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
448; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
449; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
450; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
451; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
452; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
453; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
454; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
455; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
456; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
457; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
458; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
459; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
460; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
461; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
462; GFX10-PAL-NEXT:    s_endpgm
463;
464; GFX11-PAL-LABEL: store_load_sindex_kernel:
465; GFX11-PAL:       ; %bb.0: ; %bb
466; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
467; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
468; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
469; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
470; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
471; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
472; GFX11-PAL-NEXT:    s_add_i32 s0, s0, 4
473; GFX11-PAL-NEXT:    s_add_i32 s1, s1, 4
474; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
475; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
476; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
477; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
478; GFX11-PAL-NEXT:    s_endpgm
479; GCN-LABEL: store_load_sindex_kernel:
480; GCN:       ; %bb.0: ; %bb
481; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
482; GCN-NEXT:    v_mov_b32_e32 v0, 15
483; GCN-NEXT:    s_waitcnt lgkmcnt(0)
484; GCN-NEXT:    s_lshl_b32 s1, s0, 2
485; GCN-NEXT:    s_and_b32 s0, s0, 15
486; GCN-NEXT:    s_lshl_b32 s0, s0, 2
487; GCN-NEXT:    s_add_u32 s1, 4, s1
488; GCN-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
489; GCN-NEXT:    s_waitcnt vmcnt(0)
490; GCN-NEXT:    s_add_u32 s0, 4, s0
491; GCN-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
492; GCN-NEXT:    s_waitcnt vmcnt(0)
493; GCN-NEXT:    s_endpgm
494bb:
495  %i = alloca [32 x float], align 4, addrspace(5)
496  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
497  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
498  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
499  store volatile i32 15, i32 addrspace(5)* %i8, align 4
500  %i9 = and i32 %idx, 15
501  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
502  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
503  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
504  ret void
505}
506
507define amdgpu_ps void @store_load_sindex_foo(i32 inreg %idx) {
508; GFX9-LABEL: store_load_sindex_foo:
509; GFX9:       ; %bb.0: ; %bb
510; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
511; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
512; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
513; GFX9-NEXT:    s_add_i32 s0, s0, 4
514; GFX9-NEXT:    v_mov_b32_e32 v0, 15
515; GFX9-NEXT:    scratch_store_dword off, v0, s0
516; GFX9-NEXT:    s_waitcnt vmcnt(0)
517; GFX9-NEXT:    s_and_b32 s0, s2, 15
518; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
519; GFX9-NEXT:    s_add_i32 s0, s0, 4
520; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
521; GFX9-NEXT:    s_waitcnt vmcnt(0)
522; GFX9-NEXT:    s_endpgm
523;
524; GFX10-LABEL: store_load_sindex_foo:
525; GFX10:       ; %bb.0: ; %bb
526; GFX10-NEXT:    s_add_u32 s0, s0, s3
527; GFX10-NEXT:    s_addc_u32 s1, s1, 0
528; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
529; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
530; GFX10-NEXT:    v_mov_b32_e32 v0, 15
531; GFX10-NEXT:    s_and_b32 s0, s2, 15
532; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
533; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
534; GFX10-NEXT:    s_add_i32 s1, s1, 4
535; GFX10-NEXT:    s_add_i32 s0, s0, 4
536; GFX10-NEXT:    scratch_store_dword off, v0, s1
537; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
538; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
539; GFX10-NEXT:    s_waitcnt vmcnt(0)
540; GFX10-NEXT:    s_endpgm
541;
542; GFX11-LABEL: store_load_sindex_foo:
543; GFX11:       ; %bb.0: ; %bb
544; GFX11-NEXT:    v_mov_b32_e32 v0, 15
545; GFX11-NEXT:    s_and_b32 s1, s0, 15
546; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
547; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
548; GFX11-NEXT:    s_add_i32 s0, s0, 4
549; GFX11-NEXT:    s_add_i32 s1, s1, 4
550; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
551; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
552; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
553; GFX11-NEXT:    s_waitcnt vmcnt(0)
554; GFX11-NEXT:    s_endpgm
555;
556; GFX9-PAL-LABEL: store_load_sindex_foo:
557; GFX9-PAL:       ; %bb.0: ; %bb
558; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
559; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
560; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
561; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
562; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
563; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
564; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
565; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
566; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
567; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
568; GFX9-PAL-NEXT:    s_add_i32 s1, s1, 4
569; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
570; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
571; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
572; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
573; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
574; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
575; GFX9-PAL-NEXT:    s_endpgm
576;
577; GFX940-LABEL: store_load_sindex_foo:
578; GFX940:       ; %bb.0: ; %bb
579; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
580; GFX940-NEXT:    s_and_b32 s0, s0, 15
581; GFX940-NEXT:    s_add_i32 s1, s1, 4
582; GFX940-NEXT:    v_mov_b32_e32 v0, 15
583; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
584; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
585; GFX940-NEXT:    s_waitcnt vmcnt(0)
586; GFX940-NEXT:    s_add_i32 s0, s0, 4
587; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
588; GFX940-NEXT:    s_waitcnt vmcnt(0)
589; GFX940-NEXT:    s_endpgm
590;
591; GFX10-PAL-LABEL: store_load_sindex_foo:
592; GFX10-PAL:       ; %bb.0: ; %bb
593; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
594; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
595; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
596; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
598; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
599; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
600; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
601; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
602; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
603; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
604; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
605; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
606; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
607; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
608; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
609; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
610; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
611; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
612; GFX10-PAL-NEXT:    s_endpgm
613;
614; GFX11-PAL-LABEL: store_load_sindex_foo:
615; GFX11-PAL:       ; %bb.0: ; %bb
616; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
617; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
618; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
619; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
620; GFX11-PAL-NEXT:    s_add_i32 s0, s0, 4
621; GFX11-PAL-NEXT:    s_add_i32 s1, s1, 4
622; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
623; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
624; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
625; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
626; GFX11-PAL-NEXT:    s_endpgm
627; GCN-LABEL: store_load_sindex_foo:
628; GCN:       ; %bb.0: ; %bb
629; GCN-NEXT:    s_lshl_b32 s1, s0, 2
630; GCN-NEXT:    s_and_b32 s0, s0, 15
631; GCN-NEXT:    s_lshl_b32 s0, s0, 2
632; GCN-NEXT:    s_add_u32 s1, 4, s1
633; GCN-NEXT:    v_mov_b32_e32 v0, 15
634; GCN-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
635; GCN-NEXT:    s_waitcnt vmcnt(0)
636; GCN-NEXT:    s_add_u32 s0, 4, s0
637; GCN-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
638; GCN-NEXT:    s_waitcnt vmcnt(0)
639; GCN-NEXT:    s_endpgm
640bb:
641  %i = alloca [32 x float], align 4, addrspace(5)
642  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
643  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
644  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
645  store volatile i32 15, i32 addrspace(5)* %i8, align 4
646  %i9 = and i32 %idx, 15
647  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
648  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
649  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
650  ret void
651}
652
653define amdgpu_kernel void @store_load_vindex_kernel() {
654; GFX9-LABEL: store_load_vindex_kernel:
655; GFX9:       ; %bb.0: ; %bb
656; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
657; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
658; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
659; GFX9-NEXT:    v_add_u32_e32 v1, 4, v0
660; GFX9-NEXT:    v_mov_b32_e32 v2, 15
661; GFX9-NEXT:    scratch_store_dword v1, v2, off
662; GFX9-NEXT:    s_waitcnt vmcnt(0)
663; GFX9-NEXT:    v_sub_u32_e32 v0, 4, v0
664; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
665; GFX9-NEXT:    s_waitcnt vmcnt(0)
666; GFX9-NEXT:    s_endpgm
667;
668; GFX10-LABEL: store_load_vindex_kernel:
669; GFX10:       ; %bb.0: ; %bb
670; GFX10-NEXT:    s_add_u32 s0, s0, s3
671; GFX10-NEXT:    s_addc_u32 s1, s1, 0
672; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
673; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
674; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
675; GFX10-NEXT:    v_mov_b32_e32 v2, 15
676; GFX10-NEXT:    v_add_nc_u32_e32 v1, 4, v0
677; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 4, v0
678; GFX10-NEXT:    scratch_store_dword v1, v2, off
679; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
680; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
681; GFX10-NEXT:    s_waitcnt vmcnt(0)
682; GFX10-NEXT:    s_endpgm
683;
684; GFX11-LABEL: store_load_vindex_kernel:
685; GFX11:       ; %bb.0: ; %bb
686; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
687; GFX11-NEXT:    v_mov_b32_e32 v1, 15
688; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
689; GFX11-NEXT:    v_sub_nc_u32_e32 v2, 4, v0
690; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:4 dlc
691; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
692; GFX11-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
693; GFX11-NEXT:    s_waitcnt vmcnt(0)
694; GFX11-NEXT:    s_endpgm
695;
696; GFX9-PAL-LABEL: store_load_vindex_kernel:
697; GFX9-PAL:       ; %bb.0: ; %bb
698; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
699; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
700; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
701; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
702; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 4, v0
703; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
704; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 4, v0
705; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
706; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
707; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
708; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
709; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
710; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
711; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
712; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
713; GFX9-PAL-NEXT:    s_endpgm
714;
715; GFX940-LABEL: store_load_vindex_kernel:
716; GFX940:       ; %bb.0: ; %bb
717; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
718; GFX940-NEXT:    v_mov_b32_e32 v1, 15
719; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
720; GFX940-NEXT:    s_waitcnt vmcnt(0)
721; GFX940-NEXT:    v_sub_u32_e32 v0, 4, v0
722; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
723; GFX940-NEXT:    s_waitcnt vmcnt(0)
724; GFX940-NEXT:    s_endpgm
725;
726; GFX10-PAL-LABEL: store_load_vindex_kernel:
727; GFX10-PAL:       ; %bb.0: ; %bb
728; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
729; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
730; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
731; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
732; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
733; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
734; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
735; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
736; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
737; GFX10-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
738; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
739; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v1, 4, v0
740; GFX10-PAL-NEXT:    v_sub_nc_u32_e32 v0, 4, v0
741; GFX10-PAL-NEXT:    scratch_store_dword v1, v2, off
742; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
743; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
744; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
745; GFX10-PAL-NEXT:    s_endpgm
746;
747; GFX11-PAL-LABEL: store_load_vindex_kernel:
748; GFX11-PAL:       ; %bb.0: ; %bb
749; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
750; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
751; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
752; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 4, v0
753; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:4 dlc
754; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
755; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
756; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
757; GFX11-PAL-NEXT:    s_endpgm
758; GCN-LABEL: store_load_vindex_kernel:
759; GCN:       ; %bb.0: ; %bb
760; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
761; GCN-NEXT:    v_mov_b32_e32 v1, 15
762; GCN-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
763; GCN-NEXT:    s_waitcnt vmcnt(0)
764; GCN-NEXT:    v_sub_u32_e32 v0, 4, v0
765; GCN-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
766; GCN-NEXT:    s_waitcnt vmcnt(0)
767; GCN-NEXT:    s_endpgm
768bb:
769  %i = alloca [32 x float], align 4, addrspace(5)
770  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
771  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
772  %i3 = zext i32 %i2 to i64
773  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
774  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
775  store volatile i32 15, i32 addrspace(5)* %i8, align 4
776  %i9 = sub nsw i32 31, %i2
777  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
778  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
779  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
780  ret void
781}
782
783define void @store_load_vindex_foo(i32 %idx) {
784; GFX9-LABEL: store_load_vindex_foo:
785; GFX9:       ; %bb.0: ; %bb
786; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
787; GFX9-NEXT:    v_mov_b32_e32 v1, s32
788; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
789; GFX9-NEXT:    v_mov_b32_e32 v3, 15
790; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
791; GFX9-NEXT:    scratch_store_dword v2, v3, off
792; GFX9-NEXT:    s_waitcnt vmcnt(0)
793; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
794; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
795; GFX9-NEXT:    s_waitcnt vmcnt(0)
796; GFX9-NEXT:    s_setpc_b64 s[30:31]
797;
798; GFX10-LABEL: store_load_vindex_foo:
799; GFX10:       ; %bb.0: ; %bb
800; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
801; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
802; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
803; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, s32
804; GFX10-NEXT:    v_mov_b32_e32 v2, 15
805; GFX10-NEXT:    v_lshl_add_u32 v1, v1, 2, s32
806; GFX10-NEXT:    scratch_store_dword v0, v2, off
807; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
808; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
809; GFX10-NEXT:    s_waitcnt vmcnt(0)
810; GFX10-NEXT:    s_setpc_b64 s[30:31]
811;
812; GFX11-LABEL: store_load_vindex_foo:
813; GFX11:       ; %bb.0: ; %bb
814; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
815; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
816; GFX11-NEXT:    v_and_b32_e32 v1, 15, v0
817; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
818; GFX11-NEXT:    v_mov_b32_e32 v2, 15
819; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
820; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
821; GFX11-NEXT:    scratch_store_b32 v0, v2, s32 dlc
822; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
823; GFX11-NEXT:    scratch_load_b32 v0, v1, s32 glc dlc
824; GFX11-NEXT:    s_waitcnt vmcnt(0)
825; GFX11-NEXT:    s_setpc_b64 s[30:31]
826;
827; GFX9-PAL-LABEL: store_load_vindex_foo:
828; GFX9-PAL:       ; %bb.0: ; %bb
829; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
830; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s32
831; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
832; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
833; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
834; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
835; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
836; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
837; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
838; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
839; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
840;
841; GFX940-LABEL: store_load_vindex_foo:
842; GFX940:       ; %bb.0: ; %bb
843; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
844; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
845; GFX940-NEXT:    v_mov_b32_e32 v2, 15
846; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
847; GFX940-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
848; GFX940-NEXT:    s_waitcnt vmcnt(0)
849; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
850; GFX940-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
851; GFX940-NEXT:    s_waitcnt vmcnt(0)
852; GFX940-NEXT:    s_setpc_b64 s[30:31]
853;
854; GFX10-PAL-LABEL: store_load_vindex_foo:
855; GFX10-PAL:       ; %bb.0: ; %bb
856; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
857; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
858; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
859; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, s32
860; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
861; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, s32
862; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
863; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
864; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
865; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
866; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
867;
868; GFX11-PAL-LABEL: store_load_vindex_foo:
869; GFX11-PAL:       ; %bb.0: ; %bb
870; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
871; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
872; GFX11-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
873; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
874; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
875; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_3)
876; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
877; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, s32 dlc
878; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
879; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 glc dlc
880; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
881; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
882; GCN-LABEL: store_load_vindex_foo:
883; GCN:       ; %bb.0: ; %bb
884; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
885; GCN-NEXT:    v_mov_b32_e32 v2, 15
886; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
887; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
888; GCN-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
889; GCN-NEXT:    s_waitcnt vmcnt(0)
890; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
891; GCN-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
892; GCN-NEXT:    s_waitcnt vmcnt(0)
893; GCN-NEXT:    s_setpc_b64 s[30:31]
894bb:
895  %i = alloca [32 x float], align 4, addrspace(5)
896  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
897  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
898  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
899  store volatile i32 15, i32 addrspace(5)* %i8, align 4
900  %i9 = and i32 %idx, 15
901  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
902  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
903  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
904  ret void
905}
906
907define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
908; GFX9-LABEL: private_ptr_foo:
909; GFX9:       ; %bb.0:
910; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
911; GFX9-NEXT:    v_mov_b32_e32 v1, 0x41200000
912; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:4
913; GFX9-NEXT:    s_waitcnt vmcnt(0)
914; GFX9-NEXT:    s_setpc_b64 s[30:31]
915;
916; GFX10-LABEL: private_ptr_foo:
917; GFX10:       ; %bb.0:
918; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
919; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
920; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
921; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
922; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
923; GFX10-NEXT:    s_setpc_b64 s[30:31]
924;
925; GFX11-LABEL: private_ptr_foo:
926; GFX11:       ; %bb.0:
927; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
928; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
929; GFX11-NEXT:    v_mov_b32_e32 v1, 0x41200000
930; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:4
931; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
932; GFX11-NEXT:    s_setpc_b64 s[30:31]
933;
934; GFX9-PAL-LABEL: private_ptr_foo:
935; GFX9-PAL:       ; %bb.0:
936; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
937; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
938; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
939; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
940; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
941;
942; GFX940-LABEL: private_ptr_foo:
943; GFX940:       ; %bb.0:
944; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
945; GFX940-NEXT:    v_mov_b32_e32 v1, 0x41200000
946; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4
947; GFX940-NEXT:    s_waitcnt vmcnt(0)
948; GFX940-NEXT:    s_setpc_b64 s[30:31]
949;
950; GFX10-PAL-LABEL: private_ptr_foo:
951; GFX10-PAL:       ; %bb.0:
952; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
953; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
954; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
955; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
956; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
957; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
958;
959; GFX11-PAL-LABEL: private_ptr_foo:
960; GFX11-PAL:       ; %bb.0:
961; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
962; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
963; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
964; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:4
965; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
966; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
967; GCN-LABEL: private_ptr_foo:
968; GCN:       ; %bb.0:
969; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
970; GCN-NEXT:    v_mov_b32_e32 v1, 0x41200000
971; GCN-NEXT:    scratch_store_dword v0, v1, off offset:4
972; GCN-NEXT:    s_waitcnt vmcnt(0)
973; GCN-NEXT:    s_setpc_b64 s[30:31]
974  %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
975  store float 1.000000e+01, float addrspace(5)* %gep, align 4
976  ret void
977}
978
979define amdgpu_kernel void @zero_init_small_offset_kernel() {
980; GFX9-LABEL: zero_init_small_offset_kernel:
981; GFX9:       ; %bb.0:
982; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
983; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
984; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
985; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
986; GFX9-NEXT:    s_waitcnt vmcnt(0)
987; GFX9-NEXT:    s_mov_b32 s0, 0
988; GFX9-NEXT:    s_mov_b32 s1, s0
989; GFX9-NEXT:    s_mov_b32 s2, s0
990; GFX9-NEXT:    s_mov_b32 s3, s0
991; GFX9-NEXT:    v_mov_b32_e32 v0, s0
992; GFX9-NEXT:    v_mov_b32_e32 v1, s1
993; GFX9-NEXT:    v_mov_b32_e32 v2, s2
994; GFX9-NEXT:    v_mov_b32_e32 v3, s3
995; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
996; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:272
997; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
998; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:288
999; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
1000; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:304
1001; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
1002; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:320
1003; GFX9-NEXT:    s_endpgm
1004;
1005; GFX10-LABEL: zero_init_small_offset_kernel:
1006; GFX10:       ; %bb.0:
1007; GFX10-NEXT:    s_add_u32 s0, s0, s3
1008; GFX10-NEXT:    s_addc_u32 s1, s1, 0
1009; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
1010; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
1011; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1012; GFX10-NEXT:    s_waitcnt vmcnt(0)
1013; GFX10-NEXT:    s_mov_b32 s0, 0
1014; GFX10-NEXT:    s_mov_b32 s1, s0
1015; GFX10-NEXT:    s_mov_b32 s2, s0
1016; GFX10-NEXT:    s_mov_b32 s3, s0
1017; GFX10-NEXT:    v_mov_b32_e32 v0, s0
1018; GFX10-NEXT:    v_mov_b32_e32 v1, s1
1019; GFX10-NEXT:    v_mov_b32_e32 v2, s2
1020; GFX10-NEXT:    v_mov_b32_e32 v3, s3
1021; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
1022; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
1023; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
1024; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
1025; GFX10-NEXT:    s_endpgm
1026;
1027; GFX11-LABEL: zero_init_small_offset_kernel:
1028; GFX11:       ; %bb.0:
1029; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1030; GFX11-NEXT:    s_waitcnt vmcnt(0)
1031; GFX11-NEXT:    s_mov_b32 s0, 0
1032; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1033; GFX11-NEXT:    s_mov_b32 s1, s0
1034; GFX11-NEXT:    s_mov_b32 s2, s0
1035; GFX11-NEXT:    s_mov_b32 s3, s0
1036; GFX11-NEXT:    v_mov_b32_e32 v0, s0
1037; GFX11-NEXT:    v_mov_b32_e32 v1, s1
1038; GFX11-NEXT:    v_mov_b32_e32 v2, s2
1039; GFX11-NEXT:    v_mov_b32_e32 v3, s3
1040; GFX11-NEXT:    s_clause 0x3
1041; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:272
1042; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:288
1043; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:304
1044; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:320
1045; GFX11-NEXT:    s_endpgm
1046;
1047; GFX9-PAL-LABEL: zero_init_small_offset_kernel:
1048; GFX9-PAL:       ; %bb.0:
1049; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1050; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1051; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1052; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1053; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
1054; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1056; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1057; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1058; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1059; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1060; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
1061; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1062; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
1063; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
1064; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
1065; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
1066; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
1067; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1068; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:272
1069; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1070; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:288
1071; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1072; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:304
1073; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1074; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:320
1075; GFX9-PAL-NEXT:    s_endpgm
1076;
1077; GFX940-LABEL: zero_init_small_offset_kernel:
1078; GFX940:       ; %bb.0:
1079; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
1080; GFX940-NEXT:    s_waitcnt vmcnt(0)
1081; GFX940-NEXT:    s_mov_b32 s0, 0
1082; GFX940-NEXT:    s_mov_b32 s1, s0
1083; GFX940-NEXT:    s_mov_b32 s2, s0
1084; GFX940-NEXT:    s_mov_b32 s3, s0
1085; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
1086; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
1087; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
1088; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
1089; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
1090; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
1091; GFX940-NEXT:    s_endpgm
1092;
1093; GFX1010-PAL-LABEL: zero_init_small_offset_kernel:
1094; GFX1010-PAL:       ; %bb.0:
1095; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1096; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1097; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1098; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1099; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1100; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1101; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1102; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1103; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1104; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1105; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
1106; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1107; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1108; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
1109; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1110; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
1111; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
1112; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
1113; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
1114; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
1115; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1116; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:272
1117; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1118; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1119; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:288
1120; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1121; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1122; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:304
1123; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
1124; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1125; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:320
1126; GFX1010-PAL-NEXT:    s_endpgm
1127;
1128; GFX1030-PAL-LABEL: zero_init_small_offset_kernel:
1129; GFX1030-PAL:       ; %bb.0:
1130; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1131; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1132; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1133; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1134; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1135; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1136; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1137; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1138; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1139; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1140; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1141; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
1142; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
1143; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1144; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
1145; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
1146; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
1147; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
1148; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
1149; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
1150; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
1151; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
1152; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
1153; GFX1030-PAL-NEXT:    s_endpgm
1154;
1155; GFX11-PAL-LABEL: zero_init_small_offset_kernel:
1156; GFX11-PAL:       ; %bb.0:
1157; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1158; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1159; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
1160; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1161; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
1162; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
1163; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
1164; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
1165; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
1166; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
1167; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
1168; GFX11-PAL-NEXT:    s_clause 0x3
1169; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:272
1170; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:288
1171; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:304
1172; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:320
1173; GFX11-PAL-NEXT:    s_endpgm
1174  %padding = alloca [64 x i32], align 4, addrspace(5)
1175  %alloca = alloca [32 x i16], align 2, addrspace(5)
1176  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
1177  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
1178  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
1179  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
1180  ret void
1181}
1182
1183define void @zero_init_small_offset_foo() {
1184; GFX9-LABEL: zero_init_small_offset_foo:
1185; GFX9:       ; %bb.0:
1186; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1187; GFX9-NEXT:    scratch_load_dword v0, off, s32 glc
1188; GFX9-NEXT:    s_waitcnt vmcnt(0)
1189; GFX9-NEXT:    s_mov_b32 s0, 0
1190; GFX9-NEXT:    s_mov_b32 s1, s0
1191; GFX9-NEXT:    s_mov_b32 s2, s0
1192; GFX9-NEXT:    s_mov_b32 s3, s0
1193; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1194; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1195; GFX9-NEXT:    v_mov_b32_e32 v2, s2
1196; GFX9-NEXT:    v_mov_b32_e32 v3, s3
1197; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1198; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1199; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1200; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1201; GFX9-NEXT:    s_waitcnt vmcnt(0)
1202; GFX9-NEXT:    s_setpc_b64 s[30:31]
1203;
1204; GFX10-LABEL: zero_init_small_offset_foo:
1205; GFX10:       ; %bb.0:
1206; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1207; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1208; GFX10-NEXT:    scratch_load_dword v0, off, s32 glc dlc
1209; GFX10-NEXT:    s_waitcnt vmcnt(0)
1210; GFX10-NEXT:    s_mov_b32 s0, 0
1211; GFX10-NEXT:    s_mov_b32 s1, s0
1212; GFX10-NEXT:    s_mov_b32 s2, s0
1213; GFX10-NEXT:    s_mov_b32 s3, s0
1214; GFX10-NEXT:    v_mov_b32_e32 v0, s0
1215; GFX10-NEXT:    v_mov_b32_e32 v1, s1
1216; GFX10-NEXT:    v_mov_b32_e32 v2, s2
1217; GFX10-NEXT:    v_mov_b32_e32 v3, s3
1218; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1219; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1220; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1221; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1222; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1223; GFX10-NEXT:    s_setpc_b64 s[30:31]
1224;
1225; GFX11-LABEL: zero_init_small_offset_foo:
1226; GFX11:       ; %bb.0:
1227; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1228; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1229; GFX11-NEXT:    scratch_load_b32 v0, off, s32 glc dlc
1230; GFX11-NEXT:    s_waitcnt vmcnt(0)
1231; GFX11-NEXT:    s_mov_b32 s0, 0
1232; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1233; GFX11-NEXT:    s_mov_b32 s1, s0
1234; GFX11-NEXT:    s_mov_b32 s2, s0
1235; GFX11-NEXT:    s_mov_b32 s3, s0
1236; GFX11-NEXT:    v_mov_b32_e32 v0, s0
1237; GFX11-NEXT:    v_mov_b32_e32 v1, s1
1238; GFX11-NEXT:    v_mov_b32_e32 v2, s2
1239; GFX11-NEXT:    v_mov_b32_e32 v3, s3
1240; GFX11-NEXT:    s_clause 0x3
1241; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:256
1242; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:272
1243; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:288
1244; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:304
1245; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1246; GFX11-NEXT:    s_setpc_b64 s[30:31]
1247;
1248; GFX9-PAL-LABEL: zero_init_small_offset_foo:
1249; GFX9-PAL:       ; %bb.0:
1250; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1251; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s32 glc
1252; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1253; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
1254; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
1255; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1256; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
1257; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
1258; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
1259; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
1260; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
1261; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1262; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1263; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1264; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1265; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1266; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
1267;
1268; GFX940-LABEL: zero_init_small_offset_foo:
1269; GFX940:       ; %bb.0:
1270; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1271; GFX940-NEXT:    scratch_load_dword v0, off, s32 sc0 sc1
1272; GFX940-NEXT:    s_waitcnt vmcnt(0)
1273; GFX940-NEXT:    s_mov_b32 s0, 0
1274; GFX940-NEXT:    s_mov_b32 s1, s0
1275; GFX940-NEXT:    s_mov_b32 s2, s0
1276; GFX940-NEXT:    s_mov_b32 s3, s0
1277; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
1278; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
1279; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1280; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1281; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1282; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1283; GFX940-NEXT:    s_waitcnt vmcnt(0)
1284; GFX940-NEXT:    s_setpc_b64 s[30:31]
1285;
1286; GFX10-PAL-LABEL: zero_init_small_offset_foo:
1287; GFX10-PAL:       ; %bb.0:
1288; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1289; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1290; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s32 glc dlc
1291; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1292; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
1293; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
1294; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
1295; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
1296; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
1297; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
1298; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
1299; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
1300; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1301; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1302; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1303; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1304; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1305; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
1306;
1307; GFX11-PAL-LABEL: zero_init_small_offset_foo:
1308; GFX11-PAL:       ; %bb.0:
1309; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1310; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1311; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s32 glc dlc
1312; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1313; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
1314; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1315; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
1316; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
1317; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
1318; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
1319; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
1320; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
1321; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
1322; GFX11-PAL-NEXT:    s_clause 0x3
1323; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:256
1324; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:272
1325; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:288
1326; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:304
1327; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1328; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
1329; GCN-LABEL: zero_init_small_offset_foo:
1330; GCN:       ; %bb.0:
1331; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1332; GCN-NEXT:    scratch_load_dword v0, off, s32 sc0 sc1
1333; GCN-NEXT:    s_waitcnt vmcnt(0)
1334; GCN-NEXT:    s_mov_b32 s0, 0
1335; GCN-NEXT:    s_mov_b32 s1, s0
1336; GCN-NEXT:    s_mov_b32 s2, s0
1337; GCN-NEXT:    s_mov_b32 s3, s0
1338; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
1339; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
1340; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
1341; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
1342; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
1343; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
1344; GCN-NEXT:    s_waitcnt vmcnt(0)
1345; GCN-NEXT:    s_setpc_b64 s[30:31]
1346  %padding = alloca [64 x i32], align 4, addrspace(5)
1347  %alloca = alloca [32 x i16], align 2, addrspace(5)
1348  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
1349  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
1350  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
1351  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
1352  ret void
1353}
1354
1355define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
1356; GFX9-LABEL: store_load_sindex_small_offset_kernel:
1357; GFX9:       ; %bb.0: ; %bb
1358; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
1359; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
1360; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1361; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
1362; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1363; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1364; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
1365; GFX9-NEXT:    s_and_b32 s0, s0, 15
1366; GFX9-NEXT:    v_mov_b32_e32 v0, 15
1367; GFX9-NEXT:    s_addk_i32 s1, 0x104
1368; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
1369; GFX9-NEXT:    scratch_store_dword off, v0, s1
1370; GFX9-NEXT:    s_waitcnt vmcnt(0)
1371; GFX9-NEXT:    s_addk_i32 s0, 0x104
1372; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
1373; GFX9-NEXT:    s_waitcnt vmcnt(0)
1374; GFX9-NEXT:    s_endpgm
1375;
1376; GFX10-LABEL: store_load_sindex_small_offset_kernel:
1377; GFX10:       ; %bb.0: ; %bb
1378; GFX10-NEXT:    s_add_u32 s2, s2, s5
1379; GFX10-NEXT:    s_addc_u32 s3, s3, 0
1380; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1381; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1382; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
1383; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1384; GFX10-NEXT:    s_waitcnt vmcnt(0)
1385; GFX10-NEXT:    v_mov_b32_e32 v0, 15
1386; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1387; GFX10-NEXT:    s_and_b32 s1, s0, 15
1388; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
1389; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
1390; GFX10-NEXT:    s_addk_i32 s0, 0x104
1391; GFX10-NEXT:    s_addk_i32 s1, 0x104
1392; GFX10-NEXT:    scratch_store_dword off, v0, s0
1393; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1394; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1395; GFX10-NEXT:    s_waitcnt vmcnt(0)
1396; GFX10-NEXT:    s_endpgm
1397;
1398; GFX11-LABEL: store_load_sindex_small_offset_kernel:
1399; GFX11:       ; %bb.0: ; %bb
1400; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x24
1401; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1402; GFX11-NEXT:    s_waitcnt vmcnt(0)
1403; GFX11-NEXT:    v_mov_b32_e32 v0, 15
1404; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
1405; GFX11-NEXT:    s_and_b32 s1, s0, 15
1406; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
1407; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
1408; GFX11-NEXT:    s_addk_i32 s0, 0x104
1409; GFX11-NEXT:    s_addk_i32 s1, 0x104
1410; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
1411; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1412; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1413; GFX11-NEXT:    s_waitcnt vmcnt(0)
1414; GFX11-NEXT:    s_endpgm
1415;
1416; GFX9-PAL-LABEL: store_load_sindex_small_offset_kernel:
1417; GFX9-PAL:       ; %bb.0: ; %bb
1418; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
1419; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
1420; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1421; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1422; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1423; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1424; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1425; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
1426; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
1427; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1428; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1429; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
1430; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
1431; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
1432; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x104
1433; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1434; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
1435; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1436; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
1437; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
1438; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1439; GFX9-PAL-NEXT:    s_endpgm
1440;
1441; GFX940-LABEL: store_load_sindex_small_offset_kernel:
1442; GFX940:       ; %bb.0: ; %bb
1443; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
1444; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
1445; GFX940-NEXT:    s_waitcnt vmcnt(0)
1446; GFX940-NEXT:    v_mov_b32_e32 v0, 15
1447; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
1448; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
1449; GFX940-NEXT:    s_and_b32 s0, s0, 15
1450; GFX940-NEXT:    s_addk_i32 s1, 0x104
1451; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
1452; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
1453; GFX940-NEXT:    s_waitcnt vmcnt(0)
1454; GFX940-NEXT:    s_addk_i32 s0, 0x104
1455; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
1456; GFX940-NEXT:    s_waitcnt vmcnt(0)
1457; GFX940-NEXT:    s_endpgm
1458;
1459; GFX1010-PAL-LABEL: store_load_sindex_small_offset_kernel:
1460; GFX1010-PAL:       ; %bb.0: ; %bb
1461; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
1462; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
1463; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1464; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1465; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1466; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3
1467; GFX1010-PAL-NEXT:    s_addc_u32 s5, s5, 0
1468; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
1469; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
1470; GFX1010-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1471; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1472; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1473; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1474; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
1475; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1476; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
1477; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1478; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1479; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x104
1480; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x104
1481; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
1482; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1483; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1484; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1485; GFX1010-PAL-NEXT:    s_endpgm
1486;
1487; GFX1030-PAL-LABEL: store_load_sindex_small_offset_kernel:
1488; GFX1030-PAL:       ; %bb.0: ; %bb
1489; GFX1030-PAL-NEXT:    s_getpc_b64 s[4:5]
1490; GFX1030-PAL-NEXT:    s_mov_b32 s4, s0
1491; GFX1030-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1492; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1493; GFX1030-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
1494; GFX1030-PAL-NEXT:    s_add_u32 s4, s4, s3
1495; GFX1030-PAL-NEXT:    s_addc_u32 s5, s5, 0
1496; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
1497; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
1498; GFX1030-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
1499; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1500; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1501; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
1502; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1503; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
1504; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1505; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1506; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x104
1507; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x104
1508; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
1509; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1510; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1511; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1512; GFX1030-PAL-NEXT:    s_endpgm
1513;
1514; GFX11-PAL-LABEL: store_load_sindex_small_offset_kernel:
1515; GFX11-PAL:       ; %bb.0: ; %bb
1516; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
1517; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1518; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1519; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
1520; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1521; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
1522; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1523; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1524; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x104
1525; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x104
1526; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
1527; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1528; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1529; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1530; GFX11-PAL-NEXT:    s_endpgm
1531bb:
1532  %padding = alloca [64 x i32], align 4, addrspace(5)
1533  %i = alloca [32 x float], align 4, addrspace(5)
1534  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
1535  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
1536  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
1537  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
1538  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
1539  store volatile i32 15, i32 addrspace(5)* %i8, align 4
1540  %i9 = and i32 %idx, 15
1541  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
1542  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
1543  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
1544  ret void
1545}
1546
1547define amdgpu_ps void @store_load_sindex_small_offset_foo(i32 inreg %idx) {
1548; GFX9-LABEL: store_load_sindex_small_offset_foo:
1549; GFX9:       ; %bb.0: ; %bb
1550; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
1551; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
1552; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
1553; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1554; GFX9-NEXT:    s_waitcnt vmcnt(0)
1555; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
1556; GFX9-NEXT:    s_addk_i32 s0, 0x104
1557; GFX9-NEXT:    v_mov_b32_e32 v0, 15
1558; GFX9-NEXT:    scratch_store_dword off, v0, s0
1559; GFX9-NEXT:    s_waitcnt vmcnt(0)
1560; GFX9-NEXT:    s_and_b32 s0, s2, 15
1561; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
1562; GFX9-NEXT:    s_addk_i32 s0, 0x104
1563; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
1564; GFX9-NEXT:    s_waitcnt vmcnt(0)
1565; GFX9-NEXT:    s_endpgm
1566;
1567; GFX10-LABEL: store_load_sindex_small_offset_foo:
1568; GFX10:       ; %bb.0: ; %bb
1569; GFX10-NEXT:    s_add_u32 s0, s0, s3
1570; GFX10-NEXT:    s_addc_u32 s1, s1, 0
1571; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
1572; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
1573; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1574; GFX10-NEXT:    s_waitcnt vmcnt(0)
1575; GFX10-NEXT:    v_mov_b32_e32 v0, 15
1576; GFX10-NEXT:    s_and_b32 s0, s2, 15
1577; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
1578; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
1579; GFX10-NEXT:    s_addk_i32 s1, 0x104
1580; GFX10-NEXT:    s_addk_i32 s0, 0x104
1581; GFX10-NEXT:    scratch_store_dword off, v0, s1
1582; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1583; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
1584; GFX10-NEXT:    s_waitcnt vmcnt(0)
1585; GFX10-NEXT:    s_endpgm
1586;
1587; GFX11-LABEL: store_load_sindex_small_offset_foo:
1588; GFX11:       ; %bb.0: ; %bb
1589; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1590; GFX11-NEXT:    s_waitcnt vmcnt(0)
1591; GFX11-NEXT:    v_mov_b32_e32 v0, 15
1592; GFX11-NEXT:    s_and_b32 s1, s0, 15
1593; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
1594; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
1595; GFX11-NEXT:    s_addk_i32 s0, 0x104
1596; GFX11-NEXT:    s_addk_i32 s1, 0x104
1597; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
1598; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1599; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1600; GFX11-NEXT:    s_waitcnt vmcnt(0)
1601; GFX11-NEXT:    s_endpgm
1602;
1603; GFX9-PAL-LABEL: store_load_sindex_small_offset_foo:
1604; GFX9-PAL:       ; %bb.0: ; %bb
1605; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1606; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1607; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1608; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1609; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1610; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1611; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1612; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1613; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
1614; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1615; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
1616; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
1617; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x104
1618; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
1619; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1620; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
1621; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1622; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
1623; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
1624; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1625; GFX9-PAL-NEXT:    s_endpgm
1626;
1627; GFX940-LABEL: store_load_sindex_small_offset_foo:
1628; GFX940:       ; %bb.0: ; %bb
1629; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
1630; GFX940-NEXT:    s_waitcnt vmcnt(0)
1631; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
1632; GFX940-NEXT:    s_and_b32 s0, s0, 15
1633; GFX940-NEXT:    s_addk_i32 s1, 0x104
1634; GFX940-NEXT:    v_mov_b32_e32 v0, 15
1635; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
1636; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
1637; GFX940-NEXT:    s_waitcnt vmcnt(0)
1638; GFX940-NEXT:    s_addk_i32 s0, 0x104
1639; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
1640; GFX940-NEXT:    s_waitcnt vmcnt(0)
1641; GFX940-NEXT:    s_endpgm
1642;
1643; GFX1010-PAL-LABEL: store_load_sindex_small_offset_foo:
1644; GFX1010-PAL:       ; %bb.0: ; %bb
1645; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1646; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1647; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1648; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1649; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1650; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1651; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1652; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1653; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1654; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1655; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
1656; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
1657; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1658; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
1659; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1660; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1661; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x104
1662; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x104
1663; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
1664; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1665; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1666; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1667; GFX1010-PAL-NEXT:    s_endpgm
1668;
1669; GFX1030-PAL-LABEL: store_load_sindex_small_offset_foo:
1670; GFX1030-PAL:       ; %bb.0: ; %bb
1671; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1672; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1673; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1674; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1675; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1676; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1677; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1678; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1679; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1680; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
1681; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1682; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
1683; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
1684; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1685; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1686; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x104
1687; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x104
1688; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
1689; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1690; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
1691; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1692; GFX1030-PAL-NEXT:    s_endpgm
1693;
1694; GFX11-PAL-LABEL: store_load_sindex_small_offset_foo:
1695; GFX11-PAL:       ; %bb.0: ; %bb
1696; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
1697; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1698; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
1699; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
1700; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
1701; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
1702; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x104
1703; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x104
1704; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
1705; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1706; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
1707; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1708; GFX11-PAL-NEXT:    s_endpgm
1709bb:
1710  %padding = alloca [64 x i32], align 4, addrspace(5)
1711  %i = alloca [32 x float], align 4, addrspace(5)
1712  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
1713  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
1714  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
1715  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
1716  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
1717  store volatile i32 15, i32 addrspace(5)* %i8, align 4
1718  %i9 = and i32 %idx, 15
1719  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
1720  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
1721  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
1722  ret void
1723}
1724
1725define amdgpu_kernel void @store_load_vindex_small_offset_kernel() {
1726; GFX9-LABEL: store_load_vindex_small_offset_kernel:
1727; GFX9:       ; %bb.0: ; %bb
1728; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
1729; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
1730; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
1731; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
1732; GFX9-NEXT:    s_waitcnt vmcnt(0)
1733; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1734; GFX9-NEXT:    v_add_u32_e32 v1, 0x104, v0
1735; GFX9-NEXT:    v_mov_b32_e32 v2, 15
1736; GFX9-NEXT:    scratch_store_dword v1, v2, off
1737; GFX9-NEXT:    s_waitcnt vmcnt(0)
1738; GFX9-NEXT:    v_sub_u32_e32 v0, 0x104, v0
1739; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
1740; GFX9-NEXT:    s_waitcnt vmcnt(0)
1741; GFX9-NEXT:    s_endpgm
1742;
1743; GFX10-LABEL: store_load_vindex_small_offset_kernel:
1744; GFX10:       ; %bb.0: ; %bb
1745; GFX10-NEXT:    s_add_u32 s0, s0, s3
1746; GFX10-NEXT:    s_addc_u32 s1, s1, 0
1747; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
1748; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
1749; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1750; GFX10-NEXT:    v_mov_b32_e32 v2, 15
1751; GFX10-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
1752; GFX10-NEXT:    s_waitcnt vmcnt(0)
1753; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x104, v0
1754; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 0x104, v0
1755; GFX10-NEXT:    scratch_store_dword v1, v2, off
1756; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1757; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
1758; GFX10-NEXT:    s_waitcnt vmcnt(0)
1759; GFX10-NEXT:    s_endpgm
1760;
1761; GFX11-LABEL: store_load_vindex_small_offset_kernel:
1762; GFX11:       ; %bb.0: ; %bb
1763; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1764; GFX11-NEXT:    v_mov_b32_e32 v1, 15
1765; GFX11-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
1766; GFX11-NEXT:    s_waitcnt vmcnt(0)
1767; GFX11-NEXT:    v_sub_nc_u32_e32 v2, 0x104, v0
1768; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:260 dlc
1769; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1770; GFX11-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
1771; GFX11-NEXT:    s_waitcnt vmcnt(0)
1772; GFX11-NEXT:    s_endpgm
1773;
1774; GFX9-PAL-LABEL: store_load_vindex_small_offset_kernel:
1775; GFX9-PAL:       ; %bb.0: ; %bb
1776; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
1777; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
1778; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1779; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
1780; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1781; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
1782; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1783; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1784; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
1785; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1786; GFX9-PAL-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
1787; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1788; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 0x104, v0
1789; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
1790; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1791; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 0x104, v0
1792; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
1793; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1794; GFX9-PAL-NEXT:    s_endpgm
1795;
1796; GFX940-LABEL: store_load_vindex_small_offset_kernel:
1797; GFX940:       ; %bb.0: ; %bb
1798; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
1799; GFX940-NEXT:    s_waitcnt vmcnt(0)
1800; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1801; GFX940-NEXT:    v_mov_b32_e32 v1, 15
1802; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:260 sc0 sc1
1803; GFX940-NEXT:    s_waitcnt vmcnt(0)
1804; GFX940-NEXT:    v_sub_u32_e32 v0, 0x104, v0
1805; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
1806; GFX940-NEXT:    s_waitcnt vmcnt(0)
1807; GFX940-NEXT:    s_endpgm
1808;
1809; GFX1010-PAL-LABEL: store_load_vindex_small_offset_kernel:
1810; GFX1010-PAL:       ; %bb.0: ; %bb
1811; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
1812; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
1813; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1814; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1815; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1816; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
1817; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
1818; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1819; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1820; GFX1010-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1821; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, 15
1822; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
1823; GFX1010-PAL-NEXT:    scratch_load_dword v3, off, vcc_lo offset:4 glc dlc
1824; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1825; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x104, v0
1826; GFX1010-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x104, v0
1827; GFX1010-PAL-NEXT:    scratch_store_dword v1, v2, off
1828; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1829; GFX1010-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
1830; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
1831; GFX1010-PAL-NEXT:    s_endpgm
1832;
1833; GFX1030-PAL-LABEL: store_load_vindex_small_offset_kernel:
1834; GFX1030-PAL:       ; %bb.0: ; %bb
1835; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
1836; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
1837; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1838; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
1839; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
1840; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
1841; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
1842; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1843; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1844; GFX1030-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1845; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, 15
1846; GFX1030-PAL-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
1847; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1848; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x104, v0
1849; GFX1030-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x104, v0
1850; GFX1030-PAL-NEXT:    scratch_store_dword v1, v2, off
1851; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1852; GFX1030-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
1853; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
1854; GFX1030-PAL-NEXT:    s_endpgm
1855;
1856; GFX11-PAL-LABEL: store_load_vindex_small_offset_kernel:
1857; GFX11-PAL:       ; %bb.0: ; %bb
1858; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1859; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
1860; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
1861; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1862; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 0x104, v0
1863; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:260 dlc
1864; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1865; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
1866; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1867; GFX11-PAL-NEXT:    s_endpgm
1868bb:
1869  %padding = alloca [64 x i32], align 4, addrspace(5)
1870  %i = alloca [32 x float], align 4, addrspace(5)
1871  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
1872  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
1873  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
1874  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
1875  %i3 = zext i32 %i2 to i64
1876  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
1877  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
1878  store volatile i32 15, i32 addrspace(5)* %i8, align 4
1879  %i9 = sub nsw i32 31, %i2
1880  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
1881  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
1882  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
1883  ret void
1884}
1885
1886define void @store_load_vindex_small_offset_foo(i32 %idx) {
1887; GFX9-LABEL: store_load_vindex_small_offset_foo:
1888; GFX9:       ; %bb.0: ; %bb
1889; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1890; GFX9-NEXT:    scratch_load_dword v1, off, s32 glc
1891; GFX9-NEXT:    s_waitcnt vmcnt(0)
1892; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x100
1893; GFX9-NEXT:    v_mov_b32_e32 v1, vcc_hi
1894; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
1895; GFX9-NEXT:    v_mov_b32_e32 v3, 15
1896; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
1897; GFX9-NEXT:    scratch_store_dword v2, v3, off
1898; GFX9-NEXT:    s_waitcnt vmcnt(0)
1899; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
1900; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
1901; GFX9-NEXT:    s_waitcnt vmcnt(0)
1902; GFX9-NEXT:    s_setpc_b64 s[30:31]
1903;
1904; GFX10-LABEL: store_load_vindex_small_offset_foo:
1905; GFX10:       ; %bb.0: ; %bb
1906; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1907; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1908; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
1909; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1910; GFX10-NEXT:    v_mov_b32_e32 v2, 15
1911; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
1912; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1913; GFX10-NEXT:    scratch_load_dword v3, off, s32 glc dlc
1914; GFX10-NEXT:    s_waitcnt vmcnt(0)
1915; GFX10-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
1916; GFX10-NEXT:    scratch_store_dword v0, v2, off
1917; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1918; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
1919; GFX10-NEXT:    s_waitcnt vmcnt(0)
1920; GFX10-NEXT:    s_setpc_b64 s[30:31]
1921;
1922; GFX11-LABEL: store_load_vindex_small_offset_foo:
1923; GFX11:       ; %bb.0: ; %bb
1924; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1925; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1926; GFX11-NEXT:    v_and_b32_e32 v1, 15, v0
1927; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1928; GFX11-NEXT:    v_mov_b32_e32 v2, 15
1929; GFX11-NEXT:    scratch_load_b32 v3, off, s32 glc dlc
1930; GFX11-NEXT:    s_waitcnt vmcnt(0)
1931; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
1932; GFX11-NEXT:    scratch_store_b32 v0, v2, s32 offset:256 dlc
1933; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
1934; GFX11-NEXT:    scratch_load_b32 v0, v1, s32 offset:256 glc dlc
1935; GFX11-NEXT:    s_waitcnt vmcnt(0)
1936; GFX11-NEXT:    s_setpc_b64 s[30:31]
1937;
1938; GFX9-PAL-LABEL: store_load_vindex_small_offset_foo:
1939; GFX9-PAL:       ; %bb.0: ; %bb
1940; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1941; GFX9-PAL-NEXT:    scratch_load_dword v1, off, s32 glc
1942; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1943; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x100
1944; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, vcc_hi
1945; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
1946; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
1947; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
1948; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
1949; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1950; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
1951; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
1952; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
1953; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
1954;
1955; GFX940-LABEL: store_load_vindex_small_offset_foo:
1956; GFX940:       ; %bb.0: ; %bb
1957; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1958; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
1959; GFX940-NEXT:    s_waitcnt vmcnt(0)
1960; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
1961; GFX940-NEXT:    v_mov_b32_e32 v2, 15
1962; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
1963; GFX940-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
1964; GFX940-NEXT:    s_waitcnt vmcnt(0)
1965; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1966; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
1967; GFX940-NEXT:    s_waitcnt vmcnt(0)
1968; GFX940-NEXT:    s_setpc_b64 s[30:31]
1969;
1970; GFX10-PAL-LABEL: store_load_vindex_small_offset_foo:
1971; GFX10-PAL:       ; %bb.0: ; %bb
1972; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1973; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1974; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
1975; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1976; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
1977; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
1978; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x100
1979; GFX10-PAL-NEXT:    scratch_load_dword v3, off, s32 glc dlc
1980; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1981; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
1982; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
1983; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1984; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
1985; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
1986; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
1987;
1988; GFX11-PAL-LABEL: store_load_vindex_small_offset_foo:
1989; GFX11-PAL:       ; %bb.0: ; %bb
1990; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1991; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
1992; GFX11-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
1993; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1994; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
1995; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, s32 glc dlc
1996; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
1997; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
1998; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, s32 offset:256 dlc
1999; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2000; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 offset:256 glc dlc
2001; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2002; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
2003; GCN-LABEL: store_load_vindex_small_offset_foo:
2004; GCN:       ; %bb.0: ; %bb
2005; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2006; GCN-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
2007; GCN-NEXT:    s_waitcnt vmcnt(0)
2008; GCN-NEXT:    v_mov_b32_e32 v2, 15
2009; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
2010; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
2011; GCN-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
2012; GCN-NEXT:    s_waitcnt vmcnt(0)
2013; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2014; GCN-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
2015; GCN-NEXT:    s_waitcnt vmcnt(0)
2016; GCN-NEXT:    s_setpc_b64 s[30:31]
2017bb:
2018  %padding = alloca [64 x i32], align 4, addrspace(5)
2019  %i = alloca [32 x float], align 4, addrspace(5)
2020  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
2021  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2022  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
2023  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
2024  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
2025  store volatile i32 15, i32 addrspace(5)* %i8, align 4
2026  %i9 = and i32 %idx, 15
2027  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
2028  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
2029  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
2030  ret void
2031}
2032
2033define amdgpu_kernel void @zero_init_large_offset_kernel() {
2034; GFX9-LABEL: zero_init_large_offset_kernel:
2035; GFX9:       ; %bb.0:
2036; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
2037; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
2038; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
2039; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:16 glc
2040; GFX9-NEXT:    s_waitcnt vmcnt(0)
2041; GFX9-NEXT:    s_mov_b32 s0, 0
2042; GFX9-NEXT:    s_mov_b32 s1, s0
2043; GFX9-NEXT:    s_mov_b32 s2, s0
2044; GFX9-NEXT:    s_mov_b32 s3, s0
2045; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2046; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2047; GFX9-NEXT:    v_mov_b32_e32 v2, s2
2048; GFX9-NEXT:    v_mov_b32_e32 v3, s3
2049; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
2050; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2051; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
2052; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2053; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
2054; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2055; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
2056; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2057; GFX9-NEXT:    s_endpgm
2058;
2059; GFX10-LABEL: zero_init_large_offset_kernel:
2060; GFX10:       ; %bb.0:
2061; GFX10-NEXT:    s_add_u32 s0, s0, s3
2062; GFX10-NEXT:    s_addc_u32 s1, s1, 0
2063; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
2064; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
2065; GFX10-NEXT:    scratch_load_dword v0, off, off offset:16 glc dlc
2066; GFX10-NEXT:    s_waitcnt vmcnt(0)
2067; GFX10-NEXT:    s_mov_b32 s0, 0
2068; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
2069; GFX10-NEXT:    s_mov_b32 s1, s0
2070; GFX10-NEXT:    s_mov_b32 s2, s0
2071; GFX10-NEXT:    s_mov_b32 s3, s0
2072; GFX10-NEXT:    v_mov_b32_e32 v0, s0
2073; GFX10-NEXT:    v_mov_b32_e32 v1, s1
2074; GFX10-NEXT:    v_mov_b32_e32 v2, s2
2075; GFX10-NEXT:    v_mov_b32_e32 v3, s3
2076; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2077; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
2078; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2079; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
2080; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2081; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
2082; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2083; GFX10-NEXT:    s_endpgm
2084;
2085; GFX11-LABEL: zero_init_large_offset_kernel:
2086; GFX11:       ; %bb.0:
2087; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:16 glc dlc
2088; GFX11-NEXT:    s_waitcnt vmcnt(0)
2089; GFX11-NEXT:    s_mov_b32 s0, 0
2090; GFX11-NEXT:    s_movk_i32 vcc_lo, 0x4010
2091; GFX11-NEXT:    s_mov_b32 s1, s0
2092; GFX11-NEXT:    s_mov_b32 s2, s0
2093; GFX11-NEXT:    s_mov_b32 s3, s0
2094; GFX11-NEXT:    v_mov_b32_e32 v0, s0
2095; GFX11-NEXT:    v_mov_b32_e32 v1, s1
2096; GFX11-NEXT:    v_mov_b32_e32 v2, s2
2097; GFX11-NEXT:    v_mov_b32_e32 v3, s3
2098; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2099; GFX11-NEXT:    s_movk_i32 vcc_lo, 0x4010
2100; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2101; GFX11-NEXT:    s_movk_i32 vcc_lo, 0x4010
2102; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2103; GFX11-NEXT:    s_movk_i32 vcc_lo, 0x4010
2104; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2105; GFX11-NEXT:    s_endpgm
2106;
2107; GFX9-PAL-LABEL: zero_init_large_offset_kernel:
2108; GFX9-PAL:       ; %bb.0:
2109; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2110; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2111; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2112; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2113; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
2114; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2115; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2116; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2117; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2118; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:16 glc
2119; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2120; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
2121; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2122; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
2123; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
2124; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
2125; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
2126; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
2127; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2128; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2129; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2130; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2131; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2132; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2133; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
2134; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2135; GFX9-PAL-NEXT:    s_endpgm
2136;
2137; GFX940-LABEL: zero_init_large_offset_kernel:
2138; GFX940:       ; %bb.0:
2139; GFX940-NEXT:    scratch_load_dword v0, off, off offset:16 sc0 sc1
2140; GFX940-NEXT:    s_waitcnt vmcnt(0)
2141; GFX940-NEXT:    s_mov_b32 s0, 0
2142; GFX940-NEXT:    s_mov_b32 s1, s0
2143; GFX940-NEXT:    s_mov_b32 s2, s0
2144; GFX940-NEXT:    s_mov_b32 s3, s0
2145; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
2146; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
2147; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
2148; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2149; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
2150; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2151; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
2152; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2153; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
2154; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2155; GFX940-NEXT:    s_endpgm
2156;
2157; GFX1010-PAL-LABEL: zero_init_large_offset_kernel:
2158; GFX1010-PAL:       ; %bb.0:
2159; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2160; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2161; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2162; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2163; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2164; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2165; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2166; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2167; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2168; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2169; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
2170; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:16 glc dlc
2171; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2172; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
2173; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2174; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
2175; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
2176; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
2177; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
2178; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
2179; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2180; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2181; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2182; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2183; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2184; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2185; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2186; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2187; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2188; GFX1010-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2189; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2190; GFX1010-PAL-NEXT:    s_endpgm
2191;
2192; GFX1030-PAL-LABEL: zero_init_large_offset_kernel:
2193; GFX1030-PAL:       ; %bb.0:
2194; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2195; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2196; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2197; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2198; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2199; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2200; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2201; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2202; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2203; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:16 glc dlc
2204; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2205; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
2206; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2207; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
2208; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2209; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
2210; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
2211; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
2212; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
2213; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
2214; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2215; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2216; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2217; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2218; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2219; GFX1030-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2220; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2221; GFX1030-PAL-NEXT:    s_endpgm
2222;
2223; GFX11-PAL-LABEL: zero_init_large_offset_kernel:
2224; GFX11-PAL:       ; %bb.0:
2225; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:16 glc dlc
2226; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2227; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
2228; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2229; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
2230; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
2231; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
2232; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
2233; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
2234; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
2235; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
2236; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2237; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2238; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2239; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2240; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2241; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
2242; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2243; GFX11-PAL-NEXT:    s_endpgm
2244  %padding = alloca [4096 x i32], align 4, addrspace(5)
2245  %alloca = alloca [32 x i16], align 2, addrspace(5)
2246  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
2247  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2248  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
2249  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
2250  ret void
2251}
2252
2253define void @zero_init_large_offset_foo() {
2254; GFX9-LABEL: zero_init_large_offset_foo:
2255; GFX9:       ; %bb.0:
2256; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2257; GFX9-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc
2258; GFX9-NEXT:    s_waitcnt vmcnt(0)
2259; GFX9-NEXT:    s_mov_b32 s0, 0
2260; GFX9-NEXT:    s_mov_b32 s1, s0
2261; GFX9-NEXT:    s_mov_b32 s2, s0
2262; GFX9-NEXT:    s_mov_b32 s3, s0
2263; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2264; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2265; GFX9-NEXT:    v_mov_b32_e32 v2, s2
2266; GFX9-NEXT:    v_mov_b32_e32 v3, s3
2267; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2268; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2269; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2270; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2271; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2272; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2273; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2274; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2275; GFX9-NEXT:    s_waitcnt vmcnt(0)
2276; GFX9-NEXT:    s_setpc_b64 s[30:31]
2277;
2278; GFX10-LABEL: zero_init_large_offset_foo:
2279; GFX10:       ; %bb.0:
2280; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2281; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2282; GFX10-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
2283; GFX10-NEXT:    s_waitcnt vmcnt(0)
2284; GFX10-NEXT:    s_mov_b32 s0, 0
2285; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2286; GFX10-NEXT:    s_mov_b32 s1, s0
2287; GFX10-NEXT:    s_mov_b32 s2, s0
2288; GFX10-NEXT:    s_mov_b32 s3, s0
2289; GFX10-NEXT:    v_mov_b32_e32 v0, s0
2290; GFX10-NEXT:    v_mov_b32_e32 v1, s1
2291; GFX10-NEXT:    v_mov_b32_e32 v2, s2
2292; GFX10-NEXT:    v_mov_b32_e32 v3, s3
2293; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2294; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2295; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2296; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2297; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2298; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2299; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2300; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2301; GFX10-NEXT:    s_setpc_b64 s[30:31]
2302;
2303; GFX11-LABEL: zero_init_large_offset_foo:
2304; GFX11:       ; %bb.0:
2305; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2306; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
2307; GFX11-NEXT:    scratch_load_b32 v0, off, s32 offset:16 glc dlc
2308; GFX11-NEXT:    s_waitcnt vmcnt(0)
2309; GFX11-NEXT:    s_mov_b32 s0, 0
2310; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2311; GFX11-NEXT:    s_mov_b32 s1, s0
2312; GFX11-NEXT:    s_mov_b32 s2, s0
2313; GFX11-NEXT:    s_mov_b32 s3, s0
2314; GFX11-NEXT:    v_mov_b32_e32 v0, s0
2315; GFX11-NEXT:    v_mov_b32_e32 v1, s1
2316; GFX11-NEXT:    v_mov_b32_e32 v2, s2
2317; GFX11-NEXT:    v_mov_b32_e32 v3, s3
2318; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2319; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2320; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2321; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2322; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2323; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2324; GFX11-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2325; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
2326; GFX11-NEXT:    s_setpc_b64 s[30:31]
2327;
2328; GFX9-PAL-LABEL: zero_init_large_offset_foo:
2329; GFX9-PAL:       ; %bb.0:
2330; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2331; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc
2332; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2333; GFX9-PAL-NEXT:    s_mov_b32 s0, 0
2334; GFX9-PAL-NEXT:    s_mov_b32 s1, s0
2335; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2336; GFX9-PAL-NEXT:    s_mov_b32 s3, s0
2337; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, s0
2338; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, s1
2339; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, s2
2340; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
2341; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2342; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2343; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2344; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2345; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2346; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2347; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2348; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2349; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2350; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
2351;
2352; GFX940-LABEL: zero_init_large_offset_foo:
2353; GFX940:       ; %bb.0:
2354; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2355; GFX940-NEXT:    scratch_load_dword v0, off, s32 offset:16 sc0 sc1
2356; GFX940-NEXT:    s_waitcnt vmcnt(0)
2357; GFX940-NEXT:    s_mov_b32 s0, 0
2358; GFX940-NEXT:    s_mov_b32 s1, s0
2359; GFX940-NEXT:    s_mov_b32 s2, s0
2360; GFX940-NEXT:    s_mov_b32 s3, s0
2361; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
2362; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
2363; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2364; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
2365; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2366; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
2367; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2368; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
2369; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
2370; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
2371; GFX940-NEXT:    s_waitcnt vmcnt(0)
2372; GFX940-NEXT:    s_setpc_b64 s[30:31]
2373;
2374; GFX1010-PAL-LABEL: zero_init_large_offset_foo:
2375; GFX1010-PAL:       ; %bb.0:
2376; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2377; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2378; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
2379; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2380; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
2381; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2382; GFX1010-PAL-NEXT:    s_mov_b32 s1, s0
2383; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2384; GFX1010-PAL-NEXT:    s_mov_b32 s3, s0
2385; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, s0
2386; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, s1
2387; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, s2
2388; GFX1010-PAL-NEXT:    v_mov_b32_e32 v3, s3
2389; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2390; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2391; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2392; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2393; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2394; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2395; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2396; GFX1010-PAL-NEXT:    s_waitcnt_depctr 0xffe3
2397; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2398; GFX1010-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2399; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2400; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
2401;
2402; GFX1030-PAL-LABEL: zero_init_large_offset_foo:
2403; GFX1030-PAL:       ; %bb.0:
2404; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2405; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2406; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
2407; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2408; GFX1030-PAL-NEXT:    s_mov_b32 s0, 0
2409; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2410; GFX1030-PAL-NEXT:    s_mov_b32 s1, s0
2411; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2412; GFX1030-PAL-NEXT:    s_mov_b32 s3, s0
2413; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, s0
2414; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, s1
2415; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, s2
2416; GFX1030-PAL-NEXT:    v_mov_b32_e32 v3, s3
2417; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
2418; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2419; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
2420; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2421; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
2422; GFX1030-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2423; GFX1030-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
2424; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2425; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
2426;
2427; GFX11-PAL-LABEL: zero_init_large_offset_foo:
2428; GFX11-PAL:       ; %bb.0:
2429; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2430; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2431; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s32 offset:16 glc dlc
2432; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2433; GFX11-PAL-NEXT:    s_mov_b32 s0, 0
2434; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2435; GFX11-PAL-NEXT:    s_mov_b32 s1, s0
2436; GFX11-PAL-NEXT:    s_mov_b32 s2, s0
2437; GFX11-PAL-NEXT:    s_mov_b32 s3, s0
2438; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, s0
2439; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, s1
2440; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, s2
2441; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, s3
2442; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo
2443; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2444; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:16
2445; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2446; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:32
2447; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010
2448; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], vcc_lo offset:48
2449; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2450; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
2451  %padding = alloca [4096 x i32], align 4, addrspace(5)
2452  %alloca = alloca [32 x i16], align 2, addrspace(5)
2453  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
2454  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2455  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
2456  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
2457  ret void
2458}
2459
2460define amdgpu_kernel void @store_load_sindex_large_offset_kernel(i32 %idx) {
2461; GFX9-LABEL: store_load_sindex_large_offset_kernel:
2462; GFX9:       ; %bb.0: ; %bb
2463; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
2464; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
2465; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2466; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
2467; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2468; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2469; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
2470; GFX9-NEXT:    s_and_b32 s0, s0, 15
2471; GFX9-NEXT:    v_mov_b32_e32 v0, 15
2472; GFX9-NEXT:    s_addk_i32 s1, 0x4004
2473; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
2474; GFX9-NEXT:    scratch_store_dword off, v0, s1
2475; GFX9-NEXT:    s_waitcnt vmcnt(0)
2476; GFX9-NEXT:    s_addk_i32 s0, 0x4004
2477; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
2478; GFX9-NEXT:    s_waitcnt vmcnt(0)
2479; GFX9-NEXT:    s_endpgm
2480;
2481; GFX10-LABEL: store_load_sindex_large_offset_kernel:
2482; GFX10:       ; %bb.0: ; %bb
2483; GFX10-NEXT:    s_add_u32 s2, s2, s5
2484; GFX10-NEXT:    s_addc_u32 s3, s3, 0
2485; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2486; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2487; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
2488; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2489; GFX10-NEXT:    s_waitcnt vmcnt(0)
2490; GFX10-NEXT:    v_mov_b32_e32 v0, 15
2491; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
2492; GFX10-NEXT:    s_and_b32 s1, s0, 15
2493; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
2494; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
2495; GFX10-NEXT:    s_addk_i32 s0, 0x4004
2496; GFX10-NEXT:    s_addk_i32 s1, 0x4004
2497; GFX10-NEXT:    scratch_store_dword off, v0, s0
2498; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2499; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2500; GFX10-NEXT:    s_waitcnt vmcnt(0)
2501; GFX10-NEXT:    s_endpgm
2502;
2503; GFX11-LABEL: store_load_sindex_large_offset_kernel:
2504; GFX11:       ; %bb.0: ; %bb
2505; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x24
2506; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2507; GFX11-NEXT:    s_waitcnt vmcnt(0)
2508; GFX11-NEXT:    v_mov_b32_e32 v0, 15
2509; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
2510; GFX11-NEXT:    s_and_b32 s1, s0, 15
2511; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
2512; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
2513; GFX11-NEXT:    s_addk_i32 s0, 0x4004
2514; GFX11-NEXT:    s_addk_i32 s1, 0x4004
2515; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
2516; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
2517; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2518; GFX11-NEXT:    s_waitcnt vmcnt(0)
2519; GFX11-NEXT:    s_endpgm
2520;
2521; GFX9-PAL-LABEL: store_load_sindex_large_offset_kernel:
2522; GFX9-PAL:       ; %bb.0: ; %bb
2523; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
2524; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
2525; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2526; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2527; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2528; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2529; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2530; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
2531; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
2532; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2533; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2534; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
2535; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
2536; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
2537; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x4004
2538; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2539; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
2540; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2541; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
2542; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
2543; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2544; GFX9-PAL-NEXT:    s_endpgm
2545;
2546; GFX940-LABEL: store_load_sindex_large_offset_kernel:
2547; GFX940:       ; %bb.0: ; %bb
2548; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
2549; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
2550; GFX940-NEXT:    s_waitcnt vmcnt(0)
2551; GFX940-NEXT:    v_mov_b32_e32 v0, 15
2552; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
2553; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
2554; GFX940-NEXT:    s_and_b32 s0, s0, 15
2555; GFX940-NEXT:    s_addk_i32 s1, 0x4004
2556; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
2557; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
2558; GFX940-NEXT:    s_waitcnt vmcnt(0)
2559; GFX940-NEXT:    s_addk_i32 s0, 0x4004
2560; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
2561; GFX940-NEXT:    s_waitcnt vmcnt(0)
2562; GFX940-NEXT:    s_endpgm
2563;
2564; GFX1010-PAL-LABEL: store_load_sindex_large_offset_kernel:
2565; GFX1010-PAL:       ; %bb.0: ; %bb
2566; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
2567; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
2568; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2569; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2570; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2571; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3
2572; GFX1010-PAL-NEXT:    s_addc_u32 s5, s5, 0
2573; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
2574; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
2575; GFX1010-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2576; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2577; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
2578; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2579; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
2580; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2581; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
2582; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2583; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2584; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x4004
2585; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x4004
2586; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
2587; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2588; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2589; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2590; GFX1010-PAL-NEXT:    s_endpgm
2591;
2592; GFX1030-PAL-LABEL: store_load_sindex_large_offset_kernel:
2593; GFX1030-PAL:       ; %bb.0: ; %bb
2594; GFX1030-PAL-NEXT:    s_getpc_b64 s[4:5]
2595; GFX1030-PAL-NEXT:    s_mov_b32 s4, s0
2596; GFX1030-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
2597; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2598; GFX1030-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
2599; GFX1030-PAL-NEXT:    s_add_u32 s4, s4, s3
2600; GFX1030-PAL-NEXT:    s_addc_u32 s5, s5, 0
2601; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
2602; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
2603; GFX1030-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
2604; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2605; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2606; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
2607; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2608; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
2609; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2610; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2611; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x4004
2612; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x4004
2613; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
2614; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2615; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2616; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2617; GFX1030-PAL-NEXT:    s_endpgm
2618;
2619; GFX11-PAL-LABEL: store_load_sindex_large_offset_kernel:
2620; GFX11-PAL:       ; %bb.0: ; %bb
2621; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
2622; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2623; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2624; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
2625; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2626; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
2627; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2628; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2629; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x4004
2630; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x4004
2631; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
2632; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2633; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2634; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2635; GFX11-PAL-NEXT:    s_endpgm
2636bb:
2637  %padding = alloca [4096 x i32], align 4, addrspace(5)
2638  %i = alloca [32 x float], align 4, addrspace(5)
2639  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
2640  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2641  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
2642  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
2643  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
2644  store volatile i32 15, i32 addrspace(5)* %i8, align 4
2645  %i9 = and i32 %idx, 15
2646  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
2647  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
2648  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
2649  ret void
2650}
2651
2652define amdgpu_ps void @store_load_sindex_large_offset_foo(i32 inreg %idx) {
2653; GFX9-LABEL: store_load_sindex_large_offset_foo:
2654; GFX9:       ; %bb.0: ; %bb
2655; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
2656; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
2657; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
2658; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2659; GFX9-NEXT:    s_waitcnt vmcnt(0)
2660; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
2661; GFX9-NEXT:    s_addk_i32 s0, 0x4004
2662; GFX9-NEXT:    v_mov_b32_e32 v0, 15
2663; GFX9-NEXT:    scratch_store_dword off, v0, s0
2664; GFX9-NEXT:    s_waitcnt vmcnt(0)
2665; GFX9-NEXT:    s_and_b32 s0, s2, 15
2666; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
2667; GFX9-NEXT:    s_addk_i32 s0, 0x4004
2668; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
2669; GFX9-NEXT:    s_waitcnt vmcnt(0)
2670; GFX9-NEXT:    s_endpgm
2671;
2672; GFX10-LABEL: store_load_sindex_large_offset_foo:
2673; GFX10:       ; %bb.0: ; %bb
2674; GFX10-NEXT:    s_add_u32 s0, s0, s3
2675; GFX10-NEXT:    s_addc_u32 s1, s1, 0
2676; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
2677; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
2678; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2679; GFX10-NEXT:    s_waitcnt vmcnt(0)
2680; GFX10-NEXT:    v_mov_b32_e32 v0, 15
2681; GFX10-NEXT:    s_and_b32 s0, s2, 15
2682; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
2683; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
2684; GFX10-NEXT:    s_addk_i32 s1, 0x4004
2685; GFX10-NEXT:    s_addk_i32 s0, 0x4004
2686; GFX10-NEXT:    scratch_store_dword off, v0, s1
2687; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2688; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
2689; GFX10-NEXT:    s_waitcnt vmcnt(0)
2690; GFX10-NEXT:    s_endpgm
2691;
2692; GFX11-LABEL: store_load_sindex_large_offset_foo:
2693; GFX11:       ; %bb.0: ; %bb
2694; GFX11-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2695; GFX11-NEXT:    s_waitcnt vmcnt(0)
2696; GFX11-NEXT:    v_mov_b32_e32 v0, 15
2697; GFX11-NEXT:    s_and_b32 s1, s0, 15
2698; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
2699; GFX11-NEXT:    s_lshl_b32 s1, s1, 2
2700; GFX11-NEXT:    s_addk_i32 s0, 0x4004
2701; GFX11-NEXT:    s_addk_i32 s1, 0x4004
2702; GFX11-NEXT:    scratch_store_b32 off, v0, s0 dlc
2703; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
2704; GFX11-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2705; GFX11-NEXT:    s_waitcnt vmcnt(0)
2706; GFX11-NEXT:    s_endpgm
2707;
2708; GFX9-PAL-LABEL: store_load_sindex_large_offset_foo:
2709; GFX9-PAL:       ; %bb.0: ; %bb
2710; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2711; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2712; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2713; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2714; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2715; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2716; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2717; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2718; GFX9-PAL-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
2719; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2720; GFX9-PAL-NEXT:    s_lshl_b32 s1, s0, 2
2721; GFX9-PAL-NEXT:    s_and_b32 s0, s0, 15
2722; GFX9-PAL-NEXT:    s_addk_i32 s1, 0x4004
2723; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
2724; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2725; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
2726; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2727; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
2728; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
2729; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2730; GFX9-PAL-NEXT:    s_endpgm
2731;
2732; GFX940-LABEL: store_load_sindex_large_offset_foo:
2733; GFX940:       ; %bb.0: ; %bb
2734; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
2735; GFX940-NEXT:    s_waitcnt vmcnt(0)
2736; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
2737; GFX940-NEXT:    s_and_b32 s0, s0, 15
2738; GFX940-NEXT:    s_addk_i32 s1, 0x4004
2739; GFX940-NEXT:    v_mov_b32_e32 v0, 15
2740; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
2741; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
2742; GFX940-NEXT:    s_waitcnt vmcnt(0)
2743; GFX940-NEXT:    s_addk_i32 s0, 0x4004
2744; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
2745; GFX940-NEXT:    s_waitcnt vmcnt(0)
2746; GFX940-NEXT:    s_endpgm
2747;
2748; GFX1010-PAL-LABEL: store_load_sindex_large_offset_foo:
2749; GFX1010-PAL:       ; %bb.0: ; %bb
2750; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2751; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2752; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2753; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2754; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2755; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2756; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2757; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2758; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2759; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2760; GFX1010-PAL-NEXT:    s_and_b32 s1, s0, 15
2761; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4 glc dlc
2762; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2763; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 15
2764; GFX1010-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2765; GFX1010-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2766; GFX1010-PAL-NEXT:    s_addk_i32 s0, 0x4004
2767; GFX1010-PAL-NEXT:    s_addk_i32 s1, 0x4004
2768; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s0
2769; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2770; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2771; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2772; GFX1010-PAL-NEXT:    s_endpgm
2773;
2774; GFX1030-PAL-LABEL: store_load_sindex_large_offset_foo:
2775; GFX1030-PAL:       ; %bb.0: ; %bb
2776; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2777; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2778; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2779; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2780; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2781; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2782; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2783; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2784; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2785; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
2786; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2787; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 15
2788; GFX1030-PAL-NEXT:    s_and_b32 s1, s0, 15
2789; GFX1030-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2790; GFX1030-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2791; GFX1030-PAL-NEXT:    s_addk_i32 s0, 0x4004
2792; GFX1030-PAL-NEXT:    s_addk_i32 s1, 0x4004
2793; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, s0
2794; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2795; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
2796; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2797; GFX1030-PAL-NEXT:    s_endpgm
2798;
2799; GFX11-PAL-LABEL: store_load_sindex_large_offset_foo:
2800; GFX11-PAL:       ; %bb.0: ; %bb
2801; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, off offset:4 glc dlc
2802; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2803; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 15
2804; GFX11-PAL-NEXT:    s_and_b32 s1, s0, 15
2805; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 2
2806; GFX11-PAL-NEXT:    s_lshl_b32 s1, s1, 2
2807; GFX11-PAL-NEXT:    s_addk_i32 s0, 0x4004
2808; GFX11-PAL-NEXT:    s_addk_i32 s1, 0x4004
2809; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s0 dlc
2810; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2811; GFX11-PAL-NEXT:    scratch_load_b32 v0, off, s1 glc dlc
2812; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2813; GFX11-PAL-NEXT:    s_endpgm
2814bb:
2815  %padding = alloca [4096 x i32], align 4, addrspace(5)
2816  %i = alloca [32 x float], align 4, addrspace(5)
2817  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
2818  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2819  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
2820  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
2821  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
2822  store volatile i32 15, i32 addrspace(5)* %i8, align 4
2823  %i9 = and i32 %idx, 15
2824  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
2825  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
2826  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
2827  ret void
2828}
2829
2830define amdgpu_kernel void @store_load_vindex_large_offset_kernel() {
2831; GFX9-LABEL: store_load_vindex_large_offset_kernel:
2832; GFX9:       ; %bb.0: ; %bb
2833; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
2834; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
2835; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
2836; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
2837; GFX9-NEXT:    s_waitcnt vmcnt(0)
2838; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2839; GFX9-NEXT:    v_add_u32_e32 v1, 0x4004, v0
2840; GFX9-NEXT:    v_mov_b32_e32 v2, 15
2841; GFX9-NEXT:    scratch_store_dword v1, v2, off
2842; GFX9-NEXT:    s_waitcnt vmcnt(0)
2843; GFX9-NEXT:    v_sub_u32_e32 v0, 0x4004, v0
2844; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
2845; GFX9-NEXT:    s_waitcnt vmcnt(0)
2846; GFX9-NEXT:    s_endpgm
2847;
2848; GFX10-LABEL: store_load_vindex_large_offset_kernel:
2849; GFX10:       ; %bb.0: ; %bb
2850; GFX10-NEXT:    s_add_u32 s0, s0, s3
2851; GFX10-NEXT:    s_addc_u32 s1, s1, 0
2852; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
2853; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
2854; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2855; GFX10-NEXT:    v_mov_b32_e32 v2, 15
2856; GFX10-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
2857; GFX10-NEXT:    s_waitcnt vmcnt(0)
2858; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v0
2859; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 0x4004, v0
2860; GFX10-NEXT:    scratch_store_dword v1, v2, off
2861; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2862; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
2863; GFX10-NEXT:    s_waitcnt vmcnt(0)
2864; GFX10-NEXT:    s_endpgm
2865;
2866; GFX11-LABEL: store_load_vindex_large_offset_kernel:
2867; GFX11:       ; %bb.0: ; %bb
2868; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2869; GFX11-NEXT:    v_mov_b32_e32 v1, 15
2870; GFX11-NEXT:    s_movk_i32 vcc_lo, 0x4004
2871; GFX11-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
2872; GFX11-NEXT:    s_waitcnt vmcnt(0)
2873; GFX11-NEXT:    v_sub_nc_u32_e32 v2, 0x4004, v0
2874; GFX11-NEXT:    scratch_store_b32 v0, v1, vcc_lo dlc
2875; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
2876; GFX11-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
2877; GFX11-NEXT:    s_waitcnt vmcnt(0)
2878; GFX11-NEXT:    s_endpgm
2879;
2880; GFX9-PAL-LABEL: store_load_vindex_large_offset_kernel:
2881; GFX9-PAL:       ; %bb.0: ; %bb
2882; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
2883; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
2884; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2885; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
2886; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2887; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
2888; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2889; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2890; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
2891; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
2892; GFX9-PAL-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
2893; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2894; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 0x4004, v0
2895; GFX9-PAL-NEXT:    scratch_store_dword v1, v2, off
2896; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2897; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 0x4004, v0
2898; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
2899; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
2900; GFX9-PAL-NEXT:    s_endpgm
2901;
2902; GFX940-LABEL: store_load_vindex_large_offset_kernel:
2903; GFX940:       ; %bb.0: ; %bb
2904; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
2905; GFX940-NEXT:    s_waitcnt vmcnt(0)
2906; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2907; GFX940-NEXT:    v_mov_b32_e32 v1, 15
2908; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
2909; GFX940-NEXT:    scratch_store_dword v0, v1, vcc_hi sc0 sc1
2910; GFX940-NEXT:    s_waitcnt vmcnt(0)
2911; GFX940-NEXT:    v_sub_u32_e32 v0, 0x4004, v0
2912; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
2913; GFX940-NEXT:    s_waitcnt vmcnt(0)
2914; GFX940-NEXT:    s_endpgm
2915;
2916; GFX1010-PAL-LABEL: store_load_vindex_large_offset_kernel:
2917; GFX1010-PAL:       ; %bb.0: ; %bb
2918; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
2919; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
2920; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2921; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2922; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2923; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
2924; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
2925; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2926; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2927; GFX1010-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2928; GFX1010-PAL-NEXT:    v_mov_b32_e32 v2, 15
2929; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
2930; GFX1010-PAL-NEXT:    scratch_load_dword v3, off, vcc_lo offset:4 glc dlc
2931; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2932; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v0
2933; GFX1010-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x4004, v0
2934; GFX1010-PAL-NEXT:    scratch_store_dword v1, v2, off
2935; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2936; GFX1010-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
2937; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
2938; GFX1010-PAL-NEXT:    s_endpgm
2939;
2940; GFX1030-PAL-LABEL: store_load_vindex_large_offset_kernel:
2941; GFX1030-PAL:       ; %bb.0: ; %bb
2942; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
2943; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
2944; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
2945; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
2946; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
2947; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
2948; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
2949; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
2950; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
2951; GFX1030-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2952; GFX1030-PAL-NEXT:    v_mov_b32_e32 v2, 15
2953; GFX1030-PAL-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
2954; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2955; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v0
2956; GFX1030-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0x4004, v0
2957; GFX1030-PAL-NEXT:    scratch_store_dword v1, v2, off
2958; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2959; GFX1030-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
2960; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
2961; GFX1030-PAL-NEXT:    s_endpgm
2962;
2963; GFX11-PAL-LABEL: store_load_vindex_large_offset_kernel:
2964; GFX11-PAL:       ; %bb.0: ; %bb
2965; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
2966; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
2967; GFX11-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4004
2968; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
2969; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2970; GFX11-PAL-NEXT:    v_sub_nc_u32_e32 v2, 0x4004, v0
2971; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, vcc_lo dlc
2972; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
2973; GFX11-PAL-NEXT:    scratch_load_b32 v0, v2, off offset:124 glc dlc
2974; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
2975; GFX11-PAL-NEXT:    s_endpgm
2976bb:
2977  %padding = alloca [4096 x i32], align 4, addrspace(5)
2978  %i = alloca [32 x float], align 4, addrspace(5)
2979  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
2980  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
2981  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
2982  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
2983  %i3 = zext i32 %i2 to i64
2984  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
2985  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
2986  store volatile i32 15, i32 addrspace(5)* %i8, align 4
2987  %i9 = sub nsw i32 31, %i2
2988  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
2989  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
2990  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
2991  ret void
2992}
2993
2994define void @store_load_vindex_large_offset_foo(i32 %idx) {
2995; GFX9-LABEL: store_load_vindex_large_offset_foo:
2996; GFX9:       ; %bb.0: ; %bb
2997; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2998; GFX9-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
2999; GFX9-NEXT:    s_waitcnt vmcnt(0)
3000; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
3001; GFX9-NEXT:    v_mov_b32_e32 v1, vcc_hi
3002; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
3003; GFX9-NEXT:    v_mov_b32_e32 v3, 15
3004; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
3005; GFX9-NEXT:    scratch_store_dword v2, v3, off
3006; GFX9-NEXT:    s_waitcnt vmcnt(0)
3007; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3008; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
3009; GFX9-NEXT:    s_waitcnt vmcnt(0)
3010; GFX9-NEXT:    s_setpc_b64 s[30:31]
3011;
3012; GFX10-LABEL: store_load_vindex_large_offset_foo:
3013; GFX10:       ; %bb.0: ; %bb
3014; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3015; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3016; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
3017; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3018; GFX10-NEXT:    v_mov_b32_e32 v2, 15
3019; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
3020; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3021; GFX10-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
3022; GFX10-NEXT:    s_waitcnt vmcnt(0)
3023; GFX10-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
3024; GFX10-NEXT:    scratch_store_dword v0, v2, off
3025; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3026; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
3027; GFX10-NEXT:    s_waitcnt vmcnt(0)
3028; GFX10-NEXT:    s_setpc_b64 s[30:31]
3029;
3030; GFX11-LABEL: store_load_vindex_large_offset_foo:
3031; GFX11:       ; %bb.0: ; %bb
3032; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3033; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3034; GFX11-NEXT:    v_and_b32_e32 v1, 15, v0
3035; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
3036; GFX11-NEXT:    v_mov_b32_e32 v2, 15
3037; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3038; GFX11-NEXT:    scratch_load_b32 v3, off, s32 offset:4 glc dlc
3039; GFX11-NEXT:    s_waitcnt vmcnt(0)
3040; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
3041; GFX11-NEXT:    scratch_store_b32 v0, v2, vcc_lo dlc
3042; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3043; GFX11-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3044; GFX11-NEXT:    scratch_load_b32 v0, v1, vcc_lo glc dlc
3045; GFX11-NEXT:    s_waitcnt vmcnt(0)
3046; GFX11-NEXT:    s_setpc_b64 s[30:31]
3047;
3048; GFX9-PAL-LABEL: store_load_vindex_large_offset_foo:
3049; GFX9-PAL:       ; %bb.0: ; %bb
3050; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3051; GFX9-PAL-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
3052; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3053; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
3054; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, vcc_hi
3055; GFX9-PAL-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
3056; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 15
3057; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
3058; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
3059; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3060; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3061; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
3062; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3063; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3064;
3065; GFX940-LABEL: store_load_vindex_large_offset_foo:
3066; GFX940:       ; %bb.0: ; %bb
3067; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3068; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
3069; GFX940-NEXT:    s_waitcnt vmcnt(0)
3070; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
3071; GFX940-NEXT:    v_mov_b32_e32 v2, 15
3072; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
3073; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
3074; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
3075; GFX940-NEXT:    s_waitcnt vmcnt(0)
3076; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
3077; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
3078; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
3079; GFX940-NEXT:    s_waitcnt vmcnt(0)
3080; GFX940-NEXT:    s_setpc_b64 s[30:31]
3081;
3082; GFX10-PAL-LABEL: store_load_vindex_large_offset_foo:
3083; GFX10-PAL:       ; %bb.0: ; %bb
3084; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3085; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3086; GFX10-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
3087; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3088; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
3089; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, vcc_lo
3090; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3091; GFX10-PAL-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
3092; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3093; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v1, 2, vcc_lo
3094; GFX10-PAL-NEXT:    scratch_store_dword v0, v2, off
3095; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3096; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
3097; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3098; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3099;
3100; GFX11-PAL-LABEL: store_load_vindex_large_offset_foo:
3101; GFX11-PAL:       ; %bb.0: ; %bb
3102; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3103; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3104; GFX11-PAL-NEXT:    v_and_b32_e32 v1, 15, v0
3105; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
3106; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
3107; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3108; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, s32 offset:4 glc dlc
3109; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3110; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
3111; GFX11-PAL-NEXT:    scratch_store_b32 v0, v2, vcc_lo dlc
3112; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3113; GFX11-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
3114; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, vcc_lo glc dlc
3115; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3116; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3117; GCN-LABEL: store_load_vindex_large_offset_foo:
3118; GCN:       ; %bb.0: ; %bb
3119; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3120; GCN-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
3121; GCN-NEXT:    s_waitcnt vmcnt(0)
3122; GCN-NEXT:    v_mov_b32_e32 v2, 15
3123; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
3124; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
3125; GCN-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
3126; GCN-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
3127; GCN-NEXT:    s_waitcnt vmcnt(0)
3128; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
3129; GCN-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
3130; GCN-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
3131; GCN-NEXT:    s_waitcnt vmcnt(0)
3132; GCN-NEXT:    s_setpc_b64 s[30:31]
3133bb:
3134  %padding = alloca [4096 x i32], align 4, addrspace(5)
3135  %i = alloca [32 x float], align 4, addrspace(5)
3136  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
3137  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
3138  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
3139  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
3140  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
3141  store volatile i32 15, i32 addrspace(5)* %i8, align 4
3142  %i9 = and i32 %idx, 15
3143  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
3144  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
3145  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
3146  ret void
3147}
3148
3149define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
3150; GFX9-LABEL: store_load_large_imm_offset_kernel:
3151; GFX9:       ; %bb.0: ; %bb
3152; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
3153; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
3154; GFX9-NEXT:    v_mov_b32_e32 v0, 13
3155; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
3156; GFX9-NEXT:    s_movk_i32 s0, 0x3000
3157; GFX9-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
3158; GFX9-NEXT:    s_waitcnt vmcnt(0)
3159; GFX9-NEXT:    s_add_i32 s0, s0, 4
3160; GFX9-NEXT:    v_mov_b32_e32 v0, 15
3161; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3162; GFX9-NEXT:    s_waitcnt vmcnt(0)
3163; GFX9-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3164; GFX9-NEXT:    s_waitcnt vmcnt(0)
3165; GFX9-NEXT:    s_endpgm
3166;
3167; GFX10-LABEL: store_load_large_imm_offset_kernel:
3168; GFX10:       ; %bb.0: ; %bb
3169; GFX10-NEXT:    s_add_u32 s0, s0, s3
3170; GFX10-NEXT:    s_addc_u32 s1, s1, 0
3171; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
3172; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
3173; GFX10-NEXT:    v_mov_b32_e32 v0, 13
3174; GFX10-NEXT:    v_mov_b32_e32 v1, 15
3175; GFX10-NEXT:    s_movk_i32 s0, 0x3800
3176; GFX10-NEXT:    s_add_i32 s0, s0, 4
3177; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
3178; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3179; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3180; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3181; GFX10-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3182; GFX10-NEXT:    s_waitcnt vmcnt(0)
3183; GFX10-NEXT:    s_endpgm
3184;
3185; GFX11-LABEL: store_load_large_imm_offset_kernel:
3186; GFX11:       ; %bb.0: ; %bb
3187; GFX11-NEXT:    v_mov_b32_e32 v0, 13
3188; GFX11-NEXT:    v_mov_b32_e32 v1, 0x3000
3189; GFX11-NEXT:    v_mov_b32_e32 v2, 15
3190; GFX11-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
3191; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3192; GFX11-NEXT:    scratch_store_b32 v1, v2, off offset:3716 dlc
3193; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3194; GFX11-NEXT:    scratch_load_b32 v0, v1, off offset:3716 glc dlc
3195; GFX11-NEXT:    s_waitcnt vmcnt(0)
3196; GFX11-NEXT:    s_endpgm
3197;
3198; GFX9-PAL-LABEL: store_load_large_imm_offset_kernel:
3199; GFX9-PAL:       ; %bb.0: ; %bb
3200; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
3201; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
3202; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3203; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 13
3204; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
3205; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
3206; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3207; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3208; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s1
3209; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
3210; GFX9-PAL-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
3211; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3212; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
3213; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
3214; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3215; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3216; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3217; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3218; GFX9-PAL-NEXT:    s_endpgm
3219;
3220; GFX940-LABEL: store_load_large_imm_offset_kernel:
3221; GFX940:       ; %bb.0: ; %bb
3222; GFX940-NEXT:    v_mov_b32_e32 v0, 13
3223; GFX940-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
3224; GFX940-NEXT:    s_waitcnt vmcnt(0)
3225; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3000
3226; GFX940-NEXT:    v_mov_b32_e32 v1, 15
3227; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:3716 sc0 sc1
3228; GFX940-NEXT:    s_waitcnt vmcnt(0)
3229; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:3716 sc0 sc1
3230; GFX940-NEXT:    s_waitcnt vmcnt(0)
3231; GFX940-NEXT:    s_endpgm
3232;
3233; GFX1010-PAL-LABEL: store_load_large_imm_offset_kernel:
3234; GFX1010-PAL:       ; %bb.0: ; %bb
3235; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
3236; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
3237; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3238; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3239; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3240; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1
3241; GFX1010-PAL-NEXT:    s_addc_u32 s3, s3, 0
3242; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
3243; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
3244; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 13
3245; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 15
3246; GFX1010-PAL-NEXT:    s_movk_i32 s0, 0x3800
3247; GFX1010-PAL-NEXT:    s_mov_b32 vcc_lo, 0
3248; GFX1010-PAL-NEXT:    s_add_i32 s0, s0, 4
3249; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, vcc_lo offset:4
3250; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3251; GFX1010-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3252; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3253; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3254; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
3255; GFX1010-PAL-NEXT:    s_endpgm
3256;
3257; GFX1030-PAL-LABEL: store_load_large_imm_offset_kernel:
3258; GFX1030-PAL:       ; %bb.0: ; %bb
3259; GFX1030-PAL-NEXT:    s_getpc_b64 s[2:3]
3260; GFX1030-PAL-NEXT:    s_mov_b32 s2, s0
3261; GFX1030-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
3262; GFX1030-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX1030-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
3264; GFX1030-PAL-NEXT:    s_add_u32 s2, s2, s1
3265; GFX1030-PAL-NEXT:    s_addc_u32 s3, s3, 0
3266; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
3267; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
3268; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 13
3269; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 15
3270; GFX1030-PAL-NEXT:    s_movk_i32 s0, 0x3800
3271; GFX1030-PAL-NEXT:    s_add_i32 s0, s0, 4
3272; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, off offset:4
3273; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3274; GFX1030-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3275; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3276; GFX1030-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3277; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
3278; GFX1030-PAL-NEXT:    s_endpgm
3279;
3280; GFX11-PAL-LABEL: store_load_large_imm_offset_kernel:
3281; GFX11-PAL:       ; %bb.0: ; %bb
3282; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 13
3283; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 0x3000
3284; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
3285; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
3286; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3287; GFX11-PAL-NEXT:    scratch_store_b32 v1, v2, off offset:3716 dlc
3288; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3289; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, off offset:3716 glc dlc
3290; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3291; GFX11-PAL-NEXT:    s_endpgm
3292bb:
3293  %i = alloca [4096 x i32], align 4, addrspace(5)
3294  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
3295  store volatile i32 13, i32 addrspace(5)* %i1, align 4
3296  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
3297  store volatile i32 15, i32 addrspace(5)* %i7, align 4
3298  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
3299  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
3300  ret void
3301}
3302
3303define void @store_load_large_imm_offset_foo() {
3304; GFX9-LABEL: store_load_large_imm_offset_foo:
3305; GFX9:       ; %bb.0: ; %bb
3306; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3307; GFX9-NEXT:    v_mov_b32_e32 v0, 13
3308; GFX9-NEXT:    s_movk_i32 s0, 0x3000
3309; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 4
3310; GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
3311; GFX9-NEXT:    s_waitcnt vmcnt(0)
3312; GFX9-NEXT:    s_add_i32 s0, s0, vcc_hi
3313; GFX9-NEXT:    v_mov_b32_e32 v0, 15
3314; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3315; GFX9-NEXT:    s_waitcnt vmcnt(0)
3316; GFX9-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3317; GFX9-NEXT:    s_waitcnt vmcnt(0)
3318; GFX9-NEXT:    s_setpc_b64 s[30:31]
3319;
3320; GFX10-LABEL: store_load_large_imm_offset_foo:
3321; GFX10:       ; %bb.0: ; %bb
3322; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3323; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3324; GFX10-NEXT:    v_mov_b32_e32 v0, 13
3325; GFX10-NEXT:    v_mov_b32_e32 v1, 15
3326; GFX10-NEXT:    s_movk_i32 s0, 0x3800
3327; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 4
3328; GFX10-NEXT:    s_add_i32 s0, s0, vcc_lo
3329; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
3330; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3331; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3332; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3333; GFX10-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3334; GFX10-NEXT:    s_waitcnt vmcnt(0)
3335; GFX10-NEXT:    s_setpc_b64 s[30:31]
3336;
3337; GFX11-LABEL: store_load_large_imm_offset_foo:
3338; GFX11:       ; %bb.0: ; %bb
3339; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3340; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3341; GFX11-NEXT:    v_mov_b32_e32 v0, 13
3342; GFX11-NEXT:    v_mov_b32_e32 v1, 0x3000
3343; GFX11-NEXT:    v_mov_b32_e32 v2, 15
3344; GFX11-NEXT:    scratch_store_b32 off, v0, s32 offset:4 dlc
3345; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3346; GFX11-NEXT:    scratch_store_b32 v1, v2, s32 offset:3716 dlc
3347; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3348; GFX11-NEXT:    scratch_load_b32 v0, v1, s32 offset:3716 glc dlc
3349; GFX11-NEXT:    s_waitcnt vmcnt(0)
3350; GFX11-NEXT:    s_setpc_b64 s[30:31]
3351;
3352; GFX9-PAL-LABEL: store_load_large_imm_offset_foo:
3353; GFX9-PAL:       ; %bb.0: ; %bb
3354; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3355; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 13
3356; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
3357; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 4
3358; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
3359; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3360; GFX9-PAL-NEXT:    s_add_i32 s0, s0, vcc_hi
3361; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
3362; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
3363; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3364; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
3365; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3366; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3367;
3368; GFX940-LABEL: store_load_large_imm_offset_foo:
3369; GFX940:       ; %bb.0: ; %bb
3370; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3371; GFX940-NEXT:    v_mov_b32_e32 v0, 13
3372; GFX940-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
3373; GFX940-NEXT:    s_waitcnt vmcnt(0)
3374; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3000
3375; GFX940-NEXT:    v_mov_b32_e32 v1, 15
3376; GFX940-NEXT:    scratch_store_dword v0, v1, s32 offset:3716 sc0 sc1
3377; GFX940-NEXT:    s_waitcnt vmcnt(0)
3378; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:3716 sc0 sc1
3379; GFX940-NEXT:    s_waitcnt vmcnt(0)
3380; GFX940-NEXT:    s_setpc_b64 s[30:31]
3381;
3382; GFX10-PAL-LABEL: store_load_large_imm_offset_foo:
3383; GFX10-PAL:       ; %bb.0: ; %bb
3384; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3385; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3386; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 13
3387; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3388; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x3800
3389; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 4
3390; GFX10-PAL-NEXT:    s_add_i32 s0, s0, vcc_lo
3391; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
3392; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3393; GFX10-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
3394; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3395; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
3396; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3397; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3398;
3399; GFX11-PAL-LABEL: store_load_large_imm_offset_foo:
3400; GFX11-PAL:       ; %bb.0: ; %bb
3401; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3402; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3403; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 13
3404; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 0x3000
3405; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
3406; GFX11-PAL-NEXT:    scratch_store_b32 off, v0, s32 offset:4 dlc
3407; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3408; GFX11-PAL-NEXT:    scratch_store_b32 v1, v2, s32 offset:3716 dlc
3409; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3410; GFX11-PAL-NEXT:    scratch_load_b32 v0, v1, s32 offset:3716 glc dlc
3411; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3412; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3413; GCN-LABEL: store_load_large_imm_offset_foo:
3414; GCN:       ; %bb.0: ; %bb
3415; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3416; GCN-NEXT:    v_mov_b32_e32 v0, 13
3417; GCN-NEXT:    scratch_store_dword off, v0, s32 sc0 sc1
3418; GCN-NEXT:    s_waitcnt vmcnt(0)
3419; GCN-NEXT:    v_mov_b32_e32 v0, 0x3000
3420; GCN-NEXT:    v_mov_b32_e32 v1, 15
3421; GCN-NEXT:    scratch_store_dword v0, v1, s32 offset:3712 sc0 sc1
3422; GCN-NEXT:    s_waitcnt vmcnt(0)
3423; GCN-NEXT:    scratch_load_dword v0, v0, s32 offset:3712 sc0 sc1
3424; GCN-NEXT:    s_waitcnt vmcnt(0)
3425; GCN-NEXT:    s_setpc_b64 s[30:31]
3426bb:
3427  %i = alloca [4096 x i32], align 4, addrspace(5)
3428  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
3429  store volatile i32 13, i32 addrspace(5)* %i1, align 4
3430  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
3431  store volatile i32 15, i32 addrspace(5)* %i7, align 4
3432  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
3433  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
3434  ret void
3435}
3436
3437define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
3438; GFX9-LABEL: store_load_vidx_sidx_offset:
3439; GFX9:       ; %bb.0: ; %bb
3440; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
3441; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
3442; GFX9-NEXT:    v_mov_b32_e32 v1, 4
3443; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
3444; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3445; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
3446; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3447; GFX9-NEXT:    v_mov_b32_e32 v1, 15
3448; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
3449; GFX9-NEXT:    s_waitcnt vmcnt(0)
3450; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
3451; GFX9-NEXT:    s_waitcnt vmcnt(0)
3452; GFX9-NEXT:    s_endpgm
3453;
3454; GFX10-LABEL: store_load_vidx_sidx_offset:
3455; GFX10:       ; %bb.0: ; %bb
3456; GFX10-NEXT:    s_add_u32 s2, s2, s5
3457; GFX10-NEXT:    s_addc_u32 s3, s3, 0
3458; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
3459; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
3460; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
3461; GFX10-NEXT:    v_mov_b32_e32 v1, 15
3462; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
3464; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, 4
3465; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
3466; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3467; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
3468; GFX10-NEXT:    s_waitcnt vmcnt(0)
3469; GFX10-NEXT:    s_endpgm
3470;
3471; GFX11-LABEL: store_load_vidx_sidx_offset:
3472; GFX11:       ; %bb.0: ; %bb
3473; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x24
3474; GFX11-NEXT:    v_mov_b32_e32 v1, 15
3475; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
3476; GFX11-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
3477; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:1028 dlc
3478; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3479; GFX11-NEXT:    scratch_load_b32 v0, v0, off offset:1028 glc dlc
3480; GFX11-NEXT:    s_waitcnt vmcnt(0)
3481; GFX11-NEXT:    s_endpgm
3482;
3483; GFX9-PAL-LABEL: store_load_vidx_sidx_offset:
3484; GFX9-PAL:       ; %bb.0: ; %bb
3485; GFX9-PAL-NEXT:    s_getpc_b64 s[4:5]
3486; GFX9-PAL-NEXT:    s_mov_b32 s4, s0
3487; GFX9-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
3488; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 4
3489; GFX9-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
3490; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3491; GFX9-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
3492; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s4, s3
3493; GFX9-PAL-NEXT:    v_add_u32_e32 v0, s0, v0
3494; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s5, 0
3495; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
3496; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3497; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
3498; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3499; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
3500; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3501; GFX9-PAL-NEXT:    s_endpgm
3502;
3503; GFX940-LABEL: store_load_vidx_sidx_offset:
3504; GFX940:       ; %bb.0: ; %bb
3505; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
3506; GFX940-NEXT:    v_mov_b32_e32 v1, 15
3507; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
3508; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
3509; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
3510; GFX940-NEXT:    s_waitcnt vmcnt(0)
3511; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
3512; GFX940-NEXT:    s_waitcnt vmcnt(0)
3513; GFX940-NEXT:    s_endpgm
3514;
3515; GFX10-PAL-LABEL: store_load_vidx_sidx_offset:
3516; GFX10-PAL:       ; %bb.0: ; %bb
3517; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
3518; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
3519; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
3520; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3521; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
3522; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
3523; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
3524; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
3525; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
3526; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
3527; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3528; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3529; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
3530; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, 4
3531; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
3532; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3533; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
3534; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3535; GFX10-PAL-NEXT:    s_endpgm
3536;
3537; GFX11-PAL-LABEL: store_load_vidx_sidx_offset:
3538; GFX11-PAL:       ; %bb.0: ; %bb
3539; GFX11-PAL-NEXT:    s_load_b32 s0, s[0:1], 0x0
3540; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
3541; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
3542; GFX11-PAL-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
3543; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:1028 dlc
3544; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3545; GFX11-PAL-NEXT:    scratch_load_b32 v0, v0, off offset:1028 glc dlc
3546; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3547; GFX11-PAL-NEXT:    s_endpgm
3548; GCN-LABEL: store_load_vidx_sidx_offset:
3549; GCN:       ; %bb.0: ; %bb
3550; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
3551; GCN-NEXT:    v_mov_b32_e32 v1, 15
3552; GCN-NEXT:    s_waitcnt lgkmcnt(0)
3553; GCN-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
3554; GCN-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
3555; GCN-NEXT:    s_waitcnt vmcnt(0)
3556; GCN-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
3557; GCN-NEXT:    s_waitcnt vmcnt(0)
3558; GCN-NEXT:    s_endpgm
3559bb:
3560  %alloca = alloca [32 x i32], align 4, addrspace(5)
3561  %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
3562  %add1 = add nsw i32 %sidx, %vidx
3563  %add2 = add nsw i32 %add1, 256
3564  %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
3565  store volatile i32 15, i32 addrspace(5)* %gep, align 4
3566  %load = load volatile i32, i32 addrspace(5)* %gep, align 4
3567  ret void
3568}
3569
3570define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
3571; GFX9-LABEL: store_load_i64_aligned:
3572; GFX9:       ; %bb.0: ; %bb
3573; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3574; GFX9-NEXT:    v_mov_b32_e32 v1, 15
3575; GFX9-NEXT:    v_mov_b32_e32 v2, 0
3576; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3577; GFX9-NEXT:    s_waitcnt vmcnt(0)
3578; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3579; GFX9-NEXT:    s_waitcnt vmcnt(0)
3580; GFX9-NEXT:    s_setpc_b64 s[30:31]
3581;
3582; GFX10-LABEL: store_load_i64_aligned:
3583; GFX10:       ; %bb.0: ; %bb
3584; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3585; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3586; GFX10-NEXT:    v_mov_b32_e32 v1, 15
3587; GFX10-NEXT:    v_mov_b32_e32 v2, 0
3588; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3589; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3590; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3591; GFX10-NEXT:    s_waitcnt vmcnt(0)
3592; GFX10-NEXT:    s_setpc_b64 s[30:31]
3593;
3594; GFX11-LABEL: store_load_i64_aligned:
3595; GFX11:       ; %bb.0: ; %bb
3596; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3597; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3598; GFX11-NEXT:    v_mov_b32_e32 v1, 15
3599; GFX11-NEXT:    v_mov_b32_e32 v2, 0
3600; GFX11-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3601; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3602; GFX11-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3603; GFX11-NEXT:    s_waitcnt vmcnt(0)
3604; GFX11-NEXT:    s_setpc_b64 s[30:31]
3605;
3606; GFX9-PAL-LABEL: store_load_i64_aligned:
3607; GFX9-PAL:       ; %bb.0: ; %bb
3608; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3609; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3610; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
3611; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3612; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3613; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3614; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3615; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3616;
3617; GFX940-LABEL: store_load_i64_aligned:
3618; GFX940:       ; %bb.0: ; %bb
3619; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3620; GFX940-NEXT:    v_mov_b32_e32 v2, 15
3621; GFX940-NEXT:    v_mov_b32_e32 v3, 0
3622; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
3623; GFX940-NEXT:    s_waitcnt vmcnt(0)
3624; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
3625; GFX940-NEXT:    s_waitcnt vmcnt(0)
3626; GFX940-NEXT:    s_setpc_b64 s[30:31]
3627;
3628; GFX10-PAL-LABEL: store_load_i64_aligned:
3629; GFX10-PAL:       ; %bb.0: ; %bb
3630; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3631; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3632; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3633; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
3634; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3635; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3636; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3637; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3638; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3639;
3640; GFX11-PAL-LABEL: store_load_i64_aligned:
3641; GFX11-PAL:       ; %bb.0: ; %bb
3642; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3643; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3644; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
3645; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 0
3646; GFX11-PAL-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3647; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3648; GFX11-PAL-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3649; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3650; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3651; GCN-LABEL: store_load_i64_aligned:
3652; GCN:       ; %bb.0: ; %bb
3653; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3654; GCN-NEXT:    v_mov_b32_e32 v2, 15
3655; GCN-NEXT:    v_mov_b32_e32 v3, 0
3656; GCN-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
3657; GCN-NEXT:    s_waitcnt vmcnt(0)
3658; GCN-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
3659; GCN-NEXT:    s_waitcnt vmcnt(0)
3660; GCN-NEXT:    s_setpc_b64 s[30:31]
3661bb:
3662  store volatile i64 15, i64 addrspace(5)* %arg, align 8
3663  %load = load volatile i64, i64 addrspace(5)* %arg, align 8
3664  ret void
3665}
3666
3667define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
3668; GFX9-LABEL: store_load_i64_unaligned:
3669; GFX9:       ; %bb.0: ; %bb
3670; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3671; GFX9-NEXT:    v_mov_b32_e32 v1, 15
3672; GFX9-NEXT:    v_mov_b32_e32 v2, 0
3673; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3674; GFX9-NEXT:    s_waitcnt vmcnt(0)
3675; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3676; GFX9-NEXT:    s_waitcnt vmcnt(0)
3677; GFX9-NEXT:    s_setpc_b64 s[30:31]
3678;
3679; GFX10-LABEL: store_load_i64_unaligned:
3680; GFX10:       ; %bb.0: ; %bb
3681; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3682; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3683; GFX10-NEXT:    v_mov_b32_e32 v1, 15
3684; GFX10-NEXT:    v_mov_b32_e32 v2, 0
3685; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3686; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3687; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3688; GFX10-NEXT:    s_waitcnt vmcnt(0)
3689; GFX10-NEXT:    s_setpc_b64 s[30:31]
3690;
3691; GFX11-LABEL: store_load_i64_unaligned:
3692; GFX11:       ; %bb.0: ; %bb
3693; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3694; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3695; GFX11-NEXT:    v_mov_b32_e32 v1, 15
3696; GFX11-NEXT:    v_mov_b32_e32 v2, 0
3697; GFX11-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3698; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3699; GFX11-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3700; GFX11-NEXT:    s_waitcnt vmcnt(0)
3701; GFX11-NEXT:    s_setpc_b64 s[30:31]
3702;
3703; GFX9-PAL-LABEL: store_load_i64_unaligned:
3704; GFX9-PAL:       ; %bb.0: ; %bb
3705; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3706; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
3707; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
3708; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3709; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3710; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
3711; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3712; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3713;
3714; GFX940-LABEL: store_load_i64_unaligned:
3715; GFX940:       ; %bb.0: ; %bb
3716; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3717; GFX940-NEXT:    v_mov_b32_e32 v2, 15
3718; GFX940-NEXT:    v_mov_b32_e32 v3, 0
3719; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
3720; GFX940-NEXT:    s_waitcnt vmcnt(0)
3721; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
3722; GFX940-NEXT:    s_waitcnt vmcnt(0)
3723; GFX940-NEXT:    s_setpc_b64 s[30:31]
3724;
3725; GFX10-PAL-LABEL: store_load_i64_unaligned:
3726; GFX10-PAL:       ; %bb.0: ; %bb
3727; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3728; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3729; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
3730; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
3731; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
3732; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3733; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
3734; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3735; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3736;
3737; GFX11-PAL-LABEL: store_load_i64_unaligned:
3738; GFX11-PAL:       ; %bb.0: ; %bb
3739; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3740; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3741; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 15
3742; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 0
3743; GFX11-PAL-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
3744; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3745; GFX11-PAL-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
3746; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3747; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3748; GCN-LABEL: store_load_i64_unaligned:
3749; GCN:       ; %bb.0: ; %bb
3750; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3751; GCN-NEXT:    v_mov_b32_e32 v2, 15
3752; GCN-NEXT:    v_mov_b32_e32 v3, 0
3753; GCN-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
3754; GCN-NEXT:    s_waitcnt vmcnt(0)
3755; GCN-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
3756; GCN-NEXT:    s_waitcnt vmcnt(0)
3757; GCN-NEXT:    s_setpc_b64 s[30:31]
3758bb:
3759  store volatile i64 15, i64 addrspace(5)* %arg, align 1
3760  %load = load volatile i64, i64 addrspace(5)* %arg, align 1
3761  ret void
3762}
3763
3764define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
3765; GFX9-LABEL: store_load_v3i32_unaligned:
3766; GFX9:       ; %bb.0: ; %bb
3767; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3768; GFX9-NEXT:    v_mov_b32_e32 v1, 1
3769; GFX9-NEXT:    v_mov_b32_e32 v2, 2
3770; GFX9-NEXT:    v_mov_b32_e32 v3, 3
3771; GFX9-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3772; GFX9-NEXT:    s_waitcnt vmcnt(0)
3773; GFX9-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
3774; GFX9-NEXT:    s_waitcnt vmcnt(0)
3775; GFX9-NEXT:    s_setpc_b64 s[30:31]
3776;
3777; GFX10-LABEL: store_load_v3i32_unaligned:
3778; GFX10:       ; %bb.0: ; %bb
3779; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3780; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3781; GFX10-NEXT:    v_mov_b32_e32 v1, 1
3782; GFX10-NEXT:    v_mov_b32_e32 v2, 2
3783; GFX10-NEXT:    v_mov_b32_e32 v3, 3
3784; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3785; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3786; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
3787; GFX10-NEXT:    s_waitcnt vmcnt(0)
3788; GFX10-NEXT:    s_setpc_b64 s[30:31]
3789;
3790; GFX11-LABEL: store_load_v3i32_unaligned:
3791; GFX11:       ; %bb.0: ; %bb
3792; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3793; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3794; GFX11-NEXT:    v_mov_b32_e32 v1, 1
3795; GFX11-NEXT:    v_mov_b32_e32 v2, 2
3796; GFX11-NEXT:    v_mov_b32_e32 v3, 3
3797; GFX11-NEXT:    scratch_store_b96 v0, v[1:3], off dlc
3798; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3799; GFX11-NEXT:    scratch_load_b96 v[0:2], v0, off glc dlc
3800; GFX11-NEXT:    s_waitcnt vmcnt(0)
3801; GFX11-NEXT:    s_setpc_b64 s[30:31]
3802;
3803; GFX9-PAL-LABEL: store_load_v3i32_unaligned:
3804; GFX9-PAL:       ; %bb.0: ; %bb
3805; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3806; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
3807; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 2
3808; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
3809; GFX9-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3810; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3811; GFX9-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
3812; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3813; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3814;
3815; GFX940-LABEL: store_load_v3i32_unaligned:
3816; GFX940:       ; %bb.0: ; %bb
3817; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3818; GFX940-NEXT:    v_mov_b32_e32 v2, 1
3819; GFX940-NEXT:    v_mov_b32_e32 v3, 2
3820; GFX940-NEXT:    v_mov_b32_e32 v4, 3
3821; GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
3822; GFX940-NEXT:    s_waitcnt vmcnt(0)
3823; GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
3824; GFX940-NEXT:    s_waitcnt vmcnt(0)
3825; GFX940-NEXT:    s_setpc_b64 s[30:31]
3826;
3827; GFX10-PAL-LABEL: store_load_v3i32_unaligned:
3828; GFX10-PAL:       ; %bb.0: ; %bb
3829; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3830; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3831; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
3832; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
3833; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
3834; GFX10-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
3835; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3836; GFX10-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
3837; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3838; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3839;
3840; GFX11-PAL-LABEL: store_load_v3i32_unaligned:
3841; GFX11-PAL:       ; %bb.0: ; %bb
3842; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3843; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3844; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 1
3845; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 2
3846; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, 3
3847; GFX11-PAL-NEXT:    scratch_store_b96 v0, v[1:3], off dlc
3848; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3849; GFX11-PAL-NEXT:    scratch_load_b96 v[0:2], v0, off glc dlc
3850; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3851; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3852; GCN-LABEL: store_load_v3i32_unaligned:
3853; GCN:       ; %bb.0: ; %bb
3854; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3855; GCN-NEXT:    v_mov_b32_e32 v2, 1
3856; GCN-NEXT:    v_mov_b32_e32 v3, 2
3857; GCN-NEXT:    v_mov_b32_e32 v4, 3
3858; GCN-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
3859; GCN-NEXT:    s_waitcnt vmcnt(0)
3860; GCN-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
3861; GCN-NEXT:    s_waitcnt vmcnt(0)
3862; GCN-NEXT:    s_setpc_b64 s[30:31]
3863bb:
3864  store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
3865  %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
3866  ret void
3867}
3868
3869define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
3870; GFX9-LABEL: store_load_v4i32_unaligned:
3871; GFX9:       ; %bb.0: ; %bb
3872; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3873; GFX9-NEXT:    v_mov_b32_e32 v1, 1
3874; GFX9-NEXT:    v_mov_b32_e32 v2, 2
3875; GFX9-NEXT:    v_mov_b32_e32 v3, 3
3876; GFX9-NEXT:    v_mov_b32_e32 v4, 4
3877; GFX9-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3878; GFX9-NEXT:    s_waitcnt vmcnt(0)
3879; GFX9-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
3880; GFX9-NEXT:    s_waitcnt vmcnt(0)
3881; GFX9-NEXT:    s_setpc_b64 s[30:31]
3882;
3883; GFX10-LABEL: store_load_v4i32_unaligned:
3884; GFX10:       ; %bb.0: ; %bb
3885; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3886; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3887; GFX10-NEXT:    v_mov_b32_e32 v1, 1
3888; GFX10-NEXT:    v_mov_b32_e32 v2, 2
3889; GFX10-NEXT:    v_mov_b32_e32 v3, 3
3890; GFX10-NEXT:    v_mov_b32_e32 v4, 4
3891; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3892; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3893; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
3894; GFX10-NEXT:    s_waitcnt vmcnt(0)
3895; GFX10-NEXT:    s_setpc_b64 s[30:31]
3896;
3897; GFX11-LABEL: store_load_v4i32_unaligned:
3898; GFX11:       ; %bb.0: ; %bb
3899; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3900; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3901; GFX11-NEXT:    v_mov_b32_e32 v1, 1
3902; GFX11-NEXT:    v_mov_b32_e32 v2, 2
3903; GFX11-NEXT:    v_mov_b32_e32 v3, 3
3904; GFX11-NEXT:    v_mov_b32_e32 v4, 4
3905; GFX11-NEXT:    scratch_store_b128 v0, v[1:4], off dlc
3906; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
3907; GFX11-NEXT:    scratch_load_b128 v[0:3], v0, off glc dlc
3908; GFX11-NEXT:    s_waitcnt vmcnt(0)
3909; GFX11-NEXT:    s_setpc_b64 s[30:31]
3910;
3911; GFX9-PAL-LABEL: store_load_v4i32_unaligned:
3912; GFX9-PAL:       ; %bb.0: ; %bb
3913; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3914; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
3915; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 2
3916; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
3917; GFX9-PAL-NEXT:    v_mov_b32_e32 v4, 4
3918; GFX9-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3919; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3920; GFX9-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
3921; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
3922; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
3923;
3924; GFX940-LABEL: store_load_v4i32_unaligned:
3925; GFX940:       ; %bb.0: ; %bb
3926; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3927; GFX940-NEXT:    v_mov_b32_e32 v2, 1
3928; GFX940-NEXT:    v_mov_b32_e32 v3, 2
3929; GFX940-NEXT:    v_mov_b32_e32 v4, 3
3930; GFX940-NEXT:    v_mov_b32_e32 v5, 4
3931; GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
3932; GFX940-NEXT:    s_waitcnt vmcnt(0)
3933; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
3934; GFX940-NEXT:    s_waitcnt vmcnt(0)
3935; GFX940-NEXT:    s_setpc_b64 s[30:31]
3936;
3937; GFX10-PAL-LABEL: store_load_v4i32_unaligned:
3938; GFX10-PAL:       ; %bb.0: ; %bb
3939; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3940; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3941; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
3942; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
3943; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
3944; GFX10-PAL-NEXT:    v_mov_b32_e32 v4, 4
3945; GFX10-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
3946; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3947; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
3948; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
3949; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
3950;
3951; GFX11-PAL-LABEL: store_load_v4i32_unaligned:
3952; GFX11-PAL:       ; %bb.0: ; %bb
3953; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3954; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3955; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 1
3956; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 2
3957; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, 3
3958; GFX11-PAL-NEXT:    v_mov_b32_e32 v4, 4
3959; GFX11-PAL-NEXT:    scratch_store_b128 v0, v[1:4], off dlc
3960; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
3961; GFX11-PAL-NEXT:    scratch_load_b128 v[0:3], v0, off glc dlc
3962; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
3963; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
3964; GCN-LABEL: store_load_v4i32_unaligned:
3965; GCN:       ; %bb.0: ; %bb
3966; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3967; GCN-NEXT:    v_mov_b32_e32 v2, 1
3968; GCN-NEXT:    v_mov_b32_e32 v3, 2
3969; GCN-NEXT:    v_mov_b32_e32 v4, 3
3970; GCN-NEXT:    v_mov_b32_e32 v5, 4
3971; GCN-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
3972; GCN-NEXT:    s_waitcnt vmcnt(0)
3973; GCN-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
3974; GCN-NEXT:    s_waitcnt vmcnt(0)
3975; GCN-NEXT:    s_setpc_b64 s[30:31]
3976bb:
3977  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
3978  %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
3979  ret void
3980}
3981
3982define void @store_load_i32_negative_unaligned(i8 addrspace(5)* nocapture %arg) {
3983; GFX9-LABEL: store_load_i32_negative_unaligned:
3984; GFX9:       ; %bb.0: ; %bb
3985; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3986; GFX9-NEXT:    v_add_u32_e32 v0, -1, v0
3987; GFX9-NEXT:    v_mov_b32_e32 v1, 1
3988; GFX9-NEXT:    scratch_store_byte v0, v1, off
3989; GFX9-NEXT:    s_waitcnt vmcnt(0)
3990; GFX9-NEXT:    scratch_load_ubyte v0, v0, off glc
3991; GFX9-NEXT:    s_waitcnt vmcnt(0)
3992; GFX9-NEXT:    s_setpc_b64 s[30:31]
3993;
3994; GFX10-LABEL: store_load_i32_negative_unaligned:
3995; GFX10:       ; %bb.0: ; %bb
3996; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3997; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
3998; GFX10-NEXT:    v_mov_b32_e32 v1, 1
3999; GFX10-NEXT:    scratch_store_byte v0, v1, off offset:-1
4000; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
4001; GFX10-NEXT:    scratch_load_ubyte v0, v0, off offset:-1 glc dlc
4002; GFX10-NEXT:    s_waitcnt vmcnt(0)
4003; GFX10-NEXT:    s_setpc_b64 s[30:31]
4004;
4005; GFX11-LABEL: store_load_i32_negative_unaligned:
4006; GFX11:       ; %bb.0: ; %bb
4007; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4008; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
4009; GFX11-NEXT:    v_mov_b32_e32 v1, 1
4010; GFX11-NEXT:    scratch_store_b8 v0, v1, off offset:-1 dlc
4011; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
4012; GFX11-NEXT:    scratch_load_u8 v0, v0, off offset:-1 glc dlc
4013; GFX11-NEXT:    s_waitcnt vmcnt(0)
4014; GFX11-NEXT:    s_setpc_b64 s[30:31]
4015;
4016; GFX9-PAL-LABEL: store_load_i32_negative_unaligned:
4017; GFX9-PAL:       ; %bb.0: ; %bb
4018; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4019; GFX9-PAL-NEXT:    v_add_u32_e32 v0, -1, v0
4020; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
4021; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
4022; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4023; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
4024; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4025; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
4026;
4027; GFX940-LABEL: store_load_i32_negative_unaligned:
4028; GFX940:       ; %bb.0: ; %bb
4029; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4030; GFX940-NEXT:    v_add_u32_e32 v0, -1, v0
4031; GFX940-NEXT:    v_mov_b32_e32 v1, 1
4032; GFX940-NEXT:    scratch_store_byte v0, v1, off sc0 sc1
4033; GFX940-NEXT:    s_waitcnt vmcnt(0)
4034; GFX940-NEXT:    scratch_load_ubyte v0, v0, off sc0 sc1
4035; GFX940-NEXT:    s_waitcnt vmcnt(0)
4036; GFX940-NEXT:    s_setpc_b64 s[30:31]
4037;
4038; GFX1010-PAL-LABEL: store_load_i32_negative_unaligned:
4039; GFX1010-PAL:       ; %bb.0: ; %bb
4040; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4041; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4042; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, -1, v0
4043; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
4044; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off
4045; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4046; GFX1010-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc dlc
4047; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
4048; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
4049;
4050; GFX1030-PAL-LABEL: store_load_i32_negative_unaligned:
4051; GFX1030-PAL:       ; %bb.0: ; %bb
4052; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4053; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4054; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 1
4055; GFX1030-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-1
4056; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4057; GFX1030-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-1 glc dlc
4058; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
4059; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
4060;
4061; GFX11-PAL-LABEL: store_load_i32_negative_unaligned:
4062; GFX11-PAL:       ; %bb.0: ; %bb
4063; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4064; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4065; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 1
4066; GFX11-PAL-NEXT:    scratch_store_b8 v0, v1, off offset:-1 dlc
4067; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4068; GFX11-PAL-NEXT:    scratch_load_u8 v0, v0, off offset:-1 glc dlc
4069; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4070; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
4071bb:
4072  %ptr = getelementptr inbounds i8, i8 addrspace(5)* %arg, i32 -1
4073  store volatile i8 1, i8 addrspace(5)* %ptr, align 1
4074  %load = load volatile i8, i8 addrspace(5)* %ptr, align 1
4075  ret void
4076}
4077
4078define void @store_load_i32_large_negative_unaligned(i8 addrspace(5)* nocapture %arg) {
4079; GFX9-LABEL: store_load_i32_large_negative_unaligned:
4080; GFX9:       ; %bb.0: ; %bb
4081; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4082; GFX9-NEXT:    v_add_u32_e32 v0, 0xffffef7f, v0
4083; GFX9-NEXT:    v_mov_b32_e32 v1, 1
4084; GFX9-NEXT:    scratch_store_byte v0, v1, off
4085; GFX9-NEXT:    s_waitcnt vmcnt(0)
4086; GFX9-NEXT:    scratch_load_ubyte v0, v0, off glc
4087; GFX9-NEXT:    s_waitcnt vmcnt(0)
4088; GFX9-NEXT:    s_setpc_b64 s[30:31]
4089;
4090; GFX10-LABEL: store_load_i32_large_negative_unaligned:
4091; GFX10:       ; %bb.0: ; %bb
4092; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4093; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
4094; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0xfffff000, v0
4095; GFX10-NEXT:    v_mov_b32_e32 v1, 1
4096; GFX10-NEXT:    scratch_store_byte v0, v1, off offset:-129
4097; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
4098; GFX10-NEXT:    scratch_load_ubyte v0, v0, off offset:-129 glc dlc
4099; GFX10-NEXT:    s_waitcnt vmcnt(0)
4100; GFX10-NEXT:    s_setpc_b64 s[30:31]
4101;
4102; GFX11-LABEL: store_load_i32_large_negative_unaligned:
4103; GFX11:       ; %bb.0: ; %bb
4104; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4105; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
4106; GFX11-NEXT:    v_add_nc_u32_e32 v0, 0xfffff000, v0
4107; GFX11-NEXT:    v_mov_b32_e32 v1, 1
4108; GFX11-NEXT:    scratch_store_b8 v0, v1, off offset:-129 dlc
4109; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
4110; GFX11-NEXT:    scratch_load_u8 v0, v0, off offset:-129 glc dlc
4111; GFX11-NEXT:    s_waitcnt vmcnt(0)
4112; GFX11-NEXT:    s_setpc_b64 s[30:31]
4113;
4114; GFX9-PAL-LABEL: store_load_i32_large_negative_unaligned:
4115; GFX9-PAL:       ; %bb.0: ; %bb
4116; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4117; GFX9-PAL-NEXT:    v_add_u32_e32 v0, 0xffffef7f, v0
4118; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
4119; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
4120; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4121; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
4122; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4123; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
4124;
4125; GFX940-LABEL: store_load_i32_large_negative_unaligned:
4126; GFX940:       ; %bb.0: ; %bb
4127; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4128; GFX940-NEXT:    s_movk_i32 s0, 0xef7f
4129; GFX940-NEXT:    v_mov_b32_e32 v1, 1
4130; GFX940-NEXT:    scratch_store_byte v0, v1, s0 sc0 sc1
4131; GFX940-NEXT:    s_waitcnt vmcnt(0)
4132; GFX940-NEXT:    scratch_load_ubyte v0, v0, s0 sc0 sc1
4133; GFX940-NEXT:    s_waitcnt vmcnt(0)
4134; GFX940-NEXT:    s_setpc_b64 s[30:31]
4135;
4136; GFX1010-PAL-LABEL: store_load_i32_large_negative_unaligned:
4137; GFX1010-PAL:       ; %bb.0: ; %bb
4138; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4139; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4140; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xffffefff, v0
4141; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
4142; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-128
4143; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4144; GFX1010-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-128 glc dlc
4145; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
4146; GFX1010-PAL-NEXT:    s_setpc_b64 s[30:31]
4147;
4148; GFX1030-PAL-LABEL: store_load_i32_large_negative_unaligned:
4149; GFX1030-PAL:       ; %bb.0: ; %bb
4150; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4151; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4152; GFX1030-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xfffff000, v0
4153; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 1
4154; GFX1030-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-129
4155; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4156; GFX1030-PAL-NEXT:    scratch_load_ubyte v0, v0, off offset:-129 glc dlc
4157; GFX1030-PAL-NEXT:    s_waitcnt vmcnt(0)
4158; GFX1030-PAL-NEXT:    s_setpc_b64 s[30:31]
4159;
4160; GFX11-PAL-LABEL: store_load_i32_large_negative_unaligned:
4161; GFX11-PAL:       ; %bb.0: ; %bb
4162; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4163; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4164; GFX11-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xfffff000, v0
4165; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 1
4166; GFX11-PAL-NEXT:    scratch_store_b8 v0, v1, off offset:-129 dlc
4167; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4168; GFX11-PAL-NEXT:    scratch_load_u8 v0, v0, off offset:-129 glc dlc
4169; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4170; GFX11-PAL-NEXT:    s_setpc_b64 s[30:31]
4171bb:
4172  %ptr = getelementptr inbounds i8, i8 addrspace(5)* %arg, i32 -4225
4173  store volatile i8 1, i8 addrspace(5)* %ptr, align 1
4174  %load = load volatile i8, i8 addrspace(5)* %ptr, align 1
4175  ret void
4176}
4177
4178define amdgpu_ps void @large_offset() {
4179; GFX9-LABEL: large_offset:
4180; GFX9:       ; %bb.0: ; %bb
4181; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s2
4182; GFX9-NEXT:    v_mov_b32_e32 v0, 0
4183; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
4184; GFX9-NEXT:    v_mov_b32_e32 v1, v0
4185; GFX9-NEXT:    v_mov_b32_e32 v2, v0
4186; GFX9-NEXT:    v_mov_b32_e32 v3, v0
4187; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
4188; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:3024
4189; GFX9-NEXT:    s_waitcnt vmcnt(0)
4190; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
4191; GFX9-NEXT:    scratch_load_dwordx4 v[0:3], off, vcc_hi offset:3024 glc
4192; GFX9-NEXT:    s_waitcnt vmcnt(0)
4193; GFX9-NEXT:    v_mov_b32_e32 v0, 16
4194; GFX9-NEXT:    ;;#ASMSTART
4195; GFX9-NEXT:    ; use v0
4196; GFX9-NEXT:    ;;#ASMEND
4197; GFX9-NEXT:    v_mov_b32_e32 v0, 0x810
4198; GFX9-NEXT:    ;;#ASMSTART
4199; GFX9-NEXT:    ; use v0
4200; GFX9-NEXT:    ;;#ASMEND
4201; GFX9-NEXT:    s_endpgm
4202;
4203; GFX10-LABEL: large_offset:
4204; GFX10:       ; %bb.0: ; %bb
4205; GFX10-NEXT:    s_add_u32 s0, s0, s2
4206; GFX10-NEXT:    s_addc_u32 s1, s1, 0
4207; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
4208; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
4209; GFX10-NEXT:    v_mov_b32_e32 v0, 0
4210; GFX10-NEXT:    s_movk_i32 s0, 0x810
4211; GFX10-NEXT:    s_addk_i32 s0, 0x3c0
4212; GFX10-NEXT:    v_mov_b32_e32 v1, v0
4213; GFX10-NEXT:    v_mov_b32_e32 v2, v0
4214; GFX10-NEXT:    v_mov_b32_e32 v3, v0
4215; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s0
4216; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
4217; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 glc dlc
4218; GFX10-NEXT:    s_waitcnt vmcnt(0)
4219; GFX10-NEXT:    v_mov_b32_e32 v0, 16
4220; GFX10-NEXT:    v_mov_b32_e32 v1, 0x810
4221; GFX10-NEXT:    ;;#ASMSTART
4222; GFX10-NEXT:    ; use v0
4223; GFX10-NEXT:    ;;#ASMEND
4224; GFX10-NEXT:    ;;#ASMSTART
4225; GFX10-NEXT:    ; use v1
4226; GFX10-NEXT:    ;;#ASMEND
4227; GFX10-NEXT:    s_endpgm
4228;
4229; GFX11-LABEL: large_offset:
4230; GFX11:       ; %bb.0: ; %bb
4231; GFX11-NEXT:    v_mov_b32_e32 v0, 0
4232; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
4233; GFX11-NEXT:    v_mov_b32_e32 v1, v0
4234; GFX11-NEXT:    v_mov_b32_e32 v2, v0
4235; GFX11-NEXT:    v_mov_b32_e32 v3, v0
4236; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:3024 dlc
4237; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
4238; GFX11-NEXT:    scratch_load_b128 v[0:3], off, off offset:3024 glc dlc
4239; GFX11-NEXT:    s_waitcnt vmcnt(0)
4240; GFX11-NEXT:    v_mov_b32_e32 v0, 16
4241; GFX11-NEXT:    v_mov_b32_e32 v1, 0x810
4242; GFX11-NEXT:    ;;#ASMSTART
4243; GFX11-NEXT:    ; use v0
4244; GFX11-NEXT:    ;;#ASMEND
4245; GFX11-NEXT:    ;;#ASMSTART
4246; GFX11-NEXT:    ; use v1
4247; GFX11-NEXT:    ;;#ASMEND
4248; GFX11-NEXT:    s_endpgm
4249;
4250; GFX9-PAL-LABEL: large_offset:
4251; GFX9-PAL:       ; %bb.0: ; %bb
4252; GFX9-PAL-NEXT:    s_getpc_b64 s[2:3]
4253; GFX9-PAL-NEXT:    s_mov_b32 s2, s0
4254; GFX9-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
4255; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 0
4256; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, v0
4257; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, v0
4258; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, v0
4259; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
4260; GFX9-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
4261; GFX9-PAL-NEXT:    s_add_u32 flat_scratch_lo, s2, s0
4262; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
4263; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
4264; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:3024
4265; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4266; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
4267; GFX9-PAL-NEXT:    scratch_load_dwordx4 v[0:3], off, vcc_hi offset:3024 glc
4268; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
4269; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 16
4270; GFX9-PAL-NEXT:    ;;#ASMSTART
4271; GFX9-PAL-NEXT:    ; use v0
4272; GFX9-PAL-NEXT:    ;;#ASMEND
4273; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 0x810
4274; GFX9-PAL-NEXT:    ;;#ASMSTART
4275; GFX9-PAL-NEXT:    ; use v0
4276; GFX9-PAL-NEXT:    ;;#ASMEND
4277; GFX9-PAL-NEXT:    s_endpgm
4278;
4279; GFX940-LABEL: large_offset:
4280; GFX940:       ; %bb.0: ; %bb
4281; GFX940-NEXT:    v_mov_b32_e32 v0, 0
4282; GFX940-NEXT:    v_mov_b32_e32 v1, v0
4283; GFX940-NEXT:    v_mov_b32_e32 v2, v0
4284; GFX940-NEXT:    v_mov_b32_e32 v3, v0
4285; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:3024 sc0 sc1
4286; GFX940-NEXT:    s_waitcnt vmcnt(0)
4287; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], off, off offset:3024 sc0 sc1
4288; GFX940-NEXT:    s_waitcnt vmcnt(0)
4289; GFX940-NEXT:    v_mov_b32_e32 v0, 16
4290; GFX940-NEXT:    ;;#ASMSTART
4291; GFX940-NEXT:    ; use v0
4292; GFX940-NEXT:    ;;#ASMEND
4293; GFX940-NEXT:    v_mov_b32_e32 v0, 0x810
4294; GFX940-NEXT:    ;;#ASMSTART
4295; GFX940-NEXT:    ; use v0
4296; GFX940-NEXT:    ;;#ASMEND
4297; GFX940-NEXT:    s_endpgm
4298;
4299; GFX10-PAL-LABEL: large_offset:
4300; GFX10-PAL:       ; %bb.0: ; %bb
4301; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
4302; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
4303; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
4304; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
4305; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
4306; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s0
4307; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
4308; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
4309; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
4310; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 0
4311; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x810
4312; GFX10-PAL-NEXT:    s_addk_i32 s0, 0x3c0
4313; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, v0
4314; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, v0
4315; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, v0
4316; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s0
4317; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4318; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 glc dlc
4319; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
4320; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 16
4321; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x810
4322; GFX10-PAL-NEXT:    ;;#ASMSTART
4323; GFX10-PAL-NEXT:    ; use v0
4324; GFX10-PAL-NEXT:    ;;#ASMEND
4325; GFX10-PAL-NEXT:    ;;#ASMSTART
4326; GFX10-PAL-NEXT:    ; use v1
4327; GFX10-PAL-NEXT:    ;;#ASMEND
4328; GFX10-PAL-NEXT:    s_endpgm
4329;
4330; GFX11-PAL-LABEL: large_offset:
4331; GFX11-PAL:       ; %bb.0: ; %bb
4332; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 0
4333; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
4334; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, v0
4335; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, v0
4336; GFX11-PAL-NEXT:    v_mov_b32_e32 v3, v0
4337; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:3024 dlc
4338; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
4339; GFX11-PAL-NEXT:    scratch_load_b128 v[0:3], off, off offset:3024 glc dlc
4340; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
4341; GFX11-PAL-NEXT:    v_mov_b32_e32 v0, 16
4342; GFX11-PAL-NEXT:    v_mov_b32_e32 v1, 0x810
4343; GFX11-PAL-NEXT:    ;;#ASMSTART
4344; GFX11-PAL-NEXT:    ; use v0
4345; GFX11-PAL-NEXT:    ;;#ASMEND
4346; GFX11-PAL-NEXT:    ;;#ASMSTART
4347; GFX11-PAL-NEXT:    ; use v1
4348; GFX11-PAL-NEXT:    ;;#ASMEND
4349; GFX11-PAL-NEXT:    s_endpgm
4350bb:
4351  %alloca = alloca [128 x <4 x i32>], align 16, addrspace(5)
4352  %alloca2 = alloca [128 x <4 x i32>], align 16, addrspace(5)
4353  %gep = getelementptr inbounds [128 x <4 x i32>], [128 x <4 x i32>] addrspace(5)* %alloca2, i32 0, i32 60
4354  store volatile <4 x i32> zeroinitializer, <4 x i32> addrspace(5)* %gep, align 16
4355  %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %gep, align 16
4356  call void asm sideeffect "; use $0", "s"([128 x <4 x i32>] addrspace(5)* %alloca) #0
4357  call void asm sideeffect "; use $0", "s"([128 x <4 x i32>] addrspace(5)* %alloca2) #0
4358  ret void
4359}
4360
4361declare void @llvm.memset.p5i8.i64(i8 addrspace(5)* nocapture writeonly, i8, i64, i1 immarg)
4362declare i32 @llvm.amdgcn.workitem.id.x()
4363