1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1030 -mattr=-promote-alloca -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
4
5define amdgpu_kernel void @zero_init_kernel() {
6; GFX9-LABEL: zero_init_kernel:
7; GFX9:       ; %bb.0:
8; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
9; GFX9-NEXT:    s_mov_b32 s0, 0
10; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
11; GFX9-NEXT:    s_mov_b32 s1, s0
12; GFX9-NEXT:    s_mov_b32 s2, s0
13; GFX9-NEXT:    s_mov_b32 s3, s0
14; GFX9-NEXT:    v_mov_b32_e32 v0, s0
15; GFX9-NEXT:    v_mov_b32_e32 v1, s1
16; GFX9-NEXT:    v_mov_b32_e32 v2, s2
17; GFX9-NEXT:    v_mov_b32_e32 v3, s3
18; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
19; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:64
20; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
21; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
22; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
23; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
24; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
25; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
26; GFX9-NEXT:    s_endpgm
27;
28; GFX10-LABEL: zero_init_kernel:
29; GFX10:       ; %bb.0:
30; GFX10-NEXT:    s_add_u32 s0, s0, s3
31; GFX10-NEXT:    s_addc_u32 s1, s1, 0
32; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
33; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
34; GFX10-NEXT:    s_mov_b32 s0, 0
35; GFX10-NEXT:    ; implicit-def: $vcc_hi
36; GFX10-NEXT:    s_mov_b32 s1, s0
37; GFX10-NEXT:    s_mov_b32 s2, s0
38; GFX10-NEXT:    s_mov_b32 s3, s0
39; GFX10-NEXT:    v_mov_b32_e32 v0, s0
40; GFX10-NEXT:    v_mov_b32_e32 v1, s1
41; GFX10-NEXT:    v_mov_b32_e32 v2, s2
42; GFX10-NEXT:    v_mov_b32_e32 v3, s3
43; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
44; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
45; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
46; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
47; GFX10-NEXT:    s_endpgm
48  %alloca = alloca [32 x i16], align 2, addrspace(5)
49  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
50  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
51  ret void
52}
53
54define void @zero_init_foo() {
55; GFX9-LABEL: zero_init_foo:
56; GFX9:       ; %bb.0:
57; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
58; GFX9-NEXT:    s_mov_b32 s0, 0
59; GFX9-NEXT:    s_mov_b32 s1, s0
60; GFX9-NEXT:    s_mov_b32 s2, s0
61; GFX9-NEXT:    s_mov_b32 s3, s0
62; GFX9-NEXT:    v_mov_b32_e32 v0, s0
63; GFX9-NEXT:    v_mov_b32_e32 v1, s1
64; GFX9-NEXT:    v_mov_b32_e32 v2, s2
65; GFX9-NEXT:    v_mov_b32_e32 v3, s3
66; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
67; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
68; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
69; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
70; GFX9-NEXT:    s_waitcnt vmcnt(0)
71; GFX9-NEXT:    s_setpc_b64 s[30:31]
72;
73; GFX10-LABEL: zero_init_foo:
74; GFX10:       ; %bb.0:
75; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
76; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
77; GFX10-NEXT:    s_mov_b32 s0, 0
78; GFX10-NEXT:    ; implicit-def: $vcc_hi
79; GFX10-NEXT:    s_mov_b32 s1, s0
80; GFX10-NEXT:    s_mov_b32 s2, s0
81; GFX10-NEXT:    s_mov_b32 s3, s0
82; GFX10-NEXT:    v_mov_b32_e32 v0, s0
83; GFX10-NEXT:    v_mov_b32_e32 v1, s1
84; GFX10-NEXT:    v_mov_b32_e32 v2, s2
85; GFX10-NEXT:    v_mov_b32_e32 v3, s3
86; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
87; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
88; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
89; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
90; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
91; GFX10-NEXT:    s_setpc_b64 s[30:31]
92  %alloca = alloca [32 x i16], align 2, addrspace(5)
93  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
94  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
95  ret void
96}
97
98define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
99; GFX9-LABEL: store_load_sindex_kernel:
100; GFX9:       ; %bb.0: ; %bb
101; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
102; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
103; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
104; GFX9-NEXT:    v_mov_b32_e32 v0, 15
105; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
106; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
107; GFX9-NEXT:    s_and_b32 s0, s0, 15
108; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
109; GFX9-NEXT:    s_add_u32 s1, 4, s1
110; GFX9-NEXT:    scratch_store_dword off, v0, s1
111; GFX9-NEXT:    s_add_u32 s0, 4, s0
112; GFX9-NEXT:    scratch_load_dword v0, off, s0
113; GFX9-NEXT:    s_endpgm
114;
115; GFX10-LABEL: store_load_sindex_kernel:
116; GFX10:       ; %bb.0: ; %bb
117; GFX10-NEXT:    s_add_u32 s2, s2, s5
118; GFX10-NEXT:    s_addc_u32 s3, s3, 0
119; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
120; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
121; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
122; GFX10-NEXT:    v_mov_b32_e32 v0, 15
123; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
124; GFX10-NEXT:    s_and_b32 s1, s0, 15
125; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
126; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
127; GFX10-NEXT:    s_add_u32 s0, 4, s0
128; GFX10-NEXT:    s_add_u32 s1, 4, s1
129; GFX10-NEXT:    scratch_store_dword off, v0, s0
130; GFX10-NEXT:    scratch_load_dword v0, off, s1
131; GFX10-NEXT:    s_endpgm
132bb:
133  %i = alloca [32 x float], align 4, addrspace(5)
134  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
135  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
136  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
137  store volatile i32 15, i32 addrspace(5)* %i8, align 4
138  %i9 = and i32 %idx, 15
139  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
140  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
141  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
142  ret void
143}
144
145define amdgpu_ps void @store_load_sindex_foo(i32 inreg %idx) {
146; GFX9-LABEL: store_load_sindex_foo:
147; GFX9:       ; %bb.0: ; %bb
148; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
149; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
150; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
151; GFX9-NEXT:    s_add_u32 s0, 4, s0
152; GFX9-NEXT:    v_mov_b32_e32 v0, 15
153; GFX9-NEXT:    scratch_store_dword off, v0, s0
154; GFX9-NEXT:    s_and_b32 s0, s2, 15
155; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
156; GFX9-NEXT:    s_add_u32 s0, 4, s0
157; GFX9-NEXT:    scratch_load_dword v0, off, s0
158; GFX9-NEXT:    s_endpgm
159;
160; GFX10-LABEL: store_load_sindex_foo:
161; GFX10:       ; %bb.0: ; %bb
162; GFX10-NEXT:    s_add_u32 s0, s0, s3
163; GFX10-NEXT:    s_addc_u32 s1, s1, 0
164; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
165; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
166; GFX10-NEXT:    s_and_b32 s0, s2, 15
167; GFX10-NEXT:    v_mov_b32_e32 v0, 15
168; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
169; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
170; GFX10-NEXT:    s_add_u32 s1, 4, s1
171; GFX10-NEXT:    s_add_u32 s0, 4, s0
172; GFX10-NEXT:    scratch_store_dword off, v0, s1
173; GFX10-NEXT:    scratch_load_dword v0, off, s0
174; GFX10-NEXT:    s_endpgm
175bb:
176  %i = alloca [32 x float], align 4, addrspace(5)
177  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
178  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
179  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
180  store volatile i32 15, i32 addrspace(5)* %i8, align 4
181  %i9 = and i32 %idx, 15
182  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
183  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
184  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
185  ret void
186}
187
188define amdgpu_kernel void @store_load_vindex_kernel() {
189; GFX9-LABEL: store_load_vindex_kernel:
190; GFX9:       ; %bb.0: ; %bb
191; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
192; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
193; GFX9-NEXT:    v_mov_b32_e32 v1, 4
194; GFX9-NEXT:    v_add_u32_e32 v2, v1, v0
195; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
196; GFX9-NEXT:    v_mov_b32_e32 v3, 15
197; GFX9-NEXT:    scratch_store_dword v2, v3, off
198; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
199; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124
200; GFX9-NEXT:    s_endpgm
201;
202; GFX10-LABEL: store_load_vindex_kernel:
203; GFX10:       ; %bb.0: ; %bb
204; GFX10-NEXT:    s_add_u32 s0, s0, s3
205; GFX10-NEXT:    s_addc_u32 s1, s1, 0
206; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
207; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
208; GFX10-NEXT:    v_mov_b32_e32 v1, 4
209; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
210; GFX10-NEXT:    v_mov_b32_e32 v3, 15
211; GFX10-NEXT:    v_add_nc_u32_e32 v2, v1, v0
212; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
213; GFX10-NEXT:    scratch_store_dword v2, v3, off
214; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124
215; GFX10-NEXT:    s_endpgm
216bb:
217  %i = alloca [32 x float], align 4, addrspace(5)
218  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
219  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
220  %i3 = zext i32 %i2 to i64
221  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
222  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
223  store volatile i32 15, i32 addrspace(5)* %i8, align 4
224  %i9 = sub nsw i32 31, %i2
225  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
226  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
227  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
228  ret void
229}
230
231define void @store_load_vindex_foo(i32 %idx) {
232; GFX9-LABEL: store_load_vindex_foo:
233; GFX9:       ; %bb.0: ; %bb
234; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
235; GFX9-NEXT:    v_mov_b32_e32 v1, s32
236; GFX9-NEXT:    v_mov_b32_e32 v3, 15
237; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
238; GFX9-NEXT:    v_and_b32_e32 v0, v0, v3
239; GFX9-NEXT:    scratch_store_dword v2, v3, off
240; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
241; GFX9-NEXT:    scratch_load_dword v0, v0, off
242; GFX9-NEXT:    s_waitcnt vmcnt(0)
243; GFX9-NEXT:    s_setpc_b64 s[30:31]
244;
245; GFX10-LABEL: store_load_vindex_foo:
246; GFX10:       ; %bb.0: ; %bb
247; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
248; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
249; GFX10-NEXT:    v_mov_b32_e32 v1, 15
250; GFX10-NEXT:    v_mov_b32_e32 v2, s32
251; GFX10-NEXT:    ; implicit-def: $vcc_hi
252; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
253; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
254; GFX10-NEXT:    v_lshl_add_u32 v2, v3, 2, v2
255; GFX10-NEXT:    scratch_store_dword v0, v1, off
256; GFX10-NEXT:    scratch_load_dword v0, v2, off
257; GFX10-NEXT:    s_waitcnt vmcnt(0)
258; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
259; GFX10-NEXT:    s_setpc_b64 s[30:31]
260bb:
261  %i = alloca [32 x float], align 4, addrspace(5)
262  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
263  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
264  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
265  store volatile i32 15, i32 addrspace(5)* %i8, align 4
266  %i9 = and i32 %idx, 15
267  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
268  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
269  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
270  ret void
271}
272
273define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
274; GFX9-LABEL: private_ptr_foo:
275; GFX9:       ; %bb.0:
276; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
277; GFX9-NEXT:    v_mov_b32_e32 v1, 0x41200000
278; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:4
279; GFX9-NEXT:    s_waitcnt vmcnt(0)
280; GFX9-NEXT:    s_setpc_b64 s[30:31]
281;
282; GFX10-LABEL: private_ptr_foo:
283; GFX10:       ; %bb.0:
284; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
285; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
286; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
287; GFX10-NEXT:    ; implicit-def: $vcc_hi
288; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
289; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
290; GFX10-NEXT:    s_setpc_b64 s[30:31]
291  %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
292  store float 1.000000e+01, float addrspace(5)* %gep, align 4
293  ret void
294}
295
296define amdgpu_kernel void @zero_init_small_offset_kernel() {
297; GFX9-LABEL: zero_init_small_offset_kernel:
298; GFX9:       ; %bb.0:
299; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
300; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
301; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
302; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
303; GFX9-NEXT:    s_mov_b32 s0, 0
304; GFX9-NEXT:    s_mov_b32 s1, s0
305; GFX9-NEXT:    s_mov_b32 s2, s0
306; GFX9-NEXT:    s_mov_b32 s3, s0
307; GFX9-NEXT:    s_waitcnt vmcnt(0)
308; GFX9-NEXT:    v_mov_b32_e32 v0, s0
309; GFX9-NEXT:    v_mov_b32_e32 v1, s1
310; GFX9-NEXT:    v_mov_b32_e32 v2, s2
311; GFX9-NEXT:    v_mov_b32_e32 v3, s3
312; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
313; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:272
314; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
315; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:288
316; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
317; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:304
318; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
319; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:320
320; GFX9-NEXT:    s_endpgm
321;
322; GFX10-LABEL: zero_init_small_offset_kernel:
323; GFX10:       ; %bb.0:
324; GFX10-NEXT:    s_add_u32 s0, s0, s3
325; GFX10-NEXT:    s_addc_u32 s1, s1, 0
326; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
327; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
328; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
329; GFX10-NEXT:    s_mov_b32 s0, 0
330; GFX10-NEXT:    ; implicit-def: $vcc_hi
331; GFX10-NEXT:    s_mov_b32 s1, s0
332; GFX10-NEXT:    s_mov_b32 s2, s0
333; GFX10-NEXT:    s_mov_b32 s3, s0
334; GFX10-NEXT:    s_waitcnt vmcnt(0)
335; GFX10-NEXT:    v_mov_b32_e32 v0, s0
336; GFX10-NEXT:    v_mov_b32_e32 v1, s1
337; GFX10-NEXT:    v_mov_b32_e32 v2, s2
338; GFX10-NEXT:    v_mov_b32_e32 v3, s3
339; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
340; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
341; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
342; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
343; GFX10-NEXT:    s_endpgm
344  %padding = alloca [64 x i32], align 4, addrspace(5)
345  %alloca = alloca [32 x i16], align 2, addrspace(5)
346  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
347  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
348  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
349  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
350  ret void
351}
352
353define void @zero_init_small_offset_foo() {
354; GFX9-LABEL: zero_init_small_offset_foo:
355; GFX9:       ; %bb.0:
356; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
357; GFX9-NEXT:    scratch_load_dword v0, off, s32
358; GFX9-NEXT:    s_mov_b32 s0, 0
359; GFX9-NEXT:    s_mov_b32 s1, s0
360; GFX9-NEXT:    s_mov_b32 s2, s0
361; GFX9-NEXT:    s_mov_b32 s3, s0
362; GFX9-NEXT:    s_waitcnt vmcnt(0)
363; GFX9-NEXT:    v_mov_b32_e32 v0, s0
364; GFX9-NEXT:    v_mov_b32_e32 v1, s1
365; GFX9-NEXT:    v_mov_b32_e32 v2, s2
366; GFX9-NEXT:    v_mov_b32_e32 v3, s3
367; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
368; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
369; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
370; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
371; GFX9-NEXT:    s_waitcnt vmcnt(0)
372; GFX9-NEXT:    s_setpc_b64 s[30:31]
373;
374; GFX10-LABEL: zero_init_small_offset_foo:
375; GFX10:       ; %bb.0:
376; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
377; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
378; GFX10-NEXT:    scratch_load_dword v0, off, s32
379; GFX10-NEXT:    s_mov_b32 s0, 0
380; GFX10-NEXT:    ; implicit-def: $vcc_hi
381; GFX10-NEXT:    s_mov_b32 s1, s0
382; GFX10-NEXT:    s_mov_b32 s2, s0
383; GFX10-NEXT:    s_mov_b32 s3, s0
384; GFX10-NEXT:    s_waitcnt vmcnt(0)
385; GFX10-NEXT:    v_mov_b32_e32 v0, s0
386; GFX10-NEXT:    v_mov_b32_e32 v1, s1
387; GFX10-NEXT:    v_mov_b32_e32 v2, s2
388; GFX10-NEXT:    v_mov_b32_e32 v3, s3
389; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
390; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
391; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
392; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
393; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
394; GFX10-NEXT:    s_setpc_b64 s[30:31]
395  %padding = alloca [64 x i32], align 4, addrspace(5)
396  %alloca = alloca [32 x i16], align 2, addrspace(5)
397  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
398  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
399  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
400  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
401  ret void
402}
403
404define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
405; GFX9-LABEL: store_load_sindex_small_offset_kernel:
406; GFX9:       ; %bb.0: ; %bb
407; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
408; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
409; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
410; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
411; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
412; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
413; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
414; GFX9-NEXT:    s_and_b32 s0, s0, 15
415; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
416; GFX9-NEXT:    s_waitcnt vmcnt(0)
417; GFX9-NEXT:    v_mov_b32_e32 v0, 15
418; GFX9-NEXT:    s_add_u32 s1, 0x104, s1
419; GFX9-NEXT:    scratch_store_dword off, v0, s1
420; GFX9-NEXT:    s_add_u32 s0, 0x104, s0
421; GFX9-NEXT:    scratch_load_dword v0, off, s0
422; GFX9-NEXT:    s_endpgm
423;
424; GFX10-LABEL: store_load_sindex_small_offset_kernel:
425; GFX10:       ; %bb.0: ; %bb
426; GFX10-NEXT:    s_add_u32 s2, s2, s5
427; GFX10-NEXT:    s_addc_u32 s3, s3, 0
428; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
429; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
430; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
431; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
432; GFX10-NEXT:    s_waitcnt vmcnt(0)
433; GFX10-NEXT:    v_mov_b32_e32 v0, 15
434; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
435; GFX10-NEXT:    s_and_b32 s1, s0, 15
436; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
437; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
438; GFX10-NEXT:    s_add_u32 s0, 0x104, s0
439; GFX10-NEXT:    s_add_u32 s1, 0x104, s1
440; GFX10-NEXT:    scratch_store_dword off, v0, s0
441; GFX10-NEXT:    scratch_load_dword v0, off, s1
442; GFX10-NEXT:    s_endpgm
443bb:
444  %padding = alloca [64 x i32], align 4, addrspace(5)
445  %i = alloca [32 x float], align 4, addrspace(5)
446  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
447  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
448  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
449  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
450  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
451  store volatile i32 15, i32 addrspace(5)* %i8, align 4
452  %i9 = and i32 %idx, 15
453  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
454  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
455  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
456  ret void
457}
458
459define amdgpu_ps void @store_load_sindex_small_offset_foo(i32 inreg %idx) {
460; GFX9-LABEL: store_load_sindex_small_offset_foo:
461; GFX9:       ; %bb.0: ; %bb
462; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
463; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
464; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
465; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
466; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
467; GFX9-NEXT:    s_add_u32 s0, 0x104, s0
468; GFX9-NEXT:    s_waitcnt vmcnt(0)
469; GFX9-NEXT:    v_mov_b32_e32 v0, 15
470; GFX9-NEXT:    scratch_store_dword off, v0, s0
471; GFX9-NEXT:    s_and_b32 s0, s2, 15
472; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
473; GFX9-NEXT:    s_add_u32 s0, 0x104, s0
474; GFX9-NEXT:    scratch_load_dword v0, off, s0
475; GFX9-NEXT:    s_endpgm
476;
477; GFX10-LABEL: store_load_sindex_small_offset_foo:
478; GFX10:       ; %bb.0: ; %bb
479; GFX10-NEXT:    s_add_u32 s0, s0, s3
480; GFX10-NEXT:    s_addc_u32 s1, s1, 0
481; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
482; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
483; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
484; GFX10-NEXT:    s_and_b32 s0, s2, 15
485; GFX10-NEXT:    s_waitcnt vmcnt(0)
486; GFX10-NEXT:    v_mov_b32_e32 v0, 15
487; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
488; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
489; GFX10-NEXT:    s_add_u32 s1, 0x104, s1
490; GFX10-NEXT:    s_add_u32 s0, 0x104, s0
491; GFX10-NEXT:    scratch_store_dword off, v0, s1
492; GFX10-NEXT:    scratch_load_dword v0, off, s0
493; GFX10-NEXT:    s_endpgm
494bb:
495  %padding = alloca [64 x i32], align 4, addrspace(5)
496  %i = alloca [32 x float], align 4, addrspace(5)
497  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
498  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
499  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
500  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
501  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
502  store volatile i32 15, i32 addrspace(5)* %i8, align 4
503  %i9 = and i32 %idx, 15
504  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
505  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
506  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
507  ret void
508}
509
510define amdgpu_kernel void @store_load_vindex_small_offset_kernel() {
511; GFX9-LABEL: store_load_vindex_small_offset_kernel:
512; GFX9:       ; %bb.0: ; %bb
513; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
514; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
515; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
516; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4
517; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
518; GFX9-NEXT:    s_waitcnt vmcnt(0)
519; GFX9-NEXT:    v_mov_b32_e32 v1, 0x104
520; GFX9-NEXT:    v_add_u32_e32 v2, v1, v0
521; GFX9-NEXT:    v_mov_b32_e32 v3, 15
522; GFX9-NEXT:    scratch_store_dword v2, v3, off
523; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
524; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124
525; GFX9-NEXT:    s_endpgm
526;
527; GFX10-LABEL: store_load_vindex_small_offset_kernel:
528; GFX10:       ; %bb.0: ; %bb
529; GFX10-NEXT:    s_add_u32 s0, s0, s3
530; GFX10-NEXT:    s_addc_u32 s1, s1, 0
531; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
532; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
533; GFX10-NEXT:    v_mov_b32_e32 v1, 0x104
534; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
535; GFX10-NEXT:    v_mov_b32_e32 v3, 15
536; GFX10-NEXT:    v_add_nc_u32_e32 v2, v1, v0
537; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
538; GFX10-NEXT:    scratch_load_dword v1, off, off offset:4
539; GFX10-NEXT:    scratch_store_dword v2, v3, off
540; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124
541; GFX10-NEXT:    s_endpgm
542bb:
543  %padding = alloca [64 x i32], align 4, addrspace(5)
544  %i = alloca [32 x float], align 4, addrspace(5)
545  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
546  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
547  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
548  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
549  %i3 = zext i32 %i2 to i64
550  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
551  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
552  store volatile i32 15, i32 addrspace(5)* %i8, align 4
553  %i9 = sub nsw i32 31, %i2
554  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
555  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
556  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
557  ret void
558}
559
560define void @store_load_vindex_small_offset_foo(i32 %idx) {
561; GFX9-LABEL: store_load_vindex_small_offset_foo:
562; GFX9:       ; %bb.0: ; %bb
563; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
564; GFX9-NEXT:    scratch_load_dword v1, off, s32
565; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x100
566; GFX9-NEXT:    s_waitcnt vmcnt(0)
567; GFX9-NEXT:    v_mov_b32_e32 v1, vcc_hi
568; GFX9-NEXT:    v_mov_b32_e32 v3, 15
569; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
570; GFX9-NEXT:    v_and_b32_e32 v0, v0, v3
571; GFX9-NEXT:    scratch_store_dword v2, v3, off
572; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
573; GFX9-NEXT:    scratch_load_dword v0, v0, off
574; GFX9-NEXT:    s_waitcnt vmcnt(0)
575; GFX9-NEXT:    s_setpc_b64 s[30:31]
576;
577; GFX10-LABEL: store_load_vindex_small_offset_foo:
578; GFX10:       ; %bb.0: ; %bb
579; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
580; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
581; GFX10-NEXT:    v_mov_b32_e32 v1, 15
582; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x100
583; GFX10-NEXT:    ; implicit-def: $vcc_hi
584; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
585; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
586; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
587; GFX10-NEXT:    v_lshl_add_u32 v2, v3, 2, v2
588; GFX10-NEXT:    scratch_load_dword v3, off, s32
589; GFX10-NEXT:    scratch_store_dword v0, v1, off
590; GFX10-NEXT:    scratch_load_dword v0, v2, off
591; GFX10-NEXT:    s_waitcnt vmcnt(0)
592; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
593; GFX10-NEXT:    s_setpc_b64 s[30:31]
594bb:
595  %padding = alloca [64 x i32], align 4, addrspace(5)
596  %i = alloca [32 x float], align 4, addrspace(5)
597  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
598  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
599  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
600  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
601  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
602  store volatile i32 15, i32 addrspace(5)* %i8, align 4
603  %i9 = and i32 %idx, 15
604  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
605  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
606  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
607  ret void
608}
609
610define amdgpu_kernel void @zero_init_large_offset_kernel() {
611; GFX9-LABEL: zero_init_large_offset_kernel:
612; GFX9:       ; %bb.0:
613; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
614; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
615; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
616; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
617; GFX9-NEXT:    s_mov_b32 s0, 0
618; GFX9-NEXT:    s_mov_b32 s1, s0
619; GFX9-NEXT:    s_mov_b32 s2, s0
620; GFX9-NEXT:    s_mov_b32 s3, s0
621; GFX9-NEXT:    s_waitcnt vmcnt(0)
622; GFX9-NEXT:    v_mov_b32_e32 v0, s0
623; GFX9-NEXT:    v_mov_b32_e32 v1, s1
624; GFX9-NEXT:    v_mov_b32_e32 v2, s2
625; GFX9-NEXT:    v_mov_b32_e32 v3, s3
626; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
627; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
628; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
629; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
630; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
631; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
632; GFX9-NEXT:    s_movk_i32 vcc_hi, 0x4010
633; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
634; GFX9-NEXT:    s_endpgm
635;
636; GFX10-LABEL: zero_init_large_offset_kernel:
637; GFX10:       ; %bb.0:
638; GFX10-NEXT:    s_add_u32 s0, s0, s3
639; GFX10-NEXT:    s_addc_u32 s1, s1, 0
640; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
641; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
642; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
643; GFX10-NEXT:    s_mov_b32 s0, 0
644; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
645; GFX10-NEXT:    s_mov_b32 s1, s0
646; GFX10-NEXT:    s_mov_b32 s2, s0
647; GFX10-NEXT:    s_mov_b32 s3, s0
648; GFX10-NEXT:    s_waitcnt vmcnt(0)
649; GFX10-NEXT:    v_mov_b32_e32 v0, s0
650; GFX10-NEXT:    v_mov_b32_e32 v1, s1
651; GFX10-NEXT:    v_mov_b32_e32 v2, s2
652; GFX10-NEXT:    v_mov_b32_e32 v3, s3
653; GFX10-NEXT:    ; implicit-def: $vcc_hi
654; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
655; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
656; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
657; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
658; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
659; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
660; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
661; GFX10-NEXT:    s_endpgm
662  %padding = alloca [4096 x i32], align 4, addrspace(5)
663  %alloca = alloca [32 x i16], align 2, addrspace(5)
664  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
665  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
666  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
667  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
668  ret void
669}
670
671define void @zero_init_large_offset_foo() {
672; GFX9-LABEL: zero_init_large_offset_foo:
673; GFX9:       ; %bb.0:
674; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
675; GFX9-NEXT:    scratch_load_dword v0, off, s32
676; GFX9-NEXT:    s_mov_b32 s0, 0
677; GFX9-NEXT:    s_mov_b32 s1, s0
678; GFX9-NEXT:    s_mov_b32 s2, s0
679; GFX9-NEXT:    s_mov_b32 s3, s0
680; GFX9-NEXT:    s_waitcnt vmcnt(0)
681; GFX9-NEXT:    v_mov_b32_e32 v0, s0
682; GFX9-NEXT:    v_mov_b32_e32 v1, s1
683; GFX9-NEXT:    v_mov_b32_e32 v2, s2
684; GFX9-NEXT:    v_mov_b32_e32 v3, s3
685; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
686; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
687; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
688; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
689; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
690; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
691; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
692; GFX9-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
693; GFX9-NEXT:    s_waitcnt vmcnt(0)
694; GFX9-NEXT:    s_setpc_b64 s[30:31]
695;
696; GFX10-LABEL: zero_init_large_offset_foo:
697; GFX10:       ; %bb.0:
698; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
699; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
700; GFX10-NEXT:    scratch_load_dword v0, off, s32
701; GFX10-NEXT:    s_mov_b32 s0, 0
702; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
703; GFX10-NEXT:    s_mov_b32 s1, s0
704; GFX10-NEXT:    s_mov_b32 s2, s0
705; GFX10-NEXT:    s_mov_b32 s3, s0
706; GFX10-NEXT:    s_waitcnt vmcnt(0)
707; GFX10-NEXT:    v_mov_b32_e32 v0, s0
708; GFX10-NEXT:    v_mov_b32_e32 v1, s1
709; GFX10-NEXT:    v_mov_b32_e32 v2, s2
710; GFX10-NEXT:    v_mov_b32_e32 v3, s3
711; GFX10-NEXT:    ; implicit-def: $vcc_hi
712; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
713; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
714; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
715; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
716; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:32
717; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
718; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:48
719; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
720; GFX10-NEXT:    s_setpc_b64 s[30:31]
721  %padding = alloca [4096 x i32], align 4, addrspace(5)
722  %alloca = alloca [32 x i16], align 2, addrspace(5)
723  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
724  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
725  %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
726  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
727  ret void
728}
729
730define amdgpu_kernel void @store_load_sindex_large_offset_kernel(i32 %idx) {
731; GFX9-LABEL: store_load_sindex_large_offset_kernel:
732; GFX9:       ; %bb.0: ; %bb
733; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
734; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
735; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
736; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
737; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
738; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
740; GFX9-NEXT:    s_and_b32 s0, s0, 15
741; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
742; GFX9-NEXT:    s_waitcnt vmcnt(0)
743; GFX9-NEXT:    v_mov_b32_e32 v0, 15
744; GFX9-NEXT:    s_add_u32 s1, 0x4004, s1
745; GFX9-NEXT:    scratch_store_dword off, v0, s1
746; GFX9-NEXT:    s_add_u32 s0, 0x4004, s0
747; GFX9-NEXT:    scratch_load_dword v0, off, s0
748; GFX9-NEXT:    s_endpgm
749;
750; GFX10-LABEL: store_load_sindex_large_offset_kernel:
751; GFX10:       ; %bb.0: ; %bb
752; GFX10-NEXT:    s_add_u32 s2, s2, s5
753; GFX10-NEXT:    s_addc_u32 s3, s3, 0
754; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
755; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
756; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
757; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
758; GFX10-NEXT:    s_waitcnt vmcnt(0)
759; GFX10-NEXT:    v_mov_b32_e32 v0, 15
760; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX10-NEXT:    s_and_b32 s1, s0, 15
762; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
763; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
764; GFX10-NEXT:    s_add_u32 s0, 0x4004, s0
765; GFX10-NEXT:    s_add_u32 s1, 0x4004, s1
766; GFX10-NEXT:    scratch_store_dword off, v0, s0
767; GFX10-NEXT:    scratch_load_dword v0, off, s1
768; GFX10-NEXT:    s_endpgm
769bb:
770  %padding = alloca [4096 x i32], align 4, addrspace(5)
771  %i = alloca [32 x float], align 4, addrspace(5)
772  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
773  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
774  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
775  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
776  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
777  store volatile i32 15, i32 addrspace(5)* %i8, align 4
778  %i9 = and i32 %idx, 15
779  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
780  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
781  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
782  ret void
783}
784
785define amdgpu_ps void @store_load_sindex_large_offset_foo(i32 inreg %idx) {
786; GFX9-LABEL: store_load_sindex_large_offset_foo:
787; GFX9:       ; %bb.0: ; %bb
788; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
789; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
790; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
791; GFX9-NEXT:    s_lshl_b32 s0, s2, 2
792; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
793; GFX9-NEXT:    s_add_u32 s0, 0x4004, s0
794; GFX9-NEXT:    s_waitcnt vmcnt(0)
795; GFX9-NEXT:    v_mov_b32_e32 v0, 15
796; GFX9-NEXT:    scratch_store_dword off, v0, s0
797; GFX9-NEXT:    s_and_b32 s0, s2, 15
798; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
799; GFX9-NEXT:    s_add_u32 s0, 0x4004, s0
800; GFX9-NEXT:    scratch_load_dword v0, off, s0
801; GFX9-NEXT:    s_endpgm
802;
803; GFX10-LABEL: store_load_sindex_large_offset_foo:
804; GFX10:       ; %bb.0: ; %bb
805; GFX10-NEXT:    s_add_u32 s0, s0, s3
806; GFX10-NEXT:    s_addc_u32 s1, s1, 0
807; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
808; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
809; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
810; GFX10-NEXT:    s_and_b32 s0, s2, 15
811; GFX10-NEXT:    s_waitcnt vmcnt(0)
812; GFX10-NEXT:    v_mov_b32_e32 v0, 15
813; GFX10-NEXT:    s_lshl_b32 s1, s2, 2
814; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
815; GFX10-NEXT:    s_add_u32 s1, 0x4004, s1
816; GFX10-NEXT:    s_add_u32 s0, 0x4004, s0
817; GFX10-NEXT:    scratch_store_dword off, v0, s1
818; GFX10-NEXT:    scratch_load_dword v0, off, s0
819; GFX10-NEXT:    s_endpgm
820bb:
821  %padding = alloca [4096 x i32], align 4, addrspace(5)
822  %i = alloca [32 x float], align 4, addrspace(5)
823  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
824  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
825  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
826  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
827  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
828  store volatile i32 15, i32 addrspace(5)* %i8, align 4
829  %i9 = and i32 %idx, 15
830  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
831  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
832  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
833  ret void
834}
835
836define amdgpu_kernel void @store_load_vindex_large_offset_kernel() {
837; GFX9-LABEL: store_load_vindex_large_offset_kernel:
838; GFX9:       ; %bb.0: ; %bb
839; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
840; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
841; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
842; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4
843; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
844; GFX9-NEXT:    s_waitcnt vmcnt(0)
845; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4004
846; GFX9-NEXT:    v_add_u32_e32 v2, v1, v0
847; GFX9-NEXT:    v_mov_b32_e32 v3, 15
848; GFX9-NEXT:    scratch_store_dword v2, v3, off
849; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
850; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124
851; GFX9-NEXT:    s_endpgm
852;
853; GFX10-LABEL: store_load_vindex_large_offset_kernel:
854; GFX10:       ; %bb.0: ; %bb
855; GFX10-NEXT:    s_add_u32 s0, s0, s3
856; GFX10-NEXT:    s_addc_u32 s1, s1, 0
857; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
858; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
859; GFX10-NEXT:    v_mov_b32_e32 v1, 0x4004
860; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
861; GFX10-NEXT:    v_mov_b32_e32 v3, 15
862; GFX10-NEXT:    v_add_nc_u32_e32 v2, v1, v0
863; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
864; GFX10-NEXT:    scratch_load_dword v1, off, off offset:4
865; GFX10-NEXT:    scratch_store_dword v2, v3, off
866; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:124
867; GFX10-NEXT:    s_endpgm
868bb:
869  %padding = alloca [4096 x i32], align 4, addrspace(5)
870  %i = alloca [32 x float], align 4, addrspace(5)
871  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
872  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
873  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
874  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
875  %i3 = zext i32 %i2 to i64
876  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
877  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
878  store volatile i32 15, i32 addrspace(5)* %i8, align 4
879  %i9 = sub nsw i32 31, %i2
880  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
881  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
882  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
883  ret void
884}
885
886define void @store_load_vindex_large_offset_foo(i32 %idx) {
887; GFX9-LABEL: store_load_vindex_large_offset_foo:
888; GFX9:       ; %bb.0: ; %bb
889; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
890; GFX9-NEXT:    scratch_load_dword v1, off, s32
891; GFX9-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
892; GFX9-NEXT:    s_waitcnt vmcnt(0)
893; GFX9-NEXT:    v_mov_b32_e32 v1, vcc_hi
894; GFX9-NEXT:    v_mov_b32_e32 v3, 15
895; GFX9-NEXT:    v_lshl_add_u32 v2, v0, 2, v1
896; GFX9-NEXT:    v_and_b32_e32 v0, v0, v3
897; GFX9-NEXT:    scratch_store_dword v2, v3, off
898; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
899; GFX9-NEXT:    scratch_load_dword v0, v0, off
900; GFX9-NEXT:    s_waitcnt vmcnt(0)
901; GFX9-NEXT:    s_setpc_b64 s[30:31]
902;
903; GFX10-LABEL: store_load_vindex_large_offset_foo:
904; GFX10:       ; %bb.0: ; %bb
905; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
906; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
907; GFX10-NEXT:    v_mov_b32_e32 v1, 15
908; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
909; GFX10-NEXT:    ; implicit-def: $vcc_hi
910; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
911; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
912; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
913; GFX10-NEXT:    v_lshl_add_u32 v2, v3, 2, v2
914; GFX10-NEXT:    scratch_load_dword v3, off, s32
915; GFX10-NEXT:    scratch_store_dword v0, v1, off
916; GFX10-NEXT:    scratch_load_dword v0, v2, off
917; GFX10-NEXT:    s_waitcnt vmcnt(0)
918; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
919; GFX10-NEXT:    s_setpc_b64 s[30:31]
920bb:
921  %padding = alloca [4096 x i32], align 4, addrspace(5)
922  %i = alloca [32 x float], align 4, addrspace(5)
923  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
924  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
925  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
926  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
927  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
928  store volatile i32 15, i32 addrspace(5)* %i8, align 4
929  %i9 = and i32 %idx, 15
930  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
931  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
932  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
933  ret void
934}
935
936define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
937; GFX9-LABEL: store_load_large_imm_offset_kernel:
938; GFX9:       ; %bb.0: ; %bb
939; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
940; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
941; GFX9-NEXT:    s_movk_i32 s0, 0x3000
942; GFX9-NEXT:    v_mov_b32_e32 v0, 13
943; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
944; GFX9-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
945; GFX9-NEXT:    s_add_u32 s0, 4, s0
946; GFX9-NEXT:    v_mov_b32_e32 v0, 15
947; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
948; GFX9-NEXT:    scratch_load_dword v0, off, s0 offset:3712
949; GFX9-NEXT:    s_endpgm
950;
951; GFX10-LABEL: store_load_large_imm_offset_kernel:
952; GFX10:       ; %bb.0: ; %bb
953; GFX10-NEXT:    s_add_u32 s0, s0, s3
954; GFX10-NEXT:    s_addc_u32 s1, s1, 0
955; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
956; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
957; GFX10-NEXT:    v_mov_b32_e32 v0, 13
958; GFX10-NEXT:    v_mov_b32_e32 v1, 15
959; GFX10-NEXT:    s_movk_i32 s0, 0x3800
960; GFX10-NEXT:    s_add_u32 s0, 4, s0
961; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
962; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
963; GFX10-NEXT:    scratch_load_dword v0, off, s0 offset:1664
964; GFX10-NEXT:    s_endpgm
965bb:
966  %i = alloca [4096 x i32], align 4, addrspace(5)
967  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
968  store volatile i32 13, i32 addrspace(5)* %i1, align 4
969  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
970  store volatile i32 15, i32 addrspace(5)* %i7, align 4
971  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
972  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
973  ret void
974}
975
976define void @store_load_large_imm_offset_foo() {
977; GFX9-LABEL: store_load_large_imm_offset_foo:
978; GFX9:       ; %bb.0: ; %bb
979; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
980; GFX9-NEXT:    s_movk_i32 s0, 0x3000
981; GFX9-NEXT:    v_mov_b32_e32 v0, 13
982; GFX9-NEXT:    scratch_store_dword off, v0, s32
983; GFX9-NEXT:    s_add_u32 s0, s32, s0
984; GFX9-NEXT:    v_mov_b32_e32 v0, 15
985; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
986; GFX9-NEXT:    scratch_load_dword v0, off, s0 offset:3712
987; GFX9-NEXT:    s_waitcnt vmcnt(0)
988; GFX9-NEXT:    s_setpc_b64 s[30:31]
989;
990; GFX10-LABEL: store_load_large_imm_offset_foo:
991; GFX10:       ; %bb.0: ; %bb
992; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
993; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
994; GFX10-NEXT:    v_mov_b32_e32 v0, 13
995; GFX10-NEXT:    v_mov_b32_e32 v1, 15
996; GFX10-NEXT:    s_movk_i32 s0, 0x3800
997; GFX10-NEXT:    ; implicit-def: $vcc_hi
998; GFX10-NEXT:    s_add_u32 s0, s32, s0
999; GFX10-NEXT:    scratch_store_dword off, v0, s32
1000; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
1001; GFX10-NEXT:    scratch_load_dword v0, off, s0 offset:1664
1002; GFX10-NEXT:    s_waitcnt vmcnt(0)
1003; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1004; GFX10-NEXT:    s_setpc_b64 s[30:31]
1005bb:
1006  %i = alloca [4096 x i32], align 4, addrspace(5)
1007  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
1008  store volatile i32 13, i32 addrspace(5)* %i1, align 4
1009  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
1010  store volatile i32 15, i32 addrspace(5)* %i7, align 4
1011  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
1012  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
1013  ret void
1014}
1015
1016define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
1017; GFX9-LABEL: store_load_vidx_sidx_offset:
1018; GFX9:       ; %bb.0: ; %bb
1019; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
1020; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
1021; GFX9-NEXT:    v_mov_b32_e32 v1, 4
1022; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
1023; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1024; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
1025; GFX9-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
1026; GFX9-NEXT:    v_mov_b32_e32 v1, 15
1027; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
1028; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024
1029; GFX9-NEXT:    s_endpgm
1030;
1031; GFX10-LABEL: store_load_vidx_sidx_offset:
1032; GFX10:       ; %bb.0: ; %bb
1033; GFX10-NEXT:    s_add_u32 s2, s2, s5
1034; GFX10-NEXT:    s_addc_u32 s3, s3, 0
1035; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
1036; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
1037; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
1038; GFX10-NEXT:    v_mov_b32_e32 v1, 15
1039; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1040; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
1041; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, 4
1042; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
1043; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024
1044; GFX10-NEXT:    s_endpgm
1045bb:
1046  %alloca = alloca [32 x i32], align 4, addrspace(5)
1047  %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
1048  %add1 = add nsw i32 %sidx, %vidx
1049  %add2 = add nsw i32 %add1, 256
1050  %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
1051  store volatile i32 15, i32 addrspace(5)* %gep, align 4
1052  %load = load volatile i32, i32 addrspace(5)* %gep, align 4
1053  ret void
1054}
1055
1056define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
1057; GFX9-LABEL: store_load_i64_aligned:
1058; GFX9:       ; %bb.0: ; %bb
1059; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1060; GFX9-NEXT:    v_mov_b32_e32 v1, 15
1061; GFX9-NEXT:    v_mov_b32_e32 v2, 0
1062; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
1063; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
1064; GFX9-NEXT:    s_waitcnt vmcnt(0)
1065; GFX9-NEXT:    s_setpc_b64 s[30:31]
1066;
1067; GFX10-LABEL: store_load_i64_aligned:
1068; GFX10:       ; %bb.0: ; %bb
1069; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1070; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1071; GFX10-NEXT:    v_mov_b32_e32 v1, 15
1072; GFX10-NEXT:    v_mov_b32_e32 v2, 0
1073; GFX10-NEXT:    ; implicit-def: $vcc_hi
1074; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
1075; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
1076; GFX10-NEXT:    s_waitcnt vmcnt(0)
1077; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1078; GFX10-NEXT:    s_setpc_b64 s[30:31]
1079bb:
1080  store volatile i64 15, i64 addrspace(5)* %arg, align 8
1081  %load = load volatile i64, i64 addrspace(5)* %arg, align 8
1082  ret void
1083}
1084
1085define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
1086; GFX9-LABEL: store_load_i64_unaligned:
1087; GFX9:       ; %bb.0: ; %bb
1088; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1089; GFX9-NEXT:    v_mov_b32_e32 v1, 15
1090; GFX9-NEXT:    v_mov_b32_e32 v2, 0
1091; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
1092; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
1093; GFX9-NEXT:    s_waitcnt vmcnt(0)
1094; GFX9-NEXT:    s_setpc_b64 s[30:31]
1095;
1096; GFX10-LABEL: store_load_i64_unaligned:
1097; GFX10:       ; %bb.0: ; %bb
1098; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1099; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1100; GFX10-NEXT:    v_mov_b32_e32 v1, 15
1101; GFX10-NEXT:    v_mov_b32_e32 v2, 0
1102; GFX10-NEXT:    ; implicit-def: $vcc_hi
1103; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
1104; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
1105; GFX10-NEXT:    s_waitcnt vmcnt(0)
1106; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1107; GFX10-NEXT:    s_setpc_b64 s[30:31]
1108bb:
1109  store volatile i64 15, i64 addrspace(5)* %arg, align 1
1110  %load = load volatile i64, i64 addrspace(5)* %arg, align 1
1111  ret void
1112}
1113
1114define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
1115; GFX9-LABEL: store_load_v3i32_unaligned:
1116; GFX9:       ; %bb.0: ; %bb
1117; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1118; GFX9-NEXT:    v_mov_b32_e32 v1, 1
1119; GFX9-NEXT:    v_mov_b32_e32 v2, 2
1120; GFX9-NEXT:    v_mov_b32_e32 v3, 3
1121; GFX9-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
1122; GFX9-NEXT:    scratch_load_dwordx3 v[0:2], v0, off
1123; GFX9-NEXT:    s_waitcnt vmcnt(0)
1124; GFX9-NEXT:    s_setpc_b64 s[30:31]
1125;
1126; GFX10-LABEL: store_load_v3i32_unaligned:
1127; GFX10:       ; %bb.0: ; %bb
1128; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1129; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1130; GFX10-NEXT:    v_mov_b32_e32 v1, 1
1131; GFX10-NEXT:    v_mov_b32_e32 v2, 2
1132; GFX10-NEXT:    v_mov_b32_e32 v3, 3
1133; GFX10-NEXT:    ; implicit-def: $vcc_hi
1134; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
1135; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off
1136; GFX10-NEXT:    s_waitcnt vmcnt(0)
1137; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1138; GFX10-NEXT:    s_setpc_b64 s[30:31]
1139bb:
1140  store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
1141  %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
1142  ret void
1143}
1144
1145define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
1146; GFX9-LABEL: store_load_v4i32_unaligned:
1147; GFX9:       ; %bb.0: ; %bb
1148; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1149; GFX9-NEXT:    v_mov_b32_e32 v1, 1
1150; GFX9-NEXT:    v_mov_b32_e32 v2, 2
1151; GFX9-NEXT:    v_mov_b32_e32 v3, 3
1152; GFX9-NEXT:    v_mov_b32_e32 v4, 4
1153; GFX9-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
1154; GFX9-NEXT:    scratch_load_dwordx4 v[0:3], v0, off
1155; GFX9-NEXT:    s_waitcnt vmcnt(0)
1156; GFX9-NEXT:    s_setpc_b64 s[30:31]
1157;
1158; GFX10-LABEL: store_load_v4i32_unaligned:
1159; GFX10:       ; %bb.0: ; %bb
1160; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1161; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1162; GFX10-NEXT:    v_mov_b32_e32 v1, 1
1163; GFX10-NEXT:    v_mov_b32_e32 v2, 2
1164; GFX10-NEXT:    v_mov_b32_e32 v3, 3
1165; GFX10-NEXT:    v_mov_b32_e32 v4, 4
1166; GFX10-NEXT:    ; implicit-def: $vcc_hi
1167; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
1168; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off
1169; GFX10-NEXT:    s_waitcnt vmcnt(0)
1170; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1171; GFX10-NEXT:    s_setpc_b64 s[30:31]
1172bb:
1173  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
1174  %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
1175  ret void
1176}
1177
1178declare void @llvm.memset.p5i8.i64(i8 addrspace(5)* nocapture writeonly, i8, i64, i1 immarg)
1179declare i32 @llvm.amdgcn.workitem.id.x()
1180