1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1030 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
4; RUN: llc -march=amdgcn -mcpu=gfx940 -global-isel -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GFX940 %s
5
6define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
7; GFX9-LABEL: store_load_sindex_kernel:
8; GFX9:       ; %bb.0: ; %bb
9; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
10; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
11; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
12; GFX9-NEXT:    v_mov_b32_e32 v0, 15
13; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
15; GFX9-NEXT:    s_and_b32 s0, s0, 15
16; GFX9-NEXT:    s_add_i32 s1, s1, 4
17; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
18; GFX9-NEXT:    scratch_store_dword off, v0, s1
19; GFX9-NEXT:    s_waitcnt vmcnt(0)
20; GFX9-NEXT:    s_add_i32 s0, s0, 4
21; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
22; GFX9-NEXT:    s_waitcnt vmcnt(0)
23; GFX9-NEXT:    s_endpgm
24;
25; GFX10-LABEL: store_load_sindex_kernel:
26; GFX10:       ; %bb.0: ; %bb
27; GFX10-NEXT:    s_add_u32 s2, s2, s5
28; GFX10-NEXT:    s_addc_u32 s3, s3, 0
29; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
30; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
31; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
32; GFX10-NEXT:    v_mov_b32_e32 v0, 15
33; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
34; GFX10-NEXT:    s_and_b32 s1, s0, 15
35; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
36; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
37; GFX10-NEXT:    s_add_i32 s0, s0, 4
38; GFX10-NEXT:    s_add_i32 s1, s1, 4
39; GFX10-NEXT:    scratch_store_dword off, v0, s0
40; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
41; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
42; GFX10-NEXT:    s_waitcnt vmcnt(0)
43; GFX10-NEXT:    s_endpgm
44;
45; GFX940-LABEL: store_load_sindex_kernel:
46; GFX940:       ; %bb.0: ; %bb
47; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
48; GFX940-NEXT:    v_mov_b32_e32 v0, 15
49; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
50; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
51; GFX940-NEXT:    s_and_b32 s0, s0, 15
52; GFX940-NEXT:    v_mov_b32_e32 v1, s1
53; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
54; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:4 sc0 sc1
55; GFX940-NEXT:    s_waitcnt vmcnt(0)
56; GFX940-NEXT:    v_mov_b32_e32 v0, s0
57; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
58; GFX940-NEXT:    s_waitcnt vmcnt(0)
59; GFX940-NEXT:    s_endpgm
60bb:
61  %i = alloca [32 x float], align 4, addrspace(5)
62  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
63  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
64  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
65  store volatile i32 15, i32 addrspace(5)* %i8, align 4
66  %i9 = and i32 %idx, 15
67  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
68  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
69  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
70  ret void
71}
72
73define amdgpu_kernel void @store_load_vindex_kernel() {
74; GFX9-LABEL: store_load_vindex_kernel:
75; GFX9:       ; %bb.0: ; %bb
76; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
77; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
78; GFX9-NEXT:    v_mov_b32_e32 v2, 4
79; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
80; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
81; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
82; GFX9-NEXT:    v_mov_b32_e32 v3, 15
83; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
84; GFX9-NEXT:    scratch_store_dword v1, v3, off
85; GFX9-NEXT:    s_waitcnt vmcnt(0)
86; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
87; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
88; GFX9-NEXT:    s_waitcnt vmcnt(0)
89; GFX9-NEXT:    s_endpgm
90;
91; GFX10-LABEL: store_load_vindex_kernel:
92; GFX10:       ; %bb.0: ; %bb
93; GFX10-NEXT:    s_add_u32 s0, s0, s3
94; GFX10-NEXT:    s_addc_u32 s1, s1, 0
95; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
96; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
97; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
98; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
99; GFX10-NEXT:    v_mov_b32_e32 v2, 4
100; GFX10-NEXT:    v_mov_b32_e32 v3, 15
101; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
102; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
103; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
104; GFX10-NEXT:    scratch_store_dword v0, v3, off
105; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
106; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
107; GFX10-NEXT:    s_waitcnt vmcnt(0)
108; GFX10-NEXT:    s_endpgm
109;
110; GFX940-LABEL: store_load_vindex_kernel:
111; GFX940:       ; %bb.0: ; %bb
112; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
113; GFX940-NEXT:    v_mov_b32_e32 v2, 15
114; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
115; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:4 sc0 sc1
116; GFX940-NEXT:    s_waitcnt vmcnt(0)
117; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
118; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:128 sc0 sc1
119; GFX940-NEXT:    s_waitcnt vmcnt(0)
120; GFX940-NEXT:    s_endpgm
121bb:
122  %i = alloca [32 x float], align 4, addrspace(5)
123  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
124  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
125  %i3 = zext i32 %i2 to i64
126  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
127  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
128  store volatile i32 15, i32 addrspace(5)* %i8, align 4
129  %i9 = sub nsw i32 31, %i2
130  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
131  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
132  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
133  ret void
134}
135
136define void @store_load_vindex_foo(i32 %idx) {
137; GFX9-LABEL: store_load_vindex_foo:
138; GFX9:       ; %bb.0: ; %bb
139; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
140; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
141; GFX9-NEXT:    v_mov_b32_e32 v2, s32
142; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
143; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
144; GFX9-NEXT:    v_mov_b32_e32 v3, 15
145; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
146; GFX9-NEXT:    scratch_store_dword v1, v3, off
147; GFX9-NEXT:    s_waitcnt vmcnt(0)
148; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
149; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
150; GFX9-NEXT:    s_waitcnt vmcnt(0)
151; GFX9-NEXT:    s_setpc_b64 s[30:31]
152;
153; GFX10-LABEL: store_load_vindex_foo:
154; GFX10:       ; %bb.0: ; %bb
155; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
156; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
157; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
158; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
159; GFX10-NEXT:    v_mov_b32_e32 v2, s32
160; GFX10-NEXT:    v_mov_b32_e32 v3, 15
161; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
162; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
163; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
164; GFX10-NEXT:    scratch_store_dword v0, v3, off
165; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
166; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
167; GFX10-NEXT:    s_waitcnt vmcnt(0)
168; GFX10-NEXT:    s_setpc_b64 s[30:31]
169;
170; GFX940-LABEL: store_load_vindex_foo:
171; GFX940:       ; %bb.0: ; %bb
172; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
173; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
174; GFX940-NEXT:    v_mov_b32_e32 v2, 15
175; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
176; GFX940-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
177; GFX940-NEXT:    s_waitcnt vmcnt(0)
178; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
179; GFX940-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
180; GFX940-NEXT:    s_waitcnt vmcnt(0)
181; GFX940-NEXT:    s_setpc_b64 s[30:31]
182bb:
183  %i = alloca [32 x float], align 4, addrspace(5)
184  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
185  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
186  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
187  store volatile i32 15, i32 addrspace(5)* %i8, align 4
188  %i9 = and i32 %idx, 15
189  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
190  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
191  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
192  ret void
193}
194
195define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
196; GFX9-LABEL: private_ptr_foo:
197; GFX9:       ; %bb.0:
198; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
199; GFX9-NEXT:    v_mov_b32_e32 v1, 0x41200000
200; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:4
201; GFX9-NEXT:    s_waitcnt vmcnt(0)
202; GFX9-NEXT:    s_setpc_b64 s[30:31]
203;
204; GFX10-LABEL: private_ptr_foo:
205; GFX10:       ; %bb.0:
206; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
207; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
208; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
209; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
210; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
211; GFX10-NEXT:    s_setpc_b64 s[30:31]
212;
213; GFX940-LABEL: private_ptr_foo:
214; GFX940:       ; %bb.0:
215; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
216; GFX940-NEXT:    v_mov_b32_e32 v1, 0x41200000
217; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4
218; GFX940-NEXT:    s_waitcnt vmcnt(0)
219; GFX940-NEXT:    s_setpc_b64 s[30:31]
220  %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
221  store float 1.000000e+01, float addrspace(5)* %gep, align 4
222  ret void
223}
224
225define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
226; GFX9-LABEL: store_load_sindex_small_offset_kernel:
227; GFX9:       ; %bb.0: ; %bb
228; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
229; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
230; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
231; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
232; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
233; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
234; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
235; GFX9-NEXT:    s_and_b32 s0, s0, 15
236; GFX9-NEXT:    v_mov_b32_e32 v0, 15
237; GFX9-NEXT:    s_addk_i32 s1, 0x104
238; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
239; GFX9-NEXT:    scratch_store_dword off, v0, s1
240; GFX9-NEXT:    s_waitcnt vmcnt(0)
241; GFX9-NEXT:    s_addk_i32 s0, 0x104
242; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
243; GFX9-NEXT:    s_waitcnt vmcnt(0)
244; GFX9-NEXT:    s_endpgm
245;
246; GFX10-LABEL: store_load_sindex_small_offset_kernel:
247; GFX10:       ; %bb.0: ; %bb
248; GFX10-NEXT:    s_add_u32 s2, s2, s5
249; GFX10-NEXT:    s_addc_u32 s3, s3, 0
250; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
251; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
252; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
253; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
254; GFX10-NEXT:    s_waitcnt vmcnt(0)
255; GFX10-NEXT:    v_mov_b32_e32 v0, 15
256; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
257; GFX10-NEXT:    s_and_b32 s1, s0, 15
258; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
259; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
260; GFX10-NEXT:    s_addk_i32 s0, 0x104
261; GFX10-NEXT:    s_addk_i32 s1, 0x104
262; GFX10-NEXT:    scratch_store_dword off, v0, s0
263; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
264; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
265; GFX10-NEXT:    s_waitcnt vmcnt(0)
266; GFX10-NEXT:    s_endpgm
267;
268; GFX940-LABEL: store_load_sindex_small_offset_kernel:
269; GFX940:       ; %bb.0: ; %bb
270; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
271; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
272; GFX940-NEXT:    s_waitcnt vmcnt(0)
273; GFX940-NEXT:    v_mov_b32_e32 v0, 15
274; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
275; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
276; GFX940-NEXT:    s_and_b32 s0, s0, 15
277; GFX940-NEXT:    v_mov_b32_e32 v1, s1
278; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
279; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:260 sc0 sc1
280; GFX940-NEXT:    s_waitcnt vmcnt(0)
281; GFX940-NEXT:    v_mov_b32_e32 v0, s0
282; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:260 sc0 sc1
283; GFX940-NEXT:    s_waitcnt vmcnt(0)
284; GFX940-NEXT:    s_endpgm
285bb:
286  %padding = alloca [64 x i32], align 4, addrspace(5)
287  %i = alloca [32 x float], align 4, addrspace(5)
288  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
289  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
290  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
291  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
292  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
293  store volatile i32 15, i32 addrspace(5)* %i8, align 4
294  %i9 = and i32 %idx, 15
295  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
296  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
297  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
298  ret void
299}
300
301define amdgpu_kernel void @store_load_vindex_small_offset_kernel() {
302; GFX9-LABEL: store_load_vindex_small_offset_kernel:
303; GFX9:       ; %bb.0: ; %bb
304; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
305; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
306; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
307; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
308; GFX9-NEXT:    s_waitcnt vmcnt(0)
309; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
310; GFX9-NEXT:    v_mov_b32_e32 v2, 0x104
311; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
312; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
313; GFX9-NEXT:    v_mov_b32_e32 v3, 15
314; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
315; GFX9-NEXT:    scratch_store_dword v1, v3, off
316; GFX9-NEXT:    s_waitcnt vmcnt(0)
317; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
318; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
319; GFX9-NEXT:    s_waitcnt vmcnt(0)
320; GFX9-NEXT:    s_endpgm
321;
322; GFX10-LABEL: store_load_vindex_small_offset_kernel:
323; GFX10:       ; %bb.0: ; %bb
324; GFX10-NEXT:    s_add_u32 s0, s0, s3
325; GFX10-NEXT:    s_addc_u32 s1, s1, 0
326; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
327; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
328; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
329; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
330; GFX10-NEXT:    v_mov_b32_e32 v2, 0x104
331; GFX10-NEXT:    v_mov_b32_e32 v3, 15
332; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
333; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
334; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
335; GFX10-NEXT:    scratch_load_dword v2, off, off offset:4 glc dlc
336; GFX10-NEXT:    s_waitcnt vmcnt(0)
337; GFX10-NEXT:    scratch_store_dword v0, v3, off
338; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
339; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
340; GFX10-NEXT:    s_waitcnt vmcnt(0)
341; GFX10-NEXT:    s_endpgm
342;
343; GFX940-LABEL: store_load_vindex_small_offset_kernel:
344; GFX940:       ; %bb.0: ; %bb
345; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
346; GFX940-NEXT:    s_waitcnt vmcnt(0)
347; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
348; GFX940-NEXT:    v_mov_b32_e32 v2, 15
349; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
350; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:260 sc0 sc1
351; GFX940-NEXT:    s_waitcnt vmcnt(0)
352; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
353; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:384 sc0 sc1
354; GFX940-NEXT:    s_waitcnt vmcnt(0)
355; GFX940-NEXT:    s_endpgm
356bb:
357  %padding = alloca [64 x i32], align 4, addrspace(5)
358  %i = alloca [32 x float], align 4, addrspace(5)
359  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
360  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
361  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
362  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
363  %i3 = zext i32 %i2 to i64
364  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
365  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
366  store volatile i32 15, i32 addrspace(5)* %i8, align 4
367  %i9 = sub nsw i32 31, %i2
368  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
369  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
370  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
371  ret void
372}
373
374define void @store_load_vindex_small_offset_foo(i32 %idx) {
375; GFX9-LABEL: store_load_vindex_small_offset_foo:
376; GFX9:       ; %bb.0: ; %bb
377; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
378; GFX9-NEXT:    scratch_load_dword v1, off, s32 glc
379; GFX9-NEXT:    s_waitcnt vmcnt(0)
380; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x100
381; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
382; GFX9-NEXT:    v_mov_b32_e32 v2, vcc_hi
383; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
384; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
385; GFX9-NEXT:    v_mov_b32_e32 v3, 15
386; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
387; GFX9-NEXT:    scratch_store_dword v1, v3, off
388; GFX9-NEXT:    s_waitcnt vmcnt(0)
389; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
390; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
391; GFX9-NEXT:    s_waitcnt vmcnt(0)
392; GFX9-NEXT:    s_setpc_b64 s[30:31]
393;
394; GFX10-LABEL: store_load_vindex_small_offset_foo:
395; GFX10:       ; %bb.0: ; %bb
396; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
397; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
398; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
399; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x100
400; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
401; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
402; GFX10-NEXT:    v_mov_b32_e32 v3, 15
403; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
404; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
405; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
406; GFX10-NEXT:    scratch_load_dword v2, off, s32 glc dlc
407; GFX10-NEXT:    s_waitcnt vmcnt(0)
408; GFX10-NEXT:    scratch_store_dword v0, v3, off
409; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
410; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
411; GFX10-NEXT:    s_waitcnt vmcnt(0)
412; GFX10-NEXT:    s_setpc_b64 s[30:31]
413;
414; GFX940-LABEL: store_load_vindex_small_offset_foo:
415; GFX940:       ; %bb.0: ; %bb
416; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
417; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
418; GFX940-NEXT:    s_waitcnt vmcnt(0)
419; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
420; GFX940-NEXT:    v_mov_b32_e32 v2, 15
421; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
422; GFX940-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
423; GFX940-NEXT:    s_waitcnt vmcnt(0)
424; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
425; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
426; GFX940-NEXT:    s_waitcnt vmcnt(0)
427; GFX940-NEXT:    s_setpc_b64 s[30:31]
428bb:
429  %padding = alloca [64 x i32], align 4, addrspace(5)
430  %i = alloca [32 x float], align 4, addrspace(5)
431  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
432  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
433  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
434  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
435  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
436  store volatile i32 15, i32 addrspace(5)* %i8, align 4
437  %i9 = and i32 %idx, 15
438  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
439  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
440  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
441  ret void
442}
443
444define amdgpu_kernel void @store_load_sindex_large_offset_kernel(i32 %idx) {
445; GFX9-LABEL: store_load_sindex_large_offset_kernel:
446; GFX9:       ; %bb.0: ; %bb
447; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
448; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
449; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
450; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
451; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
452; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
453; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
454; GFX9-NEXT:    s_and_b32 s0, s0, 15
455; GFX9-NEXT:    v_mov_b32_e32 v0, 15
456; GFX9-NEXT:    s_addk_i32 s1, 0x4004
457; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
458; GFX9-NEXT:    scratch_store_dword off, v0, s1
459; GFX9-NEXT:    s_waitcnt vmcnt(0)
460; GFX9-NEXT:    s_addk_i32 s0, 0x4004
461; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
462; GFX9-NEXT:    s_waitcnt vmcnt(0)
463; GFX9-NEXT:    s_endpgm
464;
465; GFX10-LABEL: store_load_sindex_large_offset_kernel:
466; GFX10:       ; %bb.0: ; %bb
467; GFX10-NEXT:    s_add_u32 s2, s2, s5
468; GFX10-NEXT:    s_addc_u32 s3, s3, 0
469; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
470; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
471; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
472; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
473; GFX10-NEXT:    s_waitcnt vmcnt(0)
474; GFX10-NEXT:    v_mov_b32_e32 v0, 15
475; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
476; GFX10-NEXT:    s_and_b32 s1, s0, 15
477; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
478; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
479; GFX10-NEXT:    s_addk_i32 s0, 0x4004
480; GFX10-NEXT:    s_addk_i32 s1, 0x4004
481; GFX10-NEXT:    scratch_store_dword off, v0, s0
482; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
483; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
484; GFX10-NEXT:    s_waitcnt vmcnt(0)
485; GFX10-NEXT:    s_endpgm
486;
487; GFX940-LABEL: store_load_sindex_large_offset_kernel:
488; GFX940:       ; %bb.0: ; %bb
489; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
490; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
491; GFX940-NEXT:    s_waitcnt vmcnt(0)
492; GFX940-NEXT:    v_mov_b32_e32 v0, 15
493; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
494; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
495; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
496; GFX940-NEXT:    s_and_b32 s0, s0, 15
497; GFX940-NEXT:    v_mov_b32_e32 v1, s1
498; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
499; GFX940-NEXT:    scratch_store_dword v1, v0, vcc_hi sc0 sc1
500; GFX940-NEXT:    s_waitcnt vmcnt(0)
501; GFX940-NEXT:    v_mov_b32_e32 v0, s0
502; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
503; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
504; GFX940-NEXT:    s_waitcnt vmcnt(0)
505; GFX940-NEXT:    s_endpgm
506bb:
507  %padding = alloca [4096 x i32], align 4, addrspace(5)
508  %i = alloca [32 x float], align 4, addrspace(5)
509  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
510  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
511  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
512  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
513  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
514  store volatile i32 15, i32 addrspace(5)* %i8, align 4
515  %i9 = and i32 %idx, 15
516  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
517  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
518  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
519  ret void
520}
521
522define amdgpu_kernel void @store_load_vindex_large_offset_kernel() {
523; GFX9-LABEL: store_load_vindex_large_offset_kernel:
524; GFX9:       ; %bb.0: ; %bb
525; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
526; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
527; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
528; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
529; GFX9-NEXT:    s_waitcnt vmcnt(0)
530; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
531; GFX9-NEXT:    v_mov_b32_e32 v2, 0x4004
532; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
533; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
534; GFX9-NEXT:    v_mov_b32_e32 v3, 15
535; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
536; GFX9-NEXT:    scratch_store_dword v1, v3, off
537; GFX9-NEXT:    s_waitcnt vmcnt(0)
538; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
539; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
540; GFX9-NEXT:    s_waitcnt vmcnt(0)
541; GFX9-NEXT:    s_endpgm
542;
543; GFX10-LABEL: store_load_vindex_large_offset_kernel:
544; GFX10:       ; %bb.0: ; %bb
545; GFX10-NEXT:    s_add_u32 s0, s0, s3
546; GFX10-NEXT:    s_addc_u32 s1, s1, 0
547; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
548; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
549; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
550; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
551; GFX10-NEXT:    v_mov_b32_e32 v2, 0x4004
552; GFX10-NEXT:    v_mov_b32_e32 v3, 15
553; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
554; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
555; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
556; GFX10-NEXT:    scratch_load_dword v2, off, off offset:4 glc dlc
557; GFX10-NEXT:    s_waitcnt vmcnt(0)
558; GFX10-NEXT:    scratch_store_dword v0, v3, off
559; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
560; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
561; GFX10-NEXT:    s_waitcnt vmcnt(0)
562; GFX10-NEXT:    s_endpgm
563;
564; GFX940-LABEL: store_load_vindex_large_offset_kernel:
565; GFX940:       ; %bb.0: ; %bb
566; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
567; GFX940-NEXT:    s_waitcnt vmcnt(0)
568; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
569; GFX940-NEXT:    v_mov_b32_e32 v2, 15
570; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
571; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
572; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
573; GFX940-NEXT:    s_waitcnt vmcnt(0)
574; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
575; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
576; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi offset:124 sc0 sc1
577; GFX940-NEXT:    s_waitcnt vmcnt(0)
578; GFX940-NEXT:    s_endpgm
579bb:
580  %padding = alloca [4096 x i32], align 4, addrspace(5)
581  %i = alloca [32 x float], align 4, addrspace(5)
582  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
583  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
584  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
585  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
586  %i3 = zext i32 %i2 to i64
587  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
588  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
589  store volatile i32 15, i32 addrspace(5)* %i8, align 4
590  %i9 = sub nsw i32 31, %i2
591  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
592  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
593  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
594  ret void
595}
596
597define void @store_load_vindex_large_offset_foo(i32 %idx) {
598; GFX9-LABEL: store_load_vindex_large_offset_foo:
599; GFX9:       ; %bb.0: ; %bb
600; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
601; GFX9-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
602; GFX9-NEXT:    s_waitcnt vmcnt(0)
603; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
604; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
605; GFX9-NEXT:    v_mov_b32_e32 v2, vcc_hi
606; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
607; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
608; GFX9-NEXT:    v_mov_b32_e32 v3, 15
609; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
610; GFX9-NEXT:    scratch_store_dword v1, v3, off
611; GFX9-NEXT:    s_waitcnt vmcnt(0)
612; GFX9-NEXT:    v_add_u32_e32 v0, v2, v0
613; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
614; GFX9-NEXT:    s_waitcnt vmcnt(0)
615; GFX9-NEXT:    s_setpc_b64 s[30:31]
616;
617; GFX10-LABEL: store_load_vindex_large_offset_foo:
618; GFX10:       ; %bb.0: ; %bb
619; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
620; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
621; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
622; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
623; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
624; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
625; GFX10-NEXT:    v_mov_b32_e32 v3, 15
626; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
627; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
628; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
629; GFX10-NEXT:    scratch_load_dword v2, off, s32 offset:4 glc dlc
630; GFX10-NEXT:    s_waitcnt vmcnt(0)
631; GFX10-NEXT:    scratch_store_dword v0, v3, off
632; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
633; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
634; GFX10-NEXT:    s_waitcnt vmcnt(0)
635; GFX10-NEXT:    s_setpc_b64 s[30:31]
636;
637; GFX940-LABEL: store_load_vindex_large_offset_foo:
638; GFX940:       ; %bb.0: ; %bb
639; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
640; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
641; GFX940-NEXT:    s_waitcnt vmcnt(0)
642; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
643; GFX940-NEXT:    v_mov_b32_e32 v2, 15
644; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
645; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
646; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
647; GFX940-NEXT:    s_waitcnt vmcnt(0)
648; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
649; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
650; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
651; GFX940-NEXT:    s_waitcnt vmcnt(0)
652; GFX940-NEXT:    s_setpc_b64 s[30:31]
653bb:
654  %padding = alloca [4096 x i32], align 4, addrspace(5)
655  %i = alloca [32 x float], align 4, addrspace(5)
656  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
657  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
658  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
659  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
660  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
661  store volatile i32 15, i32 addrspace(5)* %i8, align 4
662  %i9 = and i32 %idx, 15
663  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
664  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
665  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
666  ret void
667}
668
669define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
670; GFX9-LABEL: store_load_large_imm_offset_kernel:
671; GFX9:       ; %bb.0: ; %bb
672; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
673; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
674; GFX9-NEXT:    v_mov_b32_e32 v0, 13
675; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
676; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
677; GFX9-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
678; GFX9-NEXT:    s_waitcnt vmcnt(0)
679; GFX9-NEXT:    v_mov_b32_e32 v0, 15
680; GFX9-NEXT:    s_add_i32 s0, s0, 4
681; GFX9-NEXT:    scratch_store_dword off, v0, s0
682; GFX9-NEXT:    s_waitcnt vmcnt(0)
683; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
684; GFX9-NEXT:    s_waitcnt vmcnt(0)
685; GFX9-NEXT:    s_endpgm
686;
687; GFX10-LABEL: store_load_large_imm_offset_kernel:
688; GFX10:       ; %bb.0: ; %bb
689; GFX10-NEXT:    s_add_u32 s0, s0, s3
690; GFX10-NEXT:    s_addc_u32 s1, s1, 0
691; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
692; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
693; GFX10-NEXT:    v_mov_b32_e32 v0, 13
694; GFX10-NEXT:    v_mov_b32_e32 v1, 15
695; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
696; GFX10-NEXT:    s_add_i32 s0, s0, 4
697; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
698; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
699; GFX10-NEXT:    scratch_store_dword off, v1, s0
700; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
701; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
702; GFX10-NEXT:    s_waitcnt vmcnt(0)
703; GFX10-NEXT:    s_endpgm
704;
705; GFX940-LABEL: store_load_large_imm_offset_kernel:
706; GFX940:       ; %bb.0: ; %bb
707; GFX940-NEXT:    v_mov_b32_e32 v0, 13
708; GFX940-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
709; GFX940-NEXT:    s_waitcnt vmcnt(0)
710; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
711; GFX940-NEXT:    v_mov_b32_e32 v1, 15
712; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
713; GFX940-NEXT:    s_waitcnt vmcnt(0)
714; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
715; GFX940-NEXT:    s_waitcnt vmcnt(0)
716; GFX940-NEXT:    s_endpgm
717bb:
718  %i = alloca [4096 x i32], align 4, addrspace(5)
719  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
720  store volatile i32 13, i32 addrspace(5)* %i1, align 4
721  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
722  store volatile i32 15, i32 addrspace(5)* %i7, align 4
723  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
724  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
725  ret void
726}
727
728define void @store_load_large_imm_offset_foo() {
729; GFX9-LABEL: store_load_large_imm_offset_foo:
730; GFX9:       ; %bb.0: ; %bb
731; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
732; GFX9-NEXT:    v_mov_b32_e32 v0, 13
733; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
734; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 4
735; GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
736; GFX9-NEXT:    s_waitcnt vmcnt(0)
737; GFX9-NEXT:    v_mov_b32_e32 v0, 15
738; GFX9-NEXT:    s_add_i32 s0, s0, vcc_hi
739; GFX9-NEXT:    scratch_store_dword off, v0, s0
740; GFX9-NEXT:    s_waitcnt vmcnt(0)
741; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
742; GFX9-NEXT:    s_waitcnt vmcnt(0)
743; GFX9-NEXT:    s_setpc_b64 s[30:31]
744;
745; GFX10-LABEL: store_load_large_imm_offset_foo:
746; GFX10:       ; %bb.0: ; %bb
747; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
748; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
749; GFX10-NEXT:    v_mov_b32_e32 v0, 13
750; GFX10-NEXT:    v_mov_b32_e32 v1, 15
751; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
752; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 4
753; GFX10-NEXT:    s_add_i32 s0, s0, vcc_lo
754; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
755; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
756; GFX10-NEXT:    scratch_store_dword off, v1, s0
757; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
758; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
759; GFX10-NEXT:    s_waitcnt vmcnt(0)
760; GFX10-NEXT:    s_setpc_b64 s[30:31]
761;
762; GFX940-LABEL: store_load_large_imm_offset_foo:
763; GFX940:       ; %bb.0: ; %bb
764; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
765; GFX940-NEXT:    v_mov_b32_e32 v0, 13
766; GFX940-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
767; GFX940-NEXT:    s_waitcnt vmcnt(0)
768; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
769; GFX940-NEXT:    v_mov_b32_e32 v1, 15
770; GFX940-NEXT:    scratch_store_dword v0, v1, s32 offset:4 sc0 sc1
771; GFX940-NEXT:    s_waitcnt vmcnt(0)
772; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:4 sc0 sc1
773; GFX940-NEXT:    s_waitcnt vmcnt(0)
774; GFX940-NEXT:    s_setpc_b64 s[30:31]
775bb:
776  %i = alloca [4096 x i32], align 4, addrspace(5)
777  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
778  store volatile i32 13, i32 addrspace(5)* %i1, align 4
779  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
780  store volatile i32 15, i32 addrspace(5)* %i7, align 4
781  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
782  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
783  ret void
784}
785
786define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
787; GFX9-LABEL: store_load_vidx_sidx_offset:
788; GFX9:       ; %bb.0: ; %bb
789; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
790; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
791; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
792; GFX9-NEXT:    v_mov_b32_e32 v1, 15
793; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
794; GFX9-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
795; GFX9-NEXT:    v_add_u32_e32 v0, 4, v0
796; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
797; GFX9-NEXT:    s_waitcnt vmcnt(0)
798; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
799; GFX9-NEXT:    s_waitcnt vmcnt(0)
800; GFX9-NEXT:    s_endpgm
801;
802; GFX10-LABEL: store_load_vidx_sidx_offset:
803; GFX10:       ; %bb.0: ; %bb
804; GFX10-NEXT:    s_add_u32 s2, s2, s5
805; GFX10-NEXT:    s_addc_u32 s3, s3, 0
806; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
807; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
808; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
809; GFX10-NEXT:    v_mov_b32_e32 v1, 15
810; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX10-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
812; GFX10-NEXT:    v_add_nc_u32_e32 v0, 4, v0
813; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
814; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
815; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
816; GFX10-NEXT:    s_waitcnt vmcnt(0)
817; GFX10-NEXT:    s_endpgm
818;
819; GFX940-LABEL: store_load_vidx_sidx_offset:
820; GFX940:       ; %bb.0: ; %bb
821; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
822; GFX940-NEXT:    v_mov_b32_e32 v1, 15
823; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
824; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
825; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
826; GFX940-NEXT:    s_waitcnt vmcnt(0)
827; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
828; GFX940-NEXT:    s_waitcnt vmcnt(0)
829; GFX940-NEXT:    s_endpgm
830bb:
831  %alloca = alloca [32 x i32], align 4, addrspace(5)
832  %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
833  %add1 = add nsw i32 %sidx, %vidx
834  %add2 = add nsw i32 %add1, 256
835  %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
836  store volatile i32 15, i32 addrspace(5)* %gep, align 4
837  %load = load volatile i32, i32 addrspace(5)* %gep, align 4
838  ret void
839}
840
841define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
842; GFX9-LABEL: store_load_i64_aligned:
843; GFX9:       ; %bb.0: ; %bb
844; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
845; GFX9-NEXT:    v_mov_b32_e32 v1, 15
846; GFX9-NEXT:    v_mov_b32_e32 v2, 0
847; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
848; GFX9-NEXT:    s_waitcnt vmcnt(0)
849; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
850; GFX9-NEXT:    s_waitcnt vmcnt(0)
851; GFX9-NEXT:    s_setpc_b64 s[30:31]
852;
853; GFX10-LABEL: store_load_i64_aligned:
854; GFX10:       ; %bb.0: ; %bb
855; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
856; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
857; GFX10-NEXT:    v_mov_b32_e32 v1, 15
858; GFX10-NEXT:    v_mov_b32_e32 v2, 0
859; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
860; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
861; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
862; GFX10-NEXT:    s_waitcnt vmcnt(0)
863; GFX10-NEXT:    s_setpc_b64 s[30:31]
864;
865; GFX940-LABEL: store_load_i64_aligned:
866; GFX940:       ; %bb.0: ; %bb
867; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
868; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
869; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
870; GFX940-NEXT:    s_waitcnt vmcnt(0)
871; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
872; GFX940-NEXT:    s_waitcnt vmcnt(0)
873; GFX940-NEXT:    s_setpc_b64 s[30:31]
874bb:
875  store volatile i64 15, i64 addrspace(5)* %arg, align 8
876  %load = load volatile i64, i64 addrspace(5)* %arg, align 8
877  ret void
878}
879
880define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
881; GFX9-LABEL: store_load_i64_unaligned:
882; GFX9:       ; %bb.0: ; %bb
883; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
884; GFX9-NEXT:    v_mov_b32_e32 v1, 15
885; GFX9-NEXT:    v_mov_b32_e32 v2, 0
886; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
887; GFX9-NEXT:    s_waitcnt vmcnt(0)
888; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
889; GFX9-NEXT:    s_waitcnt vmcnt(0)
890; GFX9-NEXT:    s_setpc_b64 s[30:31]
891;
892; GFX10-LABEL: store_load_i64_unaligned:
893; GFX10:       ; %bb.0: ; %bb
894; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
895; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
896; GFX10-NEXT:    v_mov_b32_e32 v1, 15
897; GFX10-NEXT:    v_mov_b32_e32 v2, 0
898; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
899; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
900; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
901; GFX10-NEXT:    s_waitcnt vmcnt(0)
902; GFX10-NEXT:    s_setpc_b64 s[30:31]
903;
904; GFX940-LABEL: store_load_i64_unaligned:
905; GFX940:       ; %bb.0: ; %bb
906; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
907; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
908; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
909; GFX940-NEXT:    s_waitcnt vmcnt(0)
910; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
911; GFX940-NEXT:    s_waitcnt vmcnt(0)
912; GFX940-NEXT:    s_setpc_b64 s[30:31]
913bb:
914  store volatile i64 15, i64 addrspace(5)* %arg, align 1
915  %load = load volatile i64, i64 addrspace(5)* %arg, align 1
916  ret void
917}
918
919define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
920; GFX9-LABEL: store_load_v3i32_unaligned:
921; GFX9:       ; %bb.0: ; %bb
922; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
923; GFX9-NEXT:    s_mov_b32 s2, 3
924; GFX9-NEXT:    s_mov_b32 s1, 2
925; GFX9-NEXT:    s_mov_b32 s0, 1
926; GFX9-NEXT:    v_mov_b32_e32 v3, s2
927; GFX9-NEXT:    v_mov_b32_e32 v2, s1
928; GFX9-NEXT:    v_mov_b32_e32 v1, s0
929; GFX9-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
930; GFX9-NEXT:    s_waitcnt vmcnt(0)
931; GFX9-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
932; GFX9-NEXT:    s_waitcnt vmcnt(0)
933; GFX9-NEXT:    s_setpc_b64 s[30:31]
934;
935; GFX10-LABEL: store_load_v3i32_unaligned:
936; GFX10:       ; %bb.0: ; %bb
937; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
938; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
939; GFX10-NEXT:    s_mov_b32 s2, 3
940; GFX10-NEXT:    s_mov_b32 s1, 2
941; GFX10-NEXT:    s_mov_b32 s0, 1
942; GFX10-NEXT:    v_mov_b32_e32 v3, s2
943; GFX10-NEXT:    v_mov_b32_e32 v2, s1
944; GFX10-NEXT:    v_mov_b32_e32 v1, s0
945; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
946; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
947; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
948; GFX10-NEXT:    s_waitcnt vmcnt(0)
949; GFX10-NEXT:    s_setpc_b64 s[30:31]
950;
951; GFX940-LABEL: store_load_v3i32_unaligned:
952; GFX940:       ; %bb.0: ; %bb
953; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
954; GFX940-NEXT:    s_mov_b32 s2, 3
955; GFX940-NEXT:    s_mov_b32 s1, 2
956; GFX940-NEXT:    s_mov_b32 s0, 1
957; GFX940-NEXT:    v_mov_b32_e32 v4, s2
958; GFX940-NEXT:    v_mov_b32_e32 v3, s1
959; GFX940-NEXT:    v_mov_b32_e32 v2, s0
960; GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
961; GFX940-NEXT:    s_waitcnt vmcnt(0)
962; GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
963; GFX940-NEXT:    s_waitcnt vmcnt(0)
964; GFX940-NEXT:    s_setpc_b64 s[30:31]
965bb:
966  store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
967  %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
968  ret void
969}
970
971define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
972; GFX9-LABEL: store_load_v4i32_unaligned:
973; GFX9:       ; %bb.0: ; %bb
974; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
975; GFX9-NEXT:    s_mov_b32 s3, 4
976; GFX9-NEXT:    s_mov_b32 s2, 3
977; GFX9-NEXT:    s_mov_b32 s1, 2
978; GFX9-NEXT:    s_mov_b32 s0, 1
979; GFX9-NEXT:    v_mov_b32_e32 v4, s3
980; GFX9-NEXT:    v_mov_b32_e32 v3, s2
981; GFX9-NEXT:    v_mov_b32_e32 v2, s1
982; GFX9-NEXT:    v_mov_b32_e32 v1, s0
983; GFX9-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
984; GFX9-NEXT:    s_waitcnt vmcnt(0)
985; GFX9-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
986; GFX9-NEXT:    s_waitcnt vmcnt(0)
987; GFX9-NEXT:    s_setpc_b64 s[30:31]
988;
989; GFX10-LABEL: store_load_v4i32_unaligned:
990; GFX10:       ; %bb.0: ; %bb
991; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
992; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
993; GFX10-NEXT:    s_mov_b32 s3, 4
994; GFX10-NEXT:    s_mov_b32 s2, 3
995; GFX10-NEXT:    s_mov_b32 s1, 2
996; GFX10-NEXT:    s_mov_b32 s0, 1
997; GFX10-NEXT:    v_mov_b32_e32 v4, s3
998; GFX10-NEXT:    v_mov_b32_e32 v3, s2
999; GFX10-NEXT:    v_mov_b32_e32 v2, s1
1000; GFX10-NEXT:    v_mov_b32_e32 v1, s0
1001; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
1002; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1003; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
1004; GFX10-NEXT:    s_waitcnt vmcnt(0)
1005; GFX10-NEXT:    s_setpc_b64 s[30:31]
1006;
1007; GFX940-LABEL: store_load_v4i32_unaligned:
1008; GFX940:       ; %bb.0: ; %bb
1009; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1010; GFX940-NEXT:    s_mov_b32 s3, 4
1011; GFX940-NEXT:    s_mov_b32 s2, 3
1012; GFX940-NEXT:    s_mov_b32 s1, 2
1013; GFX940-NEXT:    s_mov_b32 s0, 1
1014; GFX940-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
1015; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
1016; GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
1017; GFX940-NEXT:    s_waitcnt vmcnt(0)
1018; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
1019; GFX940-NEXT:    s_waitcnt vmcnt(0)
1020; GFX940-NEXT:    s_setpc_b64 s[30:31]
1021bb:
1022  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
1023  %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
1024  ret void
1025}
1026
1027declare i32 @llvm.amdgcn.workitem.id.x()
1028