1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1030 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
4; RUN: llc -march=amdgcn -mcpu=gfx940 -global-isel -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GFX940 %s
5
6define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
7; GFX9-LABEL: store_load_sindex_kernel:
8; GFX9:       ; %bb.0: ; %bb
9; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
10; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
11; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
12; GFX9-NEXT:    v_mov_b32_e32 v0, 15
13; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
15; GFX9-NEXT:    s_and_b32 s0, s0, 15
16; GFX9-NEXT:    s_add_i32 s1, s1, 4
17; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
18; GFX9-NEXT:    scratch_store_dword off, v0, s1
19; GFX9-NEXT:    s_waitcnt vmcnt(0)
20; GFX9-NEXT:    s_add_i32 s0, s0, 4
21; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
22; GFX9-NEXT:    s_waitcnt vmcnt(0)
23; GFX9-NEXT:    s_endpgm
24;
25; GFX10-LABEL: store_load_sindex_kernel:
26; GFX10:       ; %bb.0: ; %bb
27; GFX10-NEXT:    s_add_u32 s2, s2, s5
28; GFX10-NEXT:    s_addc_u32 s3, s3, 0
29; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
30; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
31; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
32; GFX10-NEXT:    v_mov_b32_e32 v0, 15
33; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
34; GFX10-NEXT:    s_and_b32 s1, s0, 15
35; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
36; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
37; GFX10-NEXT:    s_add_i32 s0, s0, 4
38; GFX10-NEXT:    s_add_i32 s1, s1, 4
39; GFX10-NEXT:    scratch_store_dword off, v0, s0
40; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
41; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
42; GFX10-NEXT:    s_waitcnt vmcnt(0)
43; GFX10-NEXT:    s_endpgm
44;
45; GFX940-LABEL: store_load_sindex_kernel:
46; GFX940:       ; %bb.0: ; %bb
47; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
48; GFX940-NEXT:    v_mov_b32_e32 v0, 15
49; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
50; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
51; GFX940-NEXT:    s_and_b32 s0, s0, 15
52; GFX940-NEXT:    v_mov_b32_e32 v1, s1
53; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
54; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:4 sc0 sc1
55; GFX940-NEXT:    s_waitcnt vmcnt(0)
56; GFX940-NEXT:    v_mov_b32_e32 v0, s0
57; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
58; GFX940-NEXT:    s_waitcnt vmcnt(0)
59; GFX940-NEXT:    s_endpgm
60bb:
61  %i = alloca [32 x float], align 4, addrspace(5)
62  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
63  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
64  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
65  store volatile i32 15, i32 addrspace(5)* %i8, align 4
66  %i9 = and i32 %idx, 15
67  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
68  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
69  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
70  ret void
71}
72
73define amdgpu_kernel void @store_load_vindex_kernel() {
74; GFX9-LABEL: store_load_vindex_kernel:
75; GFX9:       ; %bb.0: ; %bb
76; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
77; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
78; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
79; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
80; GFX9-NEXT:    v_add_u32_e32 v1, 4, v1
81; GFX9-NEXT:    v_mov_b32_e32 v2, 15
82; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
83; GFX9-NEXT:    scratch_store_dword v1, v2, off
84; GFX9-NEXT:    s_waitcnt vmcnt(0)
85; GFX9-NEXT:    v_add_u32_e32 v0, 4, v0
86; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
87; GFX9-NEXT:    s_waitcnt vmcnt(0)
88; GFX9-NEXT:    s_endpgm
89;
90; GFX10-LABEL: store_load_vindex_kernel:
91; GFX10:       ; %bb.0: ; %bb
92; GFX10-NEXT:    s_add_u32 s0, s0, s3
93; GFX10-NEXT:    s_addc_u32 s1, s1, 0
94; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
95; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
96; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
97; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
98; GFX10-NEXT:    v_mov_b32_e32 v2, 15
99; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
100; GFX10-NEXT:    v_add_nc_u32_e32 v0, 4, v0
101; GFX10-NEXT:    v_add_nc_u32_e32 v1, 4, v1
102; GFX10-NEXT:    scratch_store_dword v0, v2, off
103; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
104; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
105; GFX10-NEXT:    s_waitcnt vmcnt(0)
106; GFX10-NEXT:    s_endpgm
107;
108; GFX940-LABEL: store_load_vindex_kernel:
109; GFX940:       ; %bb.0: ; %bb
110; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
111; GFX940-NEXT:    v_mov_b32_e32 v2, 15
112; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
113; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:4 sc0 sc1
114; GFX940-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
116; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:128 sc0 sc1
117; GFX940-NEXT:    s_waitcnt vmcnt(0)
118; GFX940-NEXT:    s_endpgm
119bb:
120  %i = alloca [32 x float], align 4, addrspace(5)
121  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
122  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
123  %i3 = zext i32 %i2 to i64
124  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
125  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
126  store volatile i32 15, i32 addrspace(5)* %i8, align 4
127  %i9 = sub nsw i32 31, %i2
128  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
129  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
130  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
131  ret void
132}
133
134define void @store_load_vindex_foo(i32 %idx) {
135; GFX9-LABEL: store_load_vindex_foo:
136; GFX9:       ; %bb.0: ; %bb
137; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
138; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
139; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
140; GFX9-NEXT:    v_add_u32_e32 v1, s32, v1
141; GFX9-NEXT:    v_mov_b32_e32 v2, 15
142; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
143; GFX9-NEXT:    scratch_store_dword v1, v2, off
144; GFX9-NEXT:    s_waitcnt vmcnt(0)
145; GFX9-NEXT:    v_add_u32_e32 v0, s32, v0
146; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
147; GFX9-NEXT:    s_waitcnt vmcnt(0)
148; GFX9-NEXT:    s_setpc_b64 s[30:31]
149;
150; GFX10-LABEL: store_load_vindex_foo:
151; GFX10:       ; %bb.0: ; %bb
152; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
153; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
154; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
155; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
156; GFX10-NEXT:    v_mov_b32_e32 v2, 15
157; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
158; GFX10-NEXT:    v_add_nc_u32_e32 v0, s32, v0
159; GFX10-NEXT:    v_add_nc_u32_e32 v1, s32, v1
160; GFX10-NEXT:    scratch_store_dword v0, v2, off
161; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
162; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
163; GFX10-NEXT:    s_waitcnt vmcnt(0)
164; GFX10-NEXT:    s_setpc_b64 s[30:31]
165;
166; GFX940-LABEL: store_load_vindex_foo:
167; GFX940:       ; %bb.0: ; %bb
168; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
169; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
170; GFX940-NEXT:    v_mov_b32_e32 v2, 15
171; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
172; GFX940-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
173; GFX940-NEXT:    s_waitcnt vmcnt(0)
174; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
175; GFX940-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
176; GFX940-NEXT:    s_waitcnt vmcnt(0)
177; GFX940-NEXT:    s_setpc_b64 s[30:31]
178bb:
179  %i = alloca [32 x float], align 4, addrspace(5)
180  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
181  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
182  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
183  store volatile i32 15, i32 addrspace(5)* %i8, align 4
184  %i9 = and i32 %idx, 15
185  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
186  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
187  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
188  ret void
189}
190
191define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
192; GFX9-LABEL: private_ptr_foo:
193; GFX9:       ; %bb.0:
194; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
195; GFX9-NEXT:    v_mov_b32_e32 v1, 0x41200000
196; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:4
197; GFX9-NEXT:    s_waitcnt vmcnt(0)
198; GFX9-NEXT:    s_setpc_b64 s[30:31]
199;
200; GFX10-LABEL: private_ptr_foo:
201; GFX10:       ; %bb.0:
202; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
203; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
204; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
205; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
206; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
207; GFX10-NEXT:    s_setpc_b64 s[30:31]
208;
209; GFX940-LABEL: private_ptr_foo:
210; GFX940:       ; %bb.0:
211; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
212; GFX940-NEXT:    v_mov_b32_e32 v1, 0x41200000
213; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4
214; GFX940-NEXT:    s_waitcnt vmcnt(0)
215; GFX940-NEXT:    s_setpc_b64 s[30:31]
216  %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
217  store float 1.000000e+01, float addrspace(5)* %gep, align 4
218  ret void
219}
220
221define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
222; GFX9-LABEL: store_load_sindex_small_offset_kernel:
223; GFX9:       ; %bb.0: ; %bb
224; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
225; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
226; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
227; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
228; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
229; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
230; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
231; GFX9-NEXT:    s_and_b32 s0, s0, 15
232; GFX9-NEXT:    v_mov_b32_e32 v0, 15
233; GFX9-NEXT:    s_addk_i32 s1, 0x104
234; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
235; GFX9-NEXT:    scratch_store_dword off, v0, s1
236; GFX9-NEXT:    s_waitcnt vmcnt(0)
237; GFX9-NEXT:    s_addk_i32 s0, 0x104
238; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
239; GFX9-NEXT:    s_waitcnt vmcnt(0)
240; GFX9-NEXT:    s_endpgm
241;
242; GFX10-LABEL: store_load_sindex_small_offset_kernel:
243; GFX10:       ; %bb.0: ; %bb
244; GFX10-NEXT:    s_add_u32 s2, s2, s5
245; GFX10-NEXT:    s_addc_u32 s3, s3, 0
246; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
247; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
248; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
249; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
250; GFX10-NEXT:    s_waitcnt vmcnt(0)
251; GFX10-NEXT:    v_mov_b32_e32 v0, 15
252; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
253; GFX10-NEXT:    s_and_b32 s1, s0, 15
254; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
255; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
256; GFX10-NEXT:    s_addk_i32 s0, 0x104
257; GFX10-NEXT:    s_addk_i32 s1, 0x104
258; GFX10-NEXT:    scratch_store_dword off, v0, s0
259; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
260; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
261; GFX10-NEXT:    s_waitcnt vmcnt(0)
262; GFX10-NEXT:    s_endpgm
263;
264; GFX940-LABEL: store_load_sindex_small_offset_kernel:
265; GFX940:       ; %bb.0: ; %bb
266; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
267; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
268; GFX940-NEXT:    s_waitcnt vmcnt(0)
269; GFX940-NEXT:    v_mov_b32_e32 v0, 15
270; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
271; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
272; GFX940-NEXT:    s_and_b32 s0, s0, 15
273; GFX940-NEXT:    v_mov_b32_e32 v1, s1
274; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
275; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:260 sc0 sc1
276; GFX940-NEXT:    s_waitcnt vmcnt(0)
277; GFX940-NEXT:    v_mov_b32_e32 v0, s0
278; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:260 sc0 sc1
279; GFX940-NEXT:    s_waitcnt vmcnt(0)
280; GFX940-NEXT:    s_endpgm
281bb:
282  %padding = alloca [64 x i32], align 4, addrspace(5)
283  %i = alloca [32 x float], align 4, addrspace(5)
284  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
285  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
286  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
287  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
288  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
289  store volatile i32 15, i32 addrspace(5)* %i8, align 4
290  %i9 = and i32 %idx, 15
291  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
292  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
293  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
294  ret void
295}
296
297define amdgpu_kernel void @store_load_vindex_small_offset_kernel() {
298; GFX9-LABEL: store_load_vindex_small_offset_kernel:
299; GFX9:       ; %bb.0: ; %bb
300; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
301; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
302; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
303; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
304; GFX9-NEXT:    s_waitcnt vmcnt(0)
305; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
306; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
307; GFX9-NEXT:    v_add_u32_e32 v1, 0x104, v1
308; GFX9-NEXT:    v_mov_b32_e32 v2, 15
309; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
310; GFX9-NEXT:    scratch_store_dword v1, v2, off
311; GFX9-NEXT:    s_waitcnt vmcnt(0)
312; GFX9-NEXT:    v_add_u32_e32 v0, 0x104, v0
313; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
314; GFX9-NEXT:    s_waitcnt vmcnt(0)
315; GFX9-NEXT:    s_endpgm
316;
317; GFX10-LABEL: store_load_vindex_small_offset_kernel:
318; GFX10:       ; %bb.0: ; %bb
319; GFX10-NEXT:    s_add_u32 s0, s0, s3
320; GFX10-NEXT:    s_addc_u32 s1, s1, 0
321; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
322; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
323; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
324; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
325; GFX10-NEXT:    v_mov_b32_e32 v2, 15
326; GFX10-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
327; GFX10-NEXT:    s_waitcnt vmcnt(0)
328; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
329; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x104, v0
330; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x104, v1
331; GFX10-NEXT:    scratch_store_dword v0, v2, off
332; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
333; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
334; GFX10-NEXT:    s_waitcnt vmcnt(0)
335; GFX10-NEXT:    s_endpgm
336;
337; GFX940-LABEL: store_load_vindex_small_offset_kernel:
338; GFX940:       ; %bb.0: ; %bb
339; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
340; GFX940-NEXT:    s_waitcnt vmcnt(0)
341; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
342; GFX940-NEXT:    v_mov_b32_e32 v2, 15
343; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
344; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:260 sc0 sc1
345; GFX940-NEXT:    s_waitcnt vmcnt(0)
346; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
347; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:384 sc0 sc1
348; GFX940-NEXT:    s_waitcnt vmcnt(0)
349; GFX940-NEXT:    s_endpgm
350bb:
351  %padding = alloca [64 x i32], align 4, addrspace(5)
352  %i = alloca [32 x float], align 4, addrspace(5)
353  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
354  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
355  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
356  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
357  %i3 = zext i32 %i2 to i64
358  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
359  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
360  store volatile i32 15, i32 addrspace(5)* %i8, align 4
361  %i9 = sub nsw i32 31, %i2
362  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
363  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
364  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
365  ret void
366}
367
368define void @store_load_vindex_small_offset_foo(i32 %idx) {
369; GFX9-LABEL: store_load_vindex_small_offset_foo:
370; GFX9:       ; %bb.0: ; %bb
371; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
372; GFX9-NEXT:    scratch_load_dword v1, off, s32 glc
373; GFX9-NEXT:    s_waitcnt vmcnt(0)
374; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
375; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x100
376; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
377; GFX9-NEXT:    v_add_u32_e32 v1, vcc_hi, v1
378; GFX9-NEXT:    v_mov_b32_e32 v2, 15
379; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
380; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x100
381; GFX9-NEXT:    scratch_store_dword v1, v2, off
382; GFX9-NEXT:    s_waitcnt vmcnt(0)
383; GFX9-NEXT:    v_add_u32_e32 v0, vcc_hi, v0
384; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
385; GFX9-NEXT:    s_waitcnt vmcnt(0)
386; GFX9-NEXT:    s_setpc_b64 s[30:31]
387;
388; GFX10-LABEL: store_load_vindex_small_offset_foo:
389; GFX10:       ; %bb.0: ; %bb
390; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
391; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
392; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
393; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
394; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x100
395; GFX10-NEXT:    v_mov_b32_e32 v2, 15
396; GFX10-NEXT:    scratch_load_dword v3, off, s32 glc dlc
397; GFX10-NEXT:    s_waitcnt vmcnt(0)
398; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
399; GFX10-NEXT:    v_add_nc_u32_e32 v0, vcc_lo, v0
400; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x100
401; GFX10-NEXT:    v_add_nc_u32_e32 v1, vcc_lo, v1
402; GFX10-NEXT:    scratch_store_dword v0, v2, off
403; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
404; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
405; GFX10-NEXT:    s_waitcnt vmcnt(0)
406; GFX10-NEXT:    s_setpc_b64 s[30:31]
407;
408; GFX940-LABEL: store_load_vindex_small_offset_foo:
409; GFX940:       ; %bb.0: ; %bb
410; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
411; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
412; GFX940-NEXT:    s_waitcnt vmcnt(0)
413; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
414; GFX940-NEXT:    v_mov_b32_e32 v2, 15
415; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
416; GFX940-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
417; GFX940-NEXT:    s_waitcnt vmcnt(0)
418; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
419; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
420; GFX940-NEXT:    s_waitcnt vmcnt(0)
421; GFX940-NEXT:    s_setpc_b64 s[30:31]
422bb:
423  %padding = alloca [64 x i32], align 4, addrspace(5)
424  %i = alloca [32 x float], align 4, addrspace(5)
425  %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
426  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
427  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
428  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
429  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
430  store volatile i32 15, i32 addrspace(5)* %i8, align 4
431  %i9 = and i32 %idx, 15
432  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
433  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
434  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
435  ret void
436}
437
438define amdgpu_kernel void @store_load_sindex_large_offset_kernel(i32 %idx) {
439; GFX9-LABEL: store_load_sindex_large_offset_kernel:
440; GFX9:       ; %bb.0: ; %bb
441; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
442; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
443; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
444; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
445; GFX9-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4 glc
446; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
447; GFX9-NEXT:    s_lshl_b32 s1, s0, 2
448; GFX9-NEXT:    s_and_b32 s0, s0, 15
449; GFX9-NEXT:    v_mov_b32_e32 v0, 15
450; GFX9-NEXT:    s_addk_i32 s1, 0x4004
451; GFX9-NEXT:    s_lshl_b32 s0, s0, 2
452; GFX9-NEXT:    scratch_store_dword off, v0, s1
453; GFX9-NEXT:    s_waitcnt vmcnt(0)
454; GFX9-NEXT:    s_addk_i32 s0, 0x4004
455; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
456; GFX9-NEXT:    s_waitcnt vmcnt(0)
457; GFX9-NEXT:    s_endpgm
458;
459; GFX10-LABEL: store_load_sindex_large_offset_kernel:
460; GFX10:       ; %bb.0: ; %bb
461; GFX10-NEXT:    s_add_u32 s2, s2, s5
462; GFX10-NEXT:    s_addc_u32 s3, s3, 0
463; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
464; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
465; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
466; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4 glc dlc
467; GFX10-NEXT:    s_waitcnt vmcnt(0)
468; GFX10-NEXT:    v_mov_b32_e32 v0, 15
469; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX10-NEXT:    s_and_b32 s1, s0, 15
471; GFX10-NEXT:    s_lshl_b32 s0, s0, 2
472; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
473; GFX10-NEXT:    s_addk_i32 s0, 0x4004
474; GFX10-NEXT:    s_addk_i32 s1, 0x4004
475; GFX10-NEXT:    scratch_store_dword off, v0, s0
476; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
477; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
478; GFX10-NEXT:    s_waitcnt vmcnt(0)
479; GFX10-NEXT:    s_endpgm
480;
481; GFX940-LABEL: store_load_sindex_large_offset_kernel:
482; GFX940:       ; %bb.0: ; %bb
483; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
484; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
485; GFX940-NEXT:    s_waitcnt vmcnt(0)
486; GFX940-NEXT:    v_mov_b32_e32 v0, 15
487; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
488; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
489; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
490; GFX940-NEXT:    s_and_b32 s0, s0, 15
491; GFX940-NEXT:    v_mov_b32_e32 v1, s1
492; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
493; GFX940-NEXT:    scratch_store_dword v1, v0, vcc_hi sc0 sc1
494; GFX940-NEXT:    s_waitcnt vmcnt(0)
495; GFX940-NEXT:    v_mov_b32_e32 v0, s0
496; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
497; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
498; GFX940-NEXT:    s_waitcnt vmcnt(0)
499; GFX940-NEXT:    s_endpgm
500bb:
501  %padding = alloca [4096 x i32], align 4, addrspace(5)
502  %i = alloca [32 x float], align 4, addrspace(5)
503  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
504  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
505  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
506  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
507  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
508  store volatile i32 15, i32 addrspace(5)* %i8, align 4
509  %i9 = and i32 %idx, 15
510  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
511  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
512  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
513  ret void
514}
515
516define amdgpu_kernel void @store_load_vindex_large_offset_kernel() {
517; GFX9-LABEL: store_load_vindex_large_offset_kernel:
518; GFX9:       ; %bb.0: ; %bb
519; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
520; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
521; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
522; GFX9-NEXT:    scratch_load_dword v1, off, vcc_hi offset:4 glc
523; GFX9-NEXT:    s_waitcnt vmcnt(0)
524; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
525; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
526; GFX9-NEXT:    v_add_u32_e32 v1, 0x4004, v1
527; GFX9-NEXT:    v_mov_b32_e32 v2, 15
528; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
529; GFX9-NEXT:    scratch_store_dword v1, v2, off
530; GFX9-NEXT:    s_waitcnt vmcnt(0)
531; GFX9-NEXT:    v_add_u32_e32 v0, 0x4004, v0
532; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
533; GFX9-NEXT:    s_waitcnt vmcnt(0)
534; GFX9-NEXT:    s_endpgm
535;
536; GFX10-LABEL: store_load_vindex_large_offset_kernel:
537; GFX10:       ; %bb.0: ; %bb
538; GFX10-NEXT:    s_add_u32 s0, s0, s3
539; GFX10-NEXT:    s_addc_u32 s1, s1, 0
540; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
541; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
542; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0, v0
543; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
544; GFX10-NEXT:    v_mov_b32_e32 v2, 15
545; GFX10-NEXT:    scratch_load_dword v3, off, off offset:4 glc dlc
546; GFX10-NEXT:    s_waitcnt vmcnt(0)
547; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
548; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x4004, v0
549; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v1
550; GFX10-NEXT:    scratch_store_dword v0, v2, off
551; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
552; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
553; GFX10-NEXT:    s_waitcnt vmcnt(0)
554; GFX10-NEXT:    s_endpgm
555;
556; GFX940-LABEL: store_load_vindex_large_offset_kernel:
557; GFX940:       ; %bb.0: ; %bb
558; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
559; GFX940-NEXT:    s_waitcnt vmcnt(0)
560; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
561; GFX940-NEXT:    v_mov_b32_e32 v2, 15
562; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
563; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
564; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
565; GFX940-NEXT:    s_waitcnt vmcnt(0)
566; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
567; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
568; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi offset:124 sc0 sc1
569; GFX940-NEXT:    s_waitcnt vmcnt(0)
570; GFX940-NEXT:    s_endpgm
571bb:
572  %padding = alloca [4096 x i32], align 4, addrspace(5)
573  %i = alloca [32 x float], align 4, addrspace(5)
574  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
575  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
576  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
577  %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
578  %i3 = zext i32 %i2 to i64
579  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
580  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
581  store volatile i32 15, i32 addrspace(5)* %i8, align 4
582  %i9 = sub nsw i32 31, %i2
583  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
584  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
585  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
586  ret void
587}
588
589define void @store_load_vindex_large_offset_foo(i32 %idx) {
590; GFX9-LABEL: store_load_vindex_large_offset_foo:
591; GFX9:       ; %bb.0: ; %bb
592; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
593; GFX9-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
594; GFX9-NEXT:    s_waitcnt vmcnt(0)
595; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
596; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
597; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
598; GFX9-NEXT:    v_add_u32_e32 v1, vcc_hi, v1
599; GFX9-NEXT:    v_mov_b32_e32 v2, 15
600; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
601; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
602; GFX9-NEXT:    scratch_store_dword v1, v2, off
603; GFX9-NEXT:    s_waitcnt vmcnt(0)
604; GFX9-NEXT:    v_add_u32_e32 v0, vcc_hi, v0
605; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
606; GFX9-NEXT:    s_waitcnt vmcnt(0)
607; GFX9-NEXT:    s_setpc_b64 s[30:31]
608;
609; GFX10-LABEL: store_load_vindex_large_offset_foo:
610; GFX10:       ; %bb.0: ; %bb
611; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
612; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
613; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
614; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
615; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
616; GFX10-NEXT:    v_mov_b32_e32 v2, 15
617; GFX10-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
618; GFX10-NEXT:    s_waitcnt vmcnt(0)
619; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
620; GFX10-NEXT:    v_add_nc_u32_e32 v0, vcc_lo, v0
621; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
622; GFX10-NEXT:    v_add_nc_u32_e32 v1, vcc_lo, v1
623; GFX10-NEXT:    scratch_store_dword v0, v2, off
624; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
625; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
626; GFX10-NEXT:    s_waitcnt vmcnt(0)
627; GFX10-NEXT:    s_setpc_b64 s[30:31]
628;
629; GFX940-LABEL: store_load_vindex_large_offset_foo:
630; GFX940:       ; %bb.0: ; %bb
631; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
632; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
633; GFX940-NEXT:    s_waitcnt vmcnt(0)
634; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
635; GFX940-NEXT:    v_mov_b32_e32 v2, 15
636; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
637; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
638; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
639; GFX940-NEXT:    s_waitcnt vmcnt(0)
640; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
641; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
642; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
643; GFX940-NEXT:    s_waitcnt vmcnt(0)
644; GFX940-NEXT:    s_setpc_b64 s[30:31]
645bb:
646  %padding = alloca [4096 x i32], align 4, addrspace(5)
647  %i = alloca [32 x float], align 4, addrspace(5)
648  %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
649  %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
650  %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
651  %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
652  %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
653  store volatile i32 15, i32 addrspace(5)* %i8, align 4
654  %i9 = and i32 %idx, 15
655  %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9
656  %i11 = bitcast float addrspace(5)* %i10 to i32 addrspace(5)*
657  %i12 = load volatile i32, i32 addrspace(5)* %i11, align 4
658  ret void
659}
660
661define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
662; GFX9-LABEL: store_load_large_imm_offset_kernel:
663; GFX9:       ; %bb.0: ; %bb
664; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
665; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s1, 0
666; GFX9-NEXT:    v_mov_b32_e32 v0, 13
667; GFX9-NEXT:    s_mov_b32 vcc_hi, 0
668; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
669; GFX9-NEXT:    scratch_store_dword off, v0, vcc_hi offset:4
670; GFX9-NEXT:    s_waitcnt vmcnt(0)
671; GFX9-NEXT:    v_mov_b32_e32 v0, 15
672; GFX9-NEXT:    s_add_i32 s0, s0, 4
673; GFX9-NEXT:    scratch_store_dword off, v0, s0
674; GFX9-NEXT:    s_waitcnt vmcnt(0)
675; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
676; GFX9-NEXT:    s_waitcnt vmcnt(0)
677; GFX9-NEXT:    s_endpgm
678;
679; GFX10-LABEL: store_load_large_imm_offset_kernel:
680; GFX10:       ; %bb.0: ; %bb
681; GFX10-NEXT:    s_add_u32 s0, s0, s3
682; GFX10-NEXT:    s_addc_u32 s1, s1, 0
683; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
684; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
685; GFX10-NEXT:    v_mov_b32_e32 v0, 13
686; GFX10-NEXT:    v_mov_b32_e32 v1, 15
687; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
688; GFX10-NEXT:    s_add_i32 s0, s0, 4
689; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
690; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
691; GFX10-NEXT:    scratch_store_dword off, v1, s0
692; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
693; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
694; GFX10-NEXT:    s_waitcnt vmcnt(0)
695; GFX10-NEXT:    s_endpgm
696;
697; GFX940-LABEL: store_load_large_imm_offset_kernel:
698; GFX940:       ; %bb.0: ; %bb
699; GFX940-NEXT:    v_mov_b32_e32 v0, 13
700; GFX940-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
701; GFX940-NEXT:    s_waitcnt vmcnt(0)
702; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
703; GFX940-NEXT:    v_mov_b32_e32 v1, 15
704; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
705; GFX940-NEXT:    s_waitcnt vmcnt(0)
706; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
707; GFX940-NEXT:    s_waitcnt vmcnt(0)
708; GFX940-NEXT:    s_endpgm
709bb:
710  %i = alloca [4096 x i32], align 4, addrspace(5)
711  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
712  store volatile i32 13, i32 addrspace(5)* %i1, align 4
713  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
714  store volatile i32 15, i32 addrspace(5)* %i7, align 4
715  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
716  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
717  ret void
718}
719
720define void @store_load_large_imm_offset_foo() {
721; GFX9-LABEL: store_load_large_imm_offset_foo:
722; GFX9:       ; %bb.0: ; %bb
723; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
724; GFX9-NEXT:    v_mov_b32_e32 v0, 13
725; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
726; GFX9-NEXT:    s_add_i32 vcc_hi, s32, 4
727; GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
728; GFX9-NEXT:    s_waitcnt vmcnt(0)
729; GFX9-NEXT:    v_mov_b32_e32 v0, 15
730; GFX9-NEXT:    s_add_i32 s0, s0, vcc_hi
731; GFX9-NEXT:    scratch_store_dword off, v0, s0
732; GFX9-NEXT:    s_waitcnt vmcnt(0)
733; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
734; GFX9-NEXT:    s_waitcnt vmcnt(0)
735; GFX9-NEXT:    s_setpc_b64 s[30:31]
736;
737; GFX10-LABEL: store_load_large_imm_offset_foo:
738; GFX10:       ; %bb.0: ; %bb
739; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
740; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
741; GFX10-NEXT:    v_mov_b32_e32 v0, 13
742; GFX10-NEXT:    v_mov_b32_e32 v1, 15
743; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
744; GFX10-NEXT:    s_add_i32 vcc_lo, s32, 4
745; GFX10-NEXT:    s_add_i32 s0, s0, vcc_lo
746; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
747; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
748; GFX10-NEXT:    scratch_store_dword off, v1, s0
749; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
750; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
751; GFX10-NEXT:    s_waitcnt vmcnt(0)
752; GFX10-NEXT:    s_setpc_b64 s[30:31]
753;
754; GFX940-LABEL: store_load_large_imm_offset_foo:
755; GFX940:       ; %bb.0: ; %bb
756; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
757; GFX940-NEXT:    v_mov_b32_e32 v0, 13
758; GFX940-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
759; GFX940-NEXT:    s_waitcnt vmcnt(0)
760; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
761; GFX940-NEXT:    v_mov_b32_e32 v1, 15
762; GFX940-NEXT:    scratch_store_dword v0, v1, s32 offset:4 sc0 sc1
763; GFX940-NEXT:    s_waitcnt vmcnt(0)
764; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:4 sc0 sc1
765; GFX940-NEXT:    s_waitcnt vmcnt(0)
766; GFX940-NEXT:    s_setpc_b64 s[30:31]
767bb:
768  %i = alloca [4096 x i32], align 4, addrspace(5)
769  %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
770  store volatile i32 13, i32 addrspace(5)* %i1, align 4
771  %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
772  store volatile i32 15, i32 addrspace(5)* %i7, align 4
773  %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
774  %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
775  ret void
776}
777
778define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
779; GFX9-LABEL: store_load_vidx_sidx_offset:
780; GFX9:       ; %bb.0: ; %bb
781; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
782; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
783; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
784; GFX9-NEXT:    v_mov_b32_e32 v1, 15
785; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
786; GFX9-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
787; GFX9-NEXT:    v_add_u32_e32 v0, 4, v0
788; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
789; GFX9-NEXT:    s_waitcnt vmcnt(0)
790; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
791; GFX9-NEXT:    s_waitcnt vmcnt(0)
792; GFX9-NEXT:    s_endpgm
793;
794; GFX10-LABEL: store_load_vidx_sidx_offset:
795; GFX10:       ; %bb.0: ; %bb
796; GFX10-NEXT:    s_add_u32 s2, s2, s5
797; GFX10-NEXT:    s_addc_u32 s3, s3, 0
798; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
799; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
800; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
801; GFX10-NEXT:    v_mov_b32_e32 v1, 15
802; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
803; GFX10-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
804; GFX10-NEXT:    v_add_nc_u32_e32 v0, 4, v0
805; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
806; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
807; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
808; GFX10-NEXT:    s_waitcnt vmcnt(0)
809; GFX10-NEXT:    s_endpgm
810;
811; GFX940-LABEL: store_load_vidx_sidx_offset:
812; GFX940:       ; %bb.0: ; %bb
813; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
814; GFX940-NEXT:    v_mov_b32_e32 v1, 15
815; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
816; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
817; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
818; GFX940-NEXT:    s_waitcnt vmcnt(0)
819; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
820; GFX940-NEXT:    s_waitcnt vmcnt(0)
821; GFX940-NEXT:    s_endpgm
822bb:
823  %alloca = alloca [32 x i32], align 4, addrspace(5)
824  %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
825  %add1 = add nsw i32 %sidx, %vidx
826  %add2 = add nsw i32 %add1, 256
827  %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
828  store volatile i32 15, i32 addrspace(5)* %gep, align 4
829  %load = load volatile i32, i32 addrspace(5)* %gep, align 4
830  ret void
831}
832
833define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
834; GFX9-LABEL: store_load_i64_aligned:
835; GFX9:       ; %bb.0: ; %bb
836; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
837; GFX9-NEXT:    v_mov_b32_e32 v1, 15
838; GFX9-NEXT:    v_mov_b32_e32 v2, 0
839; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
840; GFX9-NEXT:    s_waitcnt vmcnt(0)
841; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
842; GFX9-NEXT:    s_waitcnt vmcnt(0)
843; GFX9-NEXT:    s_setpc_b64 s[30:31]
844;
845; GFX10-LABEL: store_load_i64_aligned:
846; GFX10:       ; %bb.0: ; %bb
847; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
848; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
849; GFX10-NEXT:    v_mov_b32_e32 v1, 15
850; GFX10-NEXT:    v_mov_b32_e32 v2, 0
851; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
852; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
853; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
854; GFX10-NEXT:    s_waitcnt vmcnt(0)
855; GFX10-NEXT:    s_setpc_b64 s[30:31]
856;
857; GFX940-LABEL: store_load_i64_aligned:
858; GFX940:       ; %bb.0: ; %bb
859; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
860; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
861; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
862; GFX940-NEXT:    s_waitcnt vmcnt(0)
863; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
864; GFX940-NEXT:    s_waitcnt vmcnt(0)
865; GFX940-NEXT:    s_setpc_b64 s[30:31]
866bb:
867  store volatile i64 15, i64 addrspace(5)* %arg, align 8
868  %load = load volatile i64, i64 addrspace(5)* %arg, align 8
869  ret void
870}
871
872define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
873; GFX9-LABEL: store_load_i64_unaligned:
874; GFX9:       ; %bb.0: ; %bb
875; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
876; GFX9-NEXT:    v_mov_b32_e32 v1, 15
877; GFX9-NEXT:    v_mov_b32_e32 v2, 0
878; GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
879; GFX9-NEXT:    s_waitcnt vmcnt(0)
880; GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
881; GFX9-NEXT:    s_waitcnt vmcnt(0)
882; GFX9-NEXT:    s_setpc_b64 s[30:31]
883;
884; GFX10-LABEL: store_load_i64_unaligned:
885; GFX10:       ; %bb.0: ; %bb
886; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
887; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
888; GFX10-NEXT:    v_mov_b32_e32 v1, 15
889; GFX10-NEXT:    v_mov_b32_e32 v2, 0
890; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
891; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
892; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
893; GFX10-NEXT:    s_waitcnt vmcnt(0)
894; GFX10-NEXT:    s_setpc_b64 s[30:31]
895;
896; GFX940-LABEL: store_load_i64_unaligned:
897; GFX940:       ; %bb.0: ; %bb
898; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
899; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
900; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
901; GFX940-NEXT:    s_waitcnt vmcnt(0)
902; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
903; GFX940-NEXT:    s_waitcnt vmcnt(0)
904; GFX940-NEXT:    s_setpc_b64 s[30:31]
905bb:
906  store volatile i64 15, i64 addrspace(5)* %arg, align 1
907  %load = load volatile i64, i64 addrspace(5)* %arg, align 1
908  ret void
909}
910
911define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
912; GFX9-LABEL: store_load_v3i32_unaligned:
913; GFX9:       ; %bb.0: ; %bb
914; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
915; GFX9-NEXT:    s_mov_b32 s2, 3
916; GFX9-NEXT:    s_mov_b32 s1, 2
917; GFX9-NEXT:    s_mov_b32 s0, 1
918; GFX9-NEXT:    v_mov_b32_e32 v3, s2
919; GFX9-NEXT:    v_mov_b32_e32 v2, s1
920; GFX9-NEXT:    v_mov_b32_e32 v1, s0
921; GFX9-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
922; GFX9-NEXT:    s_waitcnt vmcnt(0)
923; GFX9-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
924; GFX9-NEXT:    s_waitcnt vmcnt(0)
925; GFX9-NEXT:    s_setpc_b64 s[30:31]
926;
927; GFX10-LABEL: store_load_v3i32_unaligned:
928; GFX10:       ; %bb.0: ; %bb
929; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
930; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
931; GFX10-NEXT:    s_mov_b32 s2, 3
932; GFX10-NEXT:    s_mov_b32 s1, 2
933; GFX10-NEXT:    s_mov_b32 s0, 1
934; GFX10-NEXT:    v_mov_b32_e32 v3, s2
935; GFX10-NEXT:    v_mov_b32_e32 v2, s1
936; GFX10-NEXT:    v_mov_b32_e32 v1, s0
937; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
938; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
939; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
940; GFX10-NEXT:    s_waitcnt vmcnt(0)
941; GFX10-NEXT:    s_setpc_b64 s[30:31]
942;
943; GFX940-LABEL: store_load_v3i32_unaligned:
944; GFX940:       ; %bb.0: ; %bb
945; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
946; GFX940-NEXT:    s_mov_b32 s2, 3
947; GFX940-NEXT:    s_mov_b32 s1, 2
948; GFX940-NEXT:    s_mov_b32 s0, 1
949; GFX940-NEXT:    v_mov_b32_e32 v4, s2
950; GFX940-NEXT:    v_mov_b32_e32 v3, s1
951; GFX940-NEXT:    v_mov_b32_e32 v2, s0
952; GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
953; GFX940-NEXT:    s_waitcnt vmcnt(0)
954; GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
955; GFX940-NEXT:    s_waitcnt vmcnt(0)
956; GFX940-NEXT:    s_setpc_b64 s[30:31]
957bb:
958  store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
959  %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
960  ret void
961}
962
963define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
964; GFX9-LABEL: store_load_v4i32_unaligned:
965; GFX9:       ; %bb.0: ; %bb
966; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
967; GFX9-NEXT:    s_mov_b32 s3, 4
968; GFX9-NEXT:    s_mov_b32 s2, 3
969; GFX9-NEXT:    s_mov_b32 s1, 2
970; GFX9-NEXT:    s_mov_b32 s0, 1
971; GFX9-NEXT:    v_mov_b32_e32 v4, s3
972; GFX9-NEXT:    v_mov_b32_e32 v3, s2
973; GFX9-NEXT:    v_mov_b32_e32 v2, s1
974; GFX9-NEXT:    v_mov_b32_e32 v1, s0
975; GFX9-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
976; GFX9-NEXT:    s_waitcnt vmcnt(0)
977; GFX9-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
978; GFX9-NEXT:    s_waitcnt vmcnt(0)
979; GFX9-NEXT:    s_setpc_b64 s[30:31]
980;
981; GFX10-LABEL: store_load_v4i32_unaligned:
982; GFX10:       ; %bb.0: ; %bb
983; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
984; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
985; GFX10-NEXT:    s_mov_b32 s3, 4
986; GFX10-NEXT:    s_mov_b32 s2, 3
987; GFX10-NEXT:    s_mov_b32 s1, 2
988; GFX10-NEXT:    s_mov_b32 s0, 1
989; GFX10-NEXT:    v_mov_b32_e32 v4, s3
990; GFX10-NEXT:    v_mov_b32_e32 v3, s2
991; GFX10-NEXT:    v_mov_b32_e32 v2, s1
992; GFX10-NEXT:    v_mov_b32_e32 v1, s0
993; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
994; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
995; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
996; GFX10-NEXT:    s_waitcnt vmcnt(0)
997; GFX10-NEXT:    s_setpc_b64 s[30:31]
998;
999; GFX940-LABEL: store_load_v4i32_unaligned:
1000; GFX940:       ; %bb.0: ; %bb
1001; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1002; GFX940-NEXT:    s_mov_b32 s3, 4
1003; GFX940-NEXT:    s_mov_b32 s2, 3
1004; GFX940-NEXT:    s_mov_b32 s1, 2
1005; GFX940-NEXT:    s_mov_b32 s0, 1
1006; GFX940-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
1007; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
1008; GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
1009; GFX940-NEXT:    s_waitcnt vmcnt(0)
1010; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
1011; GFX940-NEXT:    s_waitcnt vmcnt(0)
1012; GFX940-NEXT:    s_setpc_b64 s[30:31]
1013bb:
1014  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
1015  %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
1016  ret void
1017}
1018
1019declare i32 @llvm.amdgcn.workitem.id.x()
1020