1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx902 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
3; RUN: llc -march=amdgcn -mcpu=gfx1030 -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-SCRATCH %s
4
5define amdgpu_kernel void @vector_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
6; GCN-LABEL: vector_clause:
7; GCN:       ; %bb.0: ; %bb
8; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
9; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
10; GCN-NEXT:    v_lshlrev_b32_e32 v16, 4, v0
11; GCN-NEXT:    s_waitcnt lgkmcnt(0)
12; GCN-NEXT:    global_load_dwordx4 v[0:3], v16, s[2:3]
13; GCN-NEXT:    global_load_dwordx4 v[4:7], v16, s[2:3] offset:16
14; GCN-NEXT:    global_load_dwordx4 v[8:11], v16, s[2:3] offset:32
15; GCN-NEXT:    global_load_dwordx4 v[12:15], v16, s[2:3] offset:48
16; GCN-NEXT:    s_waitcnt vmcnt(3)
17; GCN-NEXT:    global_store_dwordx4 v16, v[0:3], s[4:5]
18; GCN-NEXT:    s_waitcnt vmcnt(3)
19; GCN-NEXT:    global_store_dwordx4 v16, v[4:7], s[4:5] offset:16
20; GCN-NEXT:    s_waitcnt vmcnt(3)
21; GCN-NEXT:    global_store_dwordx4 v16, v[8:11], s[4:5] offset:32
22; GCN-NEXT:    s_waitcnt vmcnt(3)
23; GCN-NEXT:    global_store_dwordx4 v16, v[12:15], s[4:5] offset:48
24; GCN-NEXT:    s_endpgm
25;
26; GCN-SCRATCH-LABEL: vector_clause:
27; GCN-SCRATCH:       ; %bb.0: ; %bb
28; GCN-SCRATCH-NEXT:    s_add_u32 s2, s2, s5
29; GCN-SCRATCH-NEXT:    s_addc_u32 s3, s3, 0
30; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
31; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
32; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
33; GCN-SCRATCH-NEXT:    v_lshlrev_b32_e32 v16, 4, v0
34; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
35; GCN-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
36; GCN-SCRATCH-NEXT:    s_clause 0x3
37; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[0:3], v16, s[2:3]
38; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[4:7], v16, s[2:3] offset:16
39; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[8:11], v16, s[2:3] offset:32
40; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[12:15], v16, s[2:3] offset:48
41; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(3)
42; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[0:3], s[0:1]
43; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(2)
44; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
45; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
46; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
47; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
48; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
49; GCN-SCRATCH-NEXT:    s_endpgm
50bb:
51  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
52  %tmp2 = zext i32 %tmp to i64
53  %tmp3 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp2
54  %tmp4 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp3, align 16
55  %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp2
56  %tmp6 = add nuw nsw i64 %tmp2, 1
57  %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp6
58  %tmp8 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp7, align 16
59  %tmp9 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp6
60  %tmp10 = add nuw nsw i64 %tmp2, 2
61  %tmp11 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp10
62  %tmp12 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp11, align 16
63  %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp10
64  %tmp14 = add nuw nsw i64 %tmp2, 3
65  %tmp15 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp14
66  %tmp16 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp15, align 16
67  %tmp17 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp14
68  store <4 x i32> %tmp4, <4 x i32> addrspace(1)* %tmp5, align 16
69  store <4 x i32> %tmp8, <4 x i32> addrspace(1)* %tmp9, align 16
70  store <4 x i32> %tmp12, <4 x i32> addrspace(1)* %tmp13, align 16
71  store <4 x i32> %tmp16, <4 x i32> addrspace(1)* %tmp17, align 16
72  ret void
73}
74
75define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
76; GCN-LABEL: scalar_clause:
77; GCN:       ; %bb.0: ; %bb
78; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x24
79; GCN-NEXT:    s_load_dwordx2 s[18:19], s[0:1], 0x2c
80; GCN-NEXT:    v_mov_b32_e32 v12, 0
81; GCN-NEXT:    s_waitcnt lgkmcnt(0)
82; GCN-NEXT:    s_load_dwordx4 s[0:3], s[16:17], 0x0
83; GCN-NEXT:    s_load_dwordx4 s[4:7], s[16:17], 0x10
84; GCN-NEXT:    s_load_dwordx4 s[8:11], s[16:17], 0x20
85; GCN-NEXT:    s_load_dwordx4 s[12:15], s[16:17], 0x30
86; GCN-NEXT:    s_waitcnt lgkmcnt(0)
87; GCN-NEXT:    v_mov_b32_e32 v0, s0
88; GCN-NEXT:    v_mov_b32_e32 v4, s4
89; GCN-NEXT:    v_mov_b32_e32 v8, s8
90; GCN-NEXT:    v_mov_b32_e32 v1, s1
91; GCN-NEXT:    v_mov_b32_e32 v2, s2
92; GCN-NEXT:    v_mov_b32_e32 v3, s3
93; GCN-NEXT:    v_mov_b32_e32 v5, s5
94; GCN-NEXT:    v_mov_b32_e32 v6, s6
95; GCN-NEXT:    v_mov_b32_e32 v7, s7
96; GCN-NEXT:    v_mov_b32_e32 v9, s9
97; GCN-NEXT:    v_mov_b32_e32 v10, s10
98; GCN-NEXT:    v_mov_b32_e32 v11, s11
99; GCN-NEXT:    global_store_dwordx4 v12, v[0:3], s[18:19]
100; GCN-NEXT:    global_store_dwordx4 v12, v[4:7], s[18:19] offset:16
101; GCN-NEXT:    global_store_dwordx4 v12, v[8:11], s[18:19] offset:32
102; GCN-NEXT:    v_mov_b32_e32 v0, s12
103; GCN-NEXT:    v_mov_b32_e32 v1, s13
104; GCN-NEXT:    v_mov_b32_e32 v2, s14
105; GCN-NEXT:    v_mov_b32_e32 v3, s15
106; GCN-NEXT:    global_store_dwordx4 v12, v[0:3], s[18:19] offset:48
107; GCN-NEXT:    s_endpgm
108;
109; GCN-SCRATCH-LABEL: scalar_clause:
110; GCN-SCRATCH:       ; %bb.0: ; %bb
111; GCN-SCRATCH-NEXT:    s_add_u32 s2, s2, s5
112; GCN-SCRATCH-NEXT:    s_addc_u32 s3, s3, 0
113; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
114; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
115; GCN-SCRATCH-NEXT:    s_clause 0x1
116; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x24
117; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x2c
118; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v16, 0
119; GCN-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
120; GCN-SCRATCH-NEXT:    s_clause 0x3
121; GCN-SCRATCH-NEXT:    s_load_dwordx4 s[0:3], s[12:13], 0x0
122; GCN-SCRATCH-NEXT:    s_load_dwordx4 s[4:7], s[12:13], 0x10
123; GCN-SCRATCH-NEXT:    s_load_dwordx4 s[8:11], s[12:13], 0x20
124; GCN-SCRATCH-NEXT:    s_load_dwordx4 s[12:15], s[12:13], 0x30
125; GCN-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
126; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v0, s0
127; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v4, s4
128; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v1, s1
129; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v2, s2
130; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v3, s3
131; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v8, s8
132; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v5, s5
133; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v6, s6
134; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v7, s7
135; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v12, s12
136; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v9, s9
137; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v10, s10
138; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v11, s11
139; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v13, s13
140; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v14, s14
141; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v15, s15
142; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[0:3], s[16:17]
143; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
144; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
145; GCN-SCRATCH-NEXT:    global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
146; GCN-SCRATCH-NEXT:    s_endpgm
147bb:
148  %tmp = load <4 x i32>, <4 x i32> addrspace(1)* %arg, align 16
149  %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 1
150  %tmp3 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp2, align 16
151  %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 1
152  %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 2
153  %tmp6 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp5, align 16
154  %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 2
155  %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 3
156  %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16
157  %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 3
158  store <4 x i32> %tmp, <4 x i32> addrspace(1)* %arg1, align 16
159  store <4 x i32> %tmp3, <4 x i32> addrspace(1)* %tmp4, align 16
160  store <4 x i32> %tmp6, <4 x i32> addrspace(1)* %tmp7, align 16
161  store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16
162  ret void
163}
164
165define void @mubuf_clause(<4 x i32> addrspace(5)* noalias nocapture readonly %arg, <4 x i32> addrspace(5)* noalias nocapture %arg1) {
166; GCN-LABEL: mubuf_clause:
167; GCN:       ; %bb.0: ; %bb
168; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
169; GCN-NEXT:    v_and_b32_e32 v2, 0x3ff, v2
170; GCN-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
171; GCN-NEXT:    v_add_u32_e32 v0, v0, v2
172; GCN-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
173; GCN-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
174; GCN-NEXT:    buffer_load_dword v5, v0, s[0:3], 0 offen offset:8
175; GCN-NEXT:    buffer_load_dword v6, v0, s[0:3], 0 offen offset:12
176; GCN-NEXT:    buffer_load_dword v7, v0, s[0:3], 0 offen offset:16
177; GCN-NEXT:    buffer_load_dword v8, v0, s[0:3], 0 offen offset:20
178; GCN-NEXT:    buffer_load_dword v9, v0, s[0:3], 0 offen offset:24
179; GCN-NEXT:    buffer_load_dword v10, v0, s[0:3], 0 offen offset:28
180; GCN-NEXT:    buffer_load_dword v11, v0, s[0:3], 0 offen offset:32
181; GCN-NEXT:    buffer_load_dword v12, v0, s[0:3], 0 offen offset:36
182; GCN-NEXT:    buffer_load_dword v13, v0, s[0:3], 0 offen offset:40
183; GCN-NEXT:    buffer_load_dword v14, v0, s[0:3], 0 offen offset:44
184; GCN-NEXT:    buffer_load_dword v15, v0, s[0:3], 0 offen offset:48
185; GCN-NEXT:    buffer_load_dword v16, v0, s[0:3], 0 offen offset:52
186; GCN-NEXT:    buffer_load_dword v17, v0, s[0:3], 0 offen offset:56
187; GCN-NEXT:    s_nop 0
188; GCN-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:60
189; GCN-NEXT:    v_add_u32_e32 v1, v1, v2
190; GCN-NEXT:    s_waitcnt vmcnt(12)
191; GCN-NEXT:    buffer_store_dword v6, v1, s[0:3], 0 offen offset:12
192; GCN-NEXT:    buffer_store_dword v5, v1, s[0:3], 0 offen offset:8
193; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen offset:4
194; GCN-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
195; GCN-NEXT:    s_waitcnt vmcnt(12)
196; GCN-NEXT:    buffer_store_dword v10, v1, s[0:3], 0 offen offset:28
197; GCN-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen offset:24
198; GCN-NEXT:    buffer_store_dword v8, v1, s[0:3], 0 offen offset:20
199; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen offset:16
200; GCN-NEXT:    s_waitcnt vmcnt(12)
201; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen offset:44
202; GCN-NEXT:    buffer_store_dword v13, v1, s[0:3], 0 offen offset:40
203; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen offset:36
204; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen offset:32
205; GCN-NEXT:    s_waitcnt vmcnt(12)
206; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 offen offset:60
207; GCN-NEXT:    buffer_store_dword v17, v1, s[0:3], 0 offen offset:56
208; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen offset:52
209; GCN-NEXT:    buffer_store_dword v15, v1, s[0:3], 0 offen offset:48
210; GCN-NEXT:    s_waitcnt vmcnt(0)
211; GCN-NEXT:    s_setpc_b64 s[30:31]
212;
213; GCN-SCRATCH-LABEL: mubuf_clause:
214; GCN-SCRATCH:       ; %bb.0: ; %bb
215; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
216; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
217; GCN-SCRATCH-NEXT:    v_and_b32_e32 v2, 0x3ff, v2
218; GCN-SCRATCH-NEXT:    v_lshlrev_b32_e32 v18, 4, v2
219; GCN-SCRATCH-NEXT:    v_add_nc_u32_e32 v0, v0, v18
220; GCN-SCRATCH-NEXT:    s_clause 0x3
221; GCN-SCRATCH-NEXT:    scratch_load_dwordx4 v[2:5], v0, off
222; GCN-SCRATCH-NEXT:    scratch_load_dwordx4 v[6:9], v0, off offset:16
223; GCN-SCRATCH-NEXT:    scratch_load_dwordx4 v[10:13], v0, off offset:32
224; GCN-SCRATCH-NEXT:    scratch_load_dwordx4 v[14:17], v0, off offset:48
225; GCN-SCRATCH-NEXT:    v_add_nc_u32_e32 v0, v1, v18
226; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(3)
227; GCN-SCRATCH-NEXT:    scratch_store_dwordx4 v0, v[2:5], off
228; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(2)
229; GCN-SCRATCH-NEXT:    scratch_store_dwordx4 v0, v[6:9], off offset:16
230; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
231; GCN-SCRATCH-NEXT:    scratch_store_dwordx4 v0, v[10:13], off offset:32
232; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
233; GCN-SCRATCH-NEXT:    scratch_store_dwordx4 v0, v[14:17], off offset:48
234; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
235; GCN-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
236bb:
237  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
238  %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp
239  %tmp3 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp2, align 16
240  %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp
241  %tmp5 = add nuw nsw i32 %tmp, 1
242  %tmp6 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp5
243  %tmp7 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp6, align 16
244  %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp5
245  %tmp9 = add nuw nsw i32 %tmp, 2
246  %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp9
247  %tmp11 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp10, align 16
248  %tmp12 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp9
249  %tmp13 = add nuw nsw i32 %tmp, 3
250  %tmp14 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp13
251  %tmp15 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp14, align 16
252  %tmp16 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp13
253  store <4 x i32> %tmp3, <4 x i32> addrspace(5)* %tmp4, align 16
254  store <4 x i32> %tmp7, <4 x i32> addrspace(5)* %tmp8, align 16
255  store <4 x i32> %tmp11, <4 x i32> addrspace(5)* %tmp12, align 16
256  store <4 x i32> %tmp15, <4 x i32> addrspace(5)* %tmp16, align 16
257  ret void
258}
259
260define amdgpu_kernel void @vector_clause_indirect(i64 addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture readnone %arg1, <4 x i32> addrspace(1)* noalias nocapture %arg2) {
261; GCN-LABEL: vector_clause_indirect:
262; GCN:       ; %bb.0: ; %bb
263; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
264; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
265; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
266; GCN-NEXT:    s_waitcnt lgkmcnt(0)
267; GCN-NEXT:    global_load_dwordx2 v[8:9], v0, s[2:3]
268; GCN-NEXT:    s_waitcnt vmcnt(0)
269; GCN-NEXT:    global_load_dwordx4 v[0:3], v[8:9], off
270; GCN-NEXT:    global_load_dwordx4 v[4:7], v[8:9], off offset:16
271; GCN-NEXT:    v_mov_b32_e32 v8, 0
272; GCN-NEXT:    s_waitcnt vmcnt(1)
273; GCN-NEXT:    global_store_dwordx4 v8, v[0:3], s[4:5]
274; GCN-NEXT:    s_waitcnt vmcnt(1)
275; GCN-NEXT:    global_store_dwordx4 v8, v[4:7], s[4:5] offset:16
276; GCN-NEXT:    s_endpgm
277;
278; GCN-SCRATCH-LABEL: vector_clause_indirect:
279; GCN-SCRATCH:       ; %bb.0: ; %bb
280; GCN-SCRATCH-NEXT:    s_add_u32 s2, s2, s5
281; GCN-SCRATCH-NEXT:    s_addc_u32 s3, s3, 0
282; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
283; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
284; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
285; GCN-SCRATCH-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
286; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
287; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v8, 0
288; GCN-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
289; GCN-SCRATCH-NEXT:    global_load_dwordx2 v[4:5], v0, s[2:3]
290; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
291; GCN-SCRATCH-NEXT:    s_clause 0x1
292; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[0:3], v[4:5], off
293; GCN-SCRATCH-NEXT:    global_load_dwordx4 v[4:7], v[4:5], off offset:16
294; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
295; GCN-SCRATCH-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
296; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
297; GCN-SCRATCH-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
298; GCN-SCRATCH-NEXT:    s_endpgm
299bb:
300  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
301  %tmp3 = zext i32 %tmp to i64
302  %tmp4 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 %tmp3
303  %tmp5 = bitcast i64 addrspace(1)* %tmp4 to <4 x i32> addrspace(1)* addrspace(1)*
304  %tmp6 = load <4 x i32> addrspace(1)*, <4 x i32> addrspace(1)* addrspace(1)* %tmp5, align 8
305  %tmp7 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp6, align 16
306  %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %tmp6, i64 1
307  %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16
308  store <4 x i32> %tmp7, <4 x i32> addrspace(1)* %arg2, align 16
309  %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg2, i64 1
310  store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16
311  ret void
312}
313
314define void @load_global_d16_hi(i16 addrspace(1)* %in, i16 %reg, <2 x i16> addrspace(1)* %out) {
315; GCN-LABEL: load_global_d16_hi:
316; GCN:       ; %bb.0: ; %entry
317; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
318; GCN-NEXT:    v_mov_b32_e32 v5, v2
319; GCN-NEXT:    global_load_short_d16_hi v5, v[0:1], off
320; GCN-NEXT:    s_nop 0
321; GCN-NEXT:    global_load_short_d16_hi v2, v[0:1], off offset:64
322; GCN-NEXT:    s_waitcnt vmcnt(1)
323; GCN-NEXT:    global_store_dword v[3:4], v5, off
324; GCN-NEXT:    s_waitcnt vmcnt(1)
325; GCN-NEXT:    global_store_dword v[3:4], v2, off offset:128
326; GCN-NEXT:    s_waitcnt vmcnt(0)
327; GCN-NEXT:    s_setpc_b64 s[30:31]
328;
329; GCN-SCRATCH-LABEL: load_global_d16_hi:
330; GCN-SCRATCH:       ; %bb.0: ; %entry
331; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
332; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
333; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v5, v2
334; GCN-SCRATCH-NEXT:    s_clause 0x1
335; GCN-SCRATCH-NEXT:    global_load_short_d16_hi v5, v[0:1], off
336; GCN-SCRATCH-NEXT:    global_load_short_d16_hi v2, v[0:1], off offset:64
337; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
338; GCN-SCRATCH-NEXT:    global_store_dword v[3:4], v5, off
339; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
340; GCN-SCRATCH-NEXT:    global_store_dword v[3:4], v2, off offset:128
341; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
342; GCN-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
343entry:
344  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
345  %load1 = load i16, i16 addrspace(1)* %in
346  %load2 = load i16, i16 addrspace(1)* %gep
347  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
348  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
349  store <2 x i16> %build1, <2 x i16> addrspace(1)* %out
350  %build2 = insertelement <2 x i16> undef, i16 %reg, i32 0
351  %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1
352  %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32
353  store <2 x i16> %build3, <2 x i16> addrspace(1)* %gep2
354  ret void
355}
356
357define void @load_global_d16_lo(i16 addrspace(1)* %in, i32 %reg, <2 x i16> addrspace(1)* %out) {
358; GCN-LABEL: load_global_d16_lo:
359; GCN:       ; %bb.0: ; %entry
360; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
361; GCN-NEXT:    v_mov_b32_e32 v5, v2
362; GCN-NEXT:    global_load_short_d16 v5, v[0:1], off
363; GCN-NEXT:    s_nop 0
364; GCN-NEXT:    global_load_short_d16 v2, v[0:1], off offset:64
365; GCN-NEXT:    s_waitcnt vmcnt(1)
366; GCN-NEXT:    global_store_dword v[3:4], v5, off
367; GCN-NEXT:    s_waitcnt vmcnt(1)
368; GCN-NEXT:    global_store_dword v[3:4], v2, off offset:128
369; GCN-NEXT:    s_waitcnt vmcnt(0)
370; GCN-NEXT:    s_setpc_b64 s[30:31]
371;
372; GCN-SCRATCH-LABEL: load_global_d16_lo:
373; GCN-SCRATCH:       ; %bb.0: ; %entry
374; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
375; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
376; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v5, v2
377; GCN-SCRATCH-NEXT:    s_clause 0x1
378; GCN-SCRATCH-NEXT:    global_load_short_d16 v5, v[0:1], off
379; GCN-SCRATCH-NEXT:    global_load_short_d16 v2, v[0:1], off offset:64
380; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
381; GCN-SCRATCH-NEXT:    global_store_dword v[3:4], v5, off
382; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
383; GCN-SCRATCH-NEXT:    global_store_dword v[3:4], v2, off offset:128
384; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
385; GCN-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
386entry:
387  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
388  %reg.bc1 = bitcast i32 %reg to <2 x i16>
389  %reg.bc2 = bitcast i32 %reg to <2 x i16>
390  %load1 = load i16, i16 addrspace(1)* %in
391  %load2 = load i16, i16 addrspace(1)* %gep
392  %build1 = insertelement <2 x i16> %reg.bc1, i16 %load1, i32 0
393  %build2 = insertelement <2 x i16> %reg.bc2, i16 %load2, i32 0
394  %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32
395  store <2 x i16> %build1, <2 x i16> addrspace(1)* %out
396  store <2 x i16> %build2, <2 x i16> addrspace(1)* %gep2
397  ret void
398}
399
400define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc) {
401; GCN-LABEL: flat_scratch_load:
402; GCN:       ; %bb.0: ; %.entry
403; GCN-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
404; GCN-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
405; GCN-NEXT:    s_mov_b32 s18, -1
406; GCN-NEXT:    s_mov_b32 s19, 0xe00000
407; GCN-NEXT:    s_add_u32 s16, s16, s3
408; GCN-NEXT:    s_addc_u32 s17, s17, 0
409; GCN-NEXT:    s_mov_b64 s[12:13], exec
410; GCN-NEXT:    s_wqm_b64 exec, exec
411; GCN-NEXT:    v_mov_b32_e32 v0, 0x40b00000
412; GCN-NEXT:    s_load_dwordx2 s[14:15], s[0:1], 0x24
413; GCN-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x44
414; GCN-NEXT:    buffer_store_dword v0, off, s[16:19], 0 offset:4
415; GCN-NEXT:    s_waitcnt vmcnt(0)
416; GCN-NEXT:    ;;#ASMSTART
417; GCN-NEXT:    ;;#ASMEND
418; GCN-NEXT:    buffer_load_dword v2, off, s[16:19], 0 offset:4
419; GCN-NEXT:    s_brev_b32 s0, 1
420; GCN-NEXT:    s_waitcnt lgkmcnt(0)
421; GCN-NEXT:    v_mov_b32_e32 v0, s14
422; GCN-NEXT:    s_mov_b32 s3, 0
423; GCN-NEXT:    s_mov_b32 s1, s0
424; GCN-NEXT:    s_mov_b32 s2, s0
425; GCN-NEXT:    v_mov_b32_e32 v1, s15
426; GCN-NEXT:    s_and_b64 exec, exec, s[12:13]
427; GCN-NEXT:    image_sample v0, v[0:1], s[4:11], s[0:3] dmask:0x1
428; GCN-NEXT:    s_waitcnt vmcnt(0)
429; GCN-NEXT:    v_add_f32_e32 v0, v2, v0
430; GCN-NEXT:    exp mrt0 v0, off, off, off done vm
431; GCN-NEXT:    s_endpgm
432;
433; GCN-SCRATCH-LABEL: flat_scratch_load:
434; GCN-SCRATCH:       ; %bb.0: ; %.entry
435; GCN-SCRATCH-NEXT:    s_add_u32 s2, s2, s5
436; GCN-SCRATCH-NEXT:    s_addc_u32 s3, s3, 0
437; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
438; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
439; GCN-SCRATCH-NEXT:    s_mov_b32 s9, exec_lo
440; GCN-SCRATCH-NEXT:    s_wqm_b32 exec_lo, exec_lo
441; GCN-SCRATCH-NEXT:    s_clause 0x1
442; GCN-SCRATCH-NEXT:    s_load_dwordx2 s[10:11], s[0:1], 0x24
443; GCN-SCRATCH-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x44
444; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x40b00000
445; GCN-SCRATCH-NEXT:    s_brev_b32 s8, 1
446; GCN-SCRATCH-NEXT:    scratch_store_dword off, v0, off offset:4
447; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
448; GCN-SCRATCH-NEXT:    ;;#ASMSTART
449; GCN-SCRATCH-NEXT:    ;;#ASMEND
450; GCN-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
451; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v0, s10
452; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v1, s11
453; GCN-SCRATCH-NEXT:    s_and_b32 exec_lo, exec_lo, s9
454; GCN-SCRATCH-NEXT:    s_mov_b32 s11, 0
455; GCN-SCRATCH-NEXT:    s_mov_b32 s9, s8
456; GCN-SCRATCH-NEXT:    s_mov_b32 s10, s8
457; GCN-SCRATCH-NEXT:    scratch_load_dword v2, off, off offset:4
458; GCN-SCRATCH-NEXT:    image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
459; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
460; GCN-SCRATCH-NEXT:    v_add_f32_e32 v0, v2, v0
461; GCN-SCRATCH-NEXT:    exp mrt0 v0, off, off, off done vm
462; GCN-SCRATCH-NEXT:    s_endpgm
463.entry:
464  %alloca = alloca float, align 4, addrspace(5)
465  store volatile float 5.5, float addrspace(5)* %alloca
466  call void asm sideeffect "", ""()
467  ; There was a bug with flat scratch instructions that do not not use any address registers (ST mode).
468  ; To trigger, the scratch_load has to be immediately before the image_sample in MIR.
469  %load = load float, float addrspace(5)* %alloca
470  %val = call <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 9, float %a, float %b, <8 x i32> %desc, <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 0>, i1 false, i32 0, i32 0)
471  %val0 = extractelement <2 x float> %val, i32 0
472  %valadd = fadd float %load, %val0
473  call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
474  ret void
475}
476
477define amdgpu_kernel void @flat_scratch_load_clause(float %a, float %b, <8 x i32> %desc) {
478; GCN-LABEL: flat_scratch_load_clause:
479; GCN:       ; %bb.0: ; %.entry
480; GCN-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
481; GCN-NEXT:    s_mov_b32 s5, SCRATCH_RSRC_DWORD1
482; GCN-NEXT:    s_mov_b32 s6, -1
483; GCN-NEXT:    s_mov_b32 s7, 0xe00000
484; GCN-NEXT:    s_add_u32 s4, s4, s3
485; GCN-NEXT:    s_addc_u32 s5, s5, 0
486; GCN-NEXT:    v_mov_b32_e32 v0, 0x40b00000
487; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0 offset:4
488; GCN-NEXT:    s_waitcnt vmcnt(0)
489; GCN-NEXT:    v_mov_b32_e32 v0, 0x40d00000
490; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0 offset:8
491; GCN-NEXT:    s_waitcnt vmcnt(0)
492; GCN-NEXT:    ;;#ASMSTART
493; GCN-NEXT:    ;;#ASMEND
494; GCN-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:4
495; GCN-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:8
496; GCN-NEXT:    s_waitcnt vmcnt(0)
497; GCN-NEXT:    v_add_f32_e32 v0, v0, v1
498; GCN-NEXT:    exp mrt0 v0, off, off, off done vm
499; GCN-NEXT:    s_endpgm
500;
501; GCN-SCRATCH-LABEL: flat_scratch_load_clause:
502; GCN-SCRATCH:       ; %bb.0: ; %.entry
503; GCN-SCRATCH-NEXT:    s_add_u32 s2, s2, s5
504; GCN-SCRATCH-NEXT:    s_addc_u32 s3, s3, 0
505; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
506; GCN-SCRATCH-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
507; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x40b00000
508; GCN-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0x40d00000
509; GCN-SCRATCH-NEXT:    scratch_store_dword off, v0, off offset:4
510; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
511; GCN-SCRATCH-NEXT:    scratch_store_dword off, v1, off offset:8
512; GCN-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
513; GCN-SCRATCH-NEXT:    ;;#ASMSTART
514; GCN-SCRATCH-NEXT:    ;;#ASMEND
515; GCN-SCRATCH-NEXT:    s_clause 0x1
516; GCN-SCRATCH-NEXT:    scratch_load_dword v0, off, off offset:4
517; GCN-SCRATCH-NEXT:    scratch_load_dword v1, off, off offset:8
518; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
519; GCN-SCRATCH-NEXT:    v_add_f32_e32 v0, v0, v1
520; GCN-SCRATCH-NEXT:    exp mrt0 v0, off, off, off done vm
521; GCN-SCRATCH-NEXT:    s_endpgm
522.entry:
523  %alloca = alloca float, align 4, addrspace(5)
524  %alloca2 = alloca float, align 4, addrspace(5)
525  store volatile float 5.5, float addrspace(5)* %alloca
526  store volatile float 6.5, float addrspace(5)* %alloca2
527  call void asm sideeffect "", ""()
528  %load0 = load float, float addrspace(5)* %alloca
529  %load1 = load float, float addrspace(5)* %alloca2
530  %valadd = fadd float %load0, %load1
531  call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
532  ret void
533}
534
535declare i32 @llvm.amdgcn.workitem.id.x()
536declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)
537declare <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)
538