1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
3; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -verify-machineinstrs < %s | FileCheck --check-prefix=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -verify-machineinstrs < %s | FileCheck --check-prefix=GFX6 %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX11 %s
7
8define amdgpu_kernel void @store_lds_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
9; GFX9-LABEL: store_lds_v4i32:
10; GFX9:       ; %bb.0:
11; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
12; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
13; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX9-NEXT:    v_mov_b32_e32 v4, s2
15; GFX9-NEXT:    v_mov_b32_e32 v0, s4
16; GFX9-NEXT:    v_mov_b32_e32 v1, s5
17; GFX9-NEXT:    v_mov_b32_e32 v2, s6
18; GFX9-NEXT:    v_mov_b32_e32 v3, s7
19; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
20; GFX9-NEXT:    s_endpgm
21;
22; GFX7-LABEL: store_lds_v4i32:
23; GFX7:       ; %bb.0:
24; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
25; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
26; GFX7-NEXT:    s_mov_b32 m0, -1
27; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
28; GFX7-NEXT:    v_mov_b32_e32 v4, s4
29; GFX7-NEXT:    v_mov_b32_e32 v0, s0
30; GFX7-NEXT:    v_mov_b32_e32 v1, s1
31; GFX7-NEXT:    v_mov_b32_e32 v2, s2
32; GFX7-NEXT:    v_mov_b32_e32 v3, s3
33; GFX7-NEXT:    ds_write_b128 v4, v[0:3]
34; GFX7-NEXT:    s_endpgm
35;
36; GFX6-LABEL: store_lds_v4i32:
37; GFX6:       ; %bb.0:
38; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4
39; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
40; GFX6-NEXT:    s_mov_b32 m0, -1
41; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
42; GFX6-NEXT:    v_mov_b32_e32 v0, s6
43; GFX6-NEXT:    v_mov_b32_e32 v1, s7
44; GFX6-NEXT:    v_mov_b32_e32 v4, s0
45; GFX6-NEXT:    v_mov_b32_e32 v2, s4
46; GFX6-NEXT:    v_mov_b32_e32 v3, s5
47; GFX6-NEXT:    ds_write2_b64 v4, v[2:3], v[0:1] offset1:1
48; GFX6-NEXT:    s_endpgm
49;
50; GFX10-LABEL: store_lds_v4i32:
51; GFX10:       ; %bb.0:
52; GFX10-NEXT:    s_clause 0x1
53; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
54; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
55; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
56; GFX10-NEXT:    v_mov_b32_e32 v4, s2
57; GFX10-NEXT:    v_mov_b32_e32 v0, s4
58; GFX10-NEXT:    v_mov_b32_e32 v1, s5
59; GFX10-NEXT:    v_mov_b32_e32 v2, s6
60; GFX10-NEXT:    v_mov_b32_e32 v3, s7
61; GFX10-NEXT:    ds_write_b128 v4, v[0:3]
62; GFX10-NEXT:    s_endpgm
63;
64; GFX11-LABEL: store_lds_v4i32:
65; GFX11:       ; %bb.0:
66; GFX11-NEXT:    s_clause 0x1
67; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
68; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
69; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX11-NEXT:    v_mov_b32_e32 v4, s4
71; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
72; GFX11-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
73; GFX11-NEXT:    ds_store_b128 v4, v[0:3]
74; GFX11-NEXT:    s_endpgm
75  store <4 x i32> %x, <4 x i32> addrspace(3)* %out
76  ret void
77}
78
79define amdgpu_kernel void @store_lds_v4i32_align1(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
80; GFX9-LABEL: store_lds_v4i32_align1:
81; GFX9:       ; %bb.0:
82; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
83; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
84; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
85; GFX9-NEXT:    v_mov_b32_e32 v0, s2
86; GFX9-NEXT:    v_mov_b32_e32 v1, s6
87; GFX9-NEXT:    v_mov_b32_e32 v2, s7
88; GFX9-NEXT:    ds_write_b8 v0, v1 offset:8
89; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:10
90; GFX9-NEXT:    ds_write_b8 v0, v2 offset:12
91; GFX9-NEXT:    ds_write_b8_d16_hi v0, v2 offset:14
92; GFX9-NEXT:    v_mov_b32_e32 v1, s4
93; GFX9-NEXT:    ds_write_b8 v0, v1
94; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:2
95; GFX9-NEXT:    v_mov_b32_e32 v1, s5
96; GFX9-NEXT:    s_lshr_b32 s0, s6, 8
97; GFX9-NEXT:    ds_write_b8 v0, v1 offset:4
98; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:6
99; GFX9-NEXT:    v_mov_b32_e32 v1, s0
100; GFX9-NEXT:    s_lshr_b32 s0, s6, 24
101; GFX9-NEXT:    ds_write_b8 v0, v1 offset:9
102; GFX9-NEXT:    v_mov_b32_e32 v1, s0
103; GFX9-NEXT:    s_lshr_b32 s0, s7, 8
104; GFX9-NEXT:    ds_write_b8 v0, v1 offset:11
105; GFX9-NEXT:    v_mov_b32_e32 v1, s0
106; GFX9-NEXT:    s_lshr_b32 s0, s7, 24
107; GFX9-NEXT:    ds_write_b8 v0, v1 offset:13
108; GFX9-NEXT:    v_mov_b32_e32 v1, s0
109; GFX9-NEXT:    s_lshr_b32 s0, s4, 8
110; GFX9-NEXT:    ds_write_b8 v0, v1 offset:15
111; GFX9-NEXT:    v_mov_b32_e32 v1, s0
112; GFX9-NEXT:    s_lshr_b32 s0, s4, 24
113; GFX9-NEXT:    ds_write_b8 v0, v1 offset:1
114; GFX9-NEXT:    v_mov_b32_e32 v1, s0
115; GFX9-NEXT:    s_lshr_b32 s0, s5, 8
116; GFX9-NEXT:    ds_write_b8 v0, v1 offset:3
117; GFX9-NEXT:    v_mov_b32_e32 v1, s0
118; GFX9-NEXT:    s_lshr_b32 s0, s5, 24
119; GFX9-NEXT:    ds_write_b8 v0, v1 offset:5
120; GFX9-NEXT:    v_mov_b32_e32 v1, s0
121; GFX9-NEXT:    ds_write_b8 v0, v1 offset:7
122; GFX9-NEXT:    s_endpgm
123;
124; GFX7-LABEL: store_lds_v4i32_align1:
125; GFX7:       ; %bb.0:
126; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
127; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
128; GFX7-NEXT:    s_mov_b32 m0, -1
129; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
130; GFX7-NEXT:    v_mov_b32_e32 v0, s4
131; GFX7-NEXT:    v_mov_b32_e32 v1, s2
132; GFX7-NEXT:    v_mov_b32_e32 v2, s3
133; GFX7-NEXT:    ds_write_b8 v0, v1 offset:8
134; GFX7-NEXT:    ds_write_b8 v0, v2 offset:12
135; GFX7-NEXT:    v_mov_b32_e32 v1, s0
136; GFX7-NEXT:    ds_write_b8 v0, v1
137; GFX7-NEXT:    v_mov_b32_e32 v1, s1
138; GFX7-NEXT:    s_lshr_b32 s4, s2, 8
139; GFX7-NEXT:    ds_write_b8 v0, v1 offset:4
140; GFX7-NEXT:    v_mov_b32_e32 v1, s4
141; GFX7-NEXT:    s_lshr_b32 s4, s2, 24
142; GFX7-NEXT:    ds_write_b8 v0, v1 offset:9
143; GFX7-NEXT:    v_mov_b32_e32 v1, s4
144; GFX7-NEXT:    s_lshr_b32 s2, s2, 16
145; GFX7-NEXT:    ds_write_b8 v0, v1 offset:11
146; GFX7-NEXT:    v_mov_b32_e32 v1, s2
147; GFX7-NEXT:    s_lshr_b32 s2, s3, 8
148; GFX7-NEXT:    ds_write_b8 v0, v1 offset:10
149; GFX7-NEXT:    v_mov_b32_e32 v1, s2
150; GFX7-NEXT:    s_lshr_b32 s2, s3, 24
151; GFX7-NEXT:    ds_write_b8 v0, v1 offset:13
152; GFX7-NEXT:    v_mov_b32_e32 v1, s2
153; GFX7-NEXT:    s_lshr_b32 s2, s3, 16
154; GFX7-NEXT:    ds_write_b8 v0, v1 offset:15
155; GFX7-NEXT:    v_mov_b32_e32 v1, s2
156; GFX7-NEXT:    s_lshr_b32 s2, s0, 8
157; GFX7-NEXT:    ds_write_b8 v0, v1 offset:14
158; GFX7-NEXT:    v_mov_b32_e32 v1, s2
159; GFX7-NEXT:    s_lshr_b32 s2, s0, 24
160; GFX7-NEXT:    ds_write_b8 v0, v1 offset:1
161; GFX7-NEXT:    v_mov_b32_e32 v1, s2
162; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
163; GFX7-NEXT:    ds_write_b8 v0, v1 offset:3
164; GFX7-NEXT:    v_mov_b32_e32 v1, s0
165; GFX7-NEXT:    s_lshr_b32 s0, s1, 8
166; GFX7-NEXT:    ds_write_b8 v0, v1 offset:2
167; GFX7-NEXT:    v_mov_b32_e32 v1, s0
168; GFX7-NEXT:    s_lshr_b32 s0, s1, 24
169; GFX7-NEXT:    ds_write_b8 v0, v1 offset:5
170; GFX7-NEXT:    v_mov_b32_e32 v1, s0
171; GFX7-NEXT:    s_lshr_b32 s0, s1, 16
172; GFX7-NEXT:    ds_write_b8 v0, v1 offset:7
173; GFX7-NEXT:    v_mov_b32_e32 v1, s0
174; GFX7-NEXT:    ds_write_b8 v0, v1 offset:6
175; GFX7-NEXT:    s_endpgm
176;
177; GFX6-LABEL: store_lds_v4i32_align1:
178; GFX6:       ; %bb.0:
179; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x0
180; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
181; GFX6-NEXT:    s_mov_b32 m0, -1
182; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX6-NEXT:    v_mov_b32_e32 v0, s4
184; GFX6-NEXT:    v_mov_b32_e32 v1, s2
185; GFX6-NEXT:    v_mov_b32_e32 v2, s3
186; GFX6-NEXT:    ds_write_b8 v0, v1 offset:8
187; GFX6-NEXT:    ds_write_b8 v0, v2 offset:12
188; GFX6-NEXT:    v_mov_b32_e32 v1, s0
189; GFX6-NEXT:    ds_write_b8 v0, v1
190; GFX6-NEXT:    v_mov_b32_e32 v1, s1
191; GFX6-NEXT:    s_lshr_b32 s4, s2, 8
192; GFX6-NEXT:    ds_write_b8 v0, v1 offset:4
193; GFX6-NEXT:    v_mov_b32_e32 v1, s4
194; GFX6-NEXT:    s_lshr_b32 s4, s2, 24
195; GFX6-NEXT:    ds_write_b8 v0, v1 offset:9
196; GFX6-NEXT:    v_mov_b32_e32 v1, s4
197; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
198; GFX6-NEXT:    ds_write_b8 v0, v1 offset:11
199; GFX6-NEXT:    v_mov_b32_e32 v1, s2
200; GFX6-NEXT:    s_lshr_b32 s2, s3, 8
201; GFX6-NEXT:    ds_write_b8 v0, v1 offset:10
202; GFX6-NEXT:    v_mov_b32_e32 v1, s2
203; GFX6-NEXT:    s_lshr_b32 s2, s3, 24
204; GFX6-NEXT:    ds_write_b8 v0, v1 offset:13
205; GFX6-NEXT:    v_mov_b32_e32 v1, s2
206; GFX6-NEXT:    s_lshr_b32 s2, s3, 16
207; GFX6-NEXT:    ds_write_b8 v0, v1 offset:15
208; GFX6-NEXT:    v_mov_b32_e32 v1, s2
209; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
210; GFX6-NEXT:    ds_write_b8 v0, v1 offset:14
211; GFX6-NEXT:    v_mov_b32_e32 v1, s2
212; GFX6-NEXT:    s_lshr_b32 s2, s0, 24
213; GFX6-NEXT:    ds_write_b8 v0, v1 offset:1
214; GFX6-NEXT:    v_mov_b32_e32 v1, s2
215; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
216; GFX6-NEXT:    ds_write_b8 v0, v1 offset:3
217; GFX6-NEXT:    v_mov_b32_e32 v1, s0
218; GFX6-NEXT:    s_lshr_b32 s0, s1, 8
219; GFX6-NEXT:    ds_write_b8 v0, v1 offset:2
220; GFX6-NEXT:    v_mov_b32_e32 v1, s0
221; GFX6-NEXT:    s_lshr_b32 s0, s1, 24
222; GFX6-NEXT:    ds_write_b8 v0, v1 offset:5
223; GFX6-NEXT:    v_mov_b32_e32 v1, s0
224; GFX6-NEXT:    s_lshr_b32 s0, s1, 16
225; GFX6-NEXT:    ds_write_b8 v0, v1 offset:7
226; GFX6-NEXT:    v_mov_b32_e32 v1, s0
227; GFX6-NEXT:    ds_write_b8 v0, v1 offset:6
228; GFX6-NEXT:    s_endpgm
229;
230; GFX10-LABEL: store_lds_v4i32_align1:
231; GFX10:       ; %bb.0:
232; GFX10-NEXT:    s_clause 0x1
233; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
234; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
235; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
236; GFX10-NEXT:    v_mov_b32_e32 v0, s2
237; GFX10-NEXT:    v_mov_b32_e32 v1, s6
238; GFX10-NEXT:    s_lshr_b32 s3, s7, 24
239; GFX10-NEXT:    v_mov_b32_e32 v2, s7
240; GFX10-NEXT:    s_lshr_b32 s0, s6, 8
241; GFX10-NEXT:    s_lshr_b32 s1, s6, 24
242; GFX10-NEXT:    s_lshr_b32 s6, s4, 8
243; GFX10-NEXT:    v_mov_b32_e32 v3, s4
244; GFX10-NEXT:    s_lshr_b32 s2, s7, 8
245; GFX10-NEXT:    s_lshr_b32 s4, s4, 24
246; GFX10-NEXT:    v_mov_b32_e32 v8, s3
247; GFX10-NEXT:    v_mov_b32_e32 v5, s0
248; GFX10-NEXT:    v_mov_b32_e32 v9, s6
249; GFX10-NEXT:    s_lshr_b32 s0, s5, 8
250; GFX10-NEXT:    v_mov_b32_e32 v4, s5
251; GFX10-NEXT:    v_mov_b32_e32 v6, s1
252; GFX10-NEXT:    v_mov_b32_e32 v7, s2
253; GFX10-NEXT:    ds_write_b8 v0, v1 offset:8
254; GFX10-NEXT:    ds_write_b8_d16_hi v0, v1 offset:10
255; GFX10-NEXT:    ds_write_b8 v0, v2 offset:12
256; GFX10-NEXT:    ds_write_b8_d16_hi v0, v2 offset:14
257; GFX10-NEXT:    ds_write_b8 v0, v3
258; GFX10-NEXT:    ds_write_b8_d16_hi v0, v3 offset:2
259; GFX10-NEXT:    ds_write_b8 v0, v4 offset:4
260; GFX10-NEXT:    ds_write_b8_d16_hi v0, v4 offset:6
261; GFX10-NEXT:    ds_write_b8 v0, v5 offset:9
262; GFX10-NEXT:    ds_write_b8 v0, v6 offset:11
263; GFX10-NEXT:    ds_write_b8 v0, v7 offset:13
264; GFX10-NEXT:    v_mov_b32_e32 v1, s4
265; GFX10-NEXT:    s_lshr_b32 s1, s5, 24
266; GFX10-NEXT:    v_mov_b32_e32 v2, s0
267; GFX10-NEXT:    v_mov_b32_e32 v3, s1
268; GFX10-NEXT:    ds_write_b8 v0, v8 offset:15
269; GFX10-NEXT:    ds_write_b8 v0, v9 offset:1
270; GFX10-NEXT:    ds_write_b8 v0, v1 offset:3
271; GFX10-NEXT:    ds_write_b8 v0, v2 offset:5
272; GFX10-NEXT:    ds_write_b8 v0, v3 offset:7
273; GFX10-NEXT:    s_endpgm
274;
275; GFX11-LABEL: store_lds_v4i32_align1:
276; GFX11:       ; %bb.0:
277; GFX11-NEXT:    s_clause 0x1
278; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
279; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
280; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
282; GFX11-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s0
283; GFX11-NEXT:    s_lshr_b32 s4, s2, 8
284; GFX11-NEXT:    s_lshr_b32 s5, s3, 8
285; GFX11-NEXT:    s_lshr_b32 s3, s3, 24
286; GFX11-NEXT:    s_lshr_b32 s6, s0, 8
287; GFX11-NEXT:    s_lshr_b32 s0, s0, 24
288; GFX11-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s4
289; GFX11-NEXT:    s_lshr_b32 s2, s2, 24
290; GFX11-NEXT:    s_lshr_b32 s7, s1, 8
291; GFX11-NEXT:    v_dual_mov_b32 v8, s3 :: v_dual_mov_b32 v9, s6
292; GFX11-NEXT:    v_mov_b32_e32 v10, s0
293; GFX11-NEXT:    v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v7, s5
294; GFX11-NEXT:    ds_store_b8 v0, v1 offset:8
295; GFX11-NEXT:    ds_store_b8 v0, v3
296; GFX11-NEXT:    ds_store_b8_d16_hi v0, v1 offset:10
297; GFX11-NEXT:    ds_store_b8 v0, v5 offset:9
298; GFX11-NEXT:    ds_store_b8 v0, v2 offset:12
299; GFX11-NEXT:    ds_store_b8 v0, v6 offset:11
300; GFX11-NEXT:    ds_store_b8 v0, v7 offset:13
301; GFX11-NEXT:    ds_store_b8_d16_hi v0, v2 offset:14
302; GFX11-NEXT:    ds_store_b8 v0, v8 offset:15
303; GFX11-NEXT:    v_mov_b32_e32 v1, s7
304; GFX11-NEXT:    s_lshr_b32 s0, s1, 24
305; GFX11-NEXT:    ds_store_b8_d16_hi v0, v3 offset:2
306; GFX11-NEXT:    ds_store_b8 v0, v9 offset:1
307; GFX11-NEXT:    v_mov_b32_e32 v2, s0
308; GFX11-NEXT:    ds_store_b8 v0, v4 offset:4
309; GFX11-NEXT:    ds_store_b8 v0, v10 offset:3
310; GFX11-NEXT:    ds_store_b8 v0, v1 offset:5
311; GFX11-NEXT:    ds_store_b8_d16_hi v0, v4 offset:6
312; GFX11-NEXT:    ds_store_b8 v0, v2 offset:7
313; GFX11-NEXT:    s_endpgm
314  store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 1
315  ret void
316}
317
318define amdgpu_kernel void @store_lds_v4i32_align2(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
319; GFX9-LABEL: store_lds_v4i32_align2:
320; GFX9:       ; %bb.0:
321; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
322; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
323; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX9-NEXT:    v_mov_b32_e32 v0, s2
325; GFX9-NEXT:    v_mov_b32_e32 v1, s6
326; GFX9-NEXT:    v_mov_b32_e32 v2, s7
327; GFX9-NEXT:    ds_write_b16 v0, v1 offset:8
328; GFX9-NEXT:    ds_write_b16_d16_hi v0, v1 offset:10
329; GFX9-NEXT:    ds_write_b16 v0, v2 offset:12
330; GFX9-NEXT:    ds_write_b16_d16_hi v0, v2 offset:14
331; GFX9-NEXT:    v_mov_b32_e32 v1, s4
332; GFX9-NEXT:    ds_write_b16 v0, v1
333; GFX9-NEXT:    ds_write_b16_d16_hi v0, v1 offset:2
334; GFX9-NEXT:    v_mov_b32_e32 v1, s5
335; GFX9-NEXT:    ds_write_b16 v0, v1 offset:4
336; GFX9-NEXT:    ds_write_b16_d16_hi v0, v1 offset:6
337; GFX9-NEXT:    s_endpgm
338;
339; GFX7-LABEL: store_lds_v4i32_align2:
340; GFX7:       ; %bb.0:
341; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
342; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
343; GFX7-NEXT:    s_mov_b32 m0, -1
344; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
345; GFX7-NEXT:    v_mov_b32_e32 v0, s4
346; GFX7-NEXT:    v_mov_b32_e32 v1, s2
347; GFX7-NEXT:    v_mov_b32_e32 v2, s3
348; GFX7-NEXT:    ds_write_b16 v0, v1 offset:8
349; GFX7-NEXT:    ds_write_b16 v0, v2 offset:12
350; GFX7-NEXT:    v_mov_b32_e32 v1, s0
351; GFX7-NEXT:    ds_write_b16 v0, v1
352; GFX7-NEXT:    v_mov_b32_e32 v1, s1
353; GFX7-NEXT:    s_lshr_b32 s2, s2, 16
354; GFX7-NEXT:    ds_write_b16 v0, v1 offset:4
355; GFX7-NEXT:    v_mov_b32_e32 v1, s2
356; GFX7-NEXT:    s_lshr_b32 s2, s3, 16
357; GFX7-NEXT:    ds_write_b16 v0, v1 offset:10
358; GFX7-NEXT:    v_mov_b32_e32 v1, s2
359; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
360; GFX7-NEXT:    ds_write_b16 v0, v1 offset:14
361; GFX7-NEXT:    v_mov_b32_e32 v1, s0
362; GFX7-NEXT:    s_lshr_b32 s0, s1, 16
363; GFX7-NEXT:    ds_write_b16 v0, v1 offset:2
364; GFX7-NEXT:    v_mov_b32_e32 v1, s0
365; GFX7-NEXT:    ds_write_b16 v0, v1 offset:6
366; GFX7-NEXT:    s_endpgm
367;
368; GFX6-LABEL: store_lds_v4i32_align2:
369; GFX6:       ; %bb.0:
370; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x0
371; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
372; GFX6-NEXT:    s_mov_b32 m0, -1
373; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
374; GFX6-NEXT:    v_mov_b32_e32 v0, s4
375; GFX6-NEXT:    v_mov_b32_e32 v1, s2
376; GFX6-NEXT:    v_mov_b32_e32 v2, s3
377; GFX6-NEXT:    ds_write_b16 v0, v1 offset:8
378; GFX6-NEXT:    ds_write_b16 v0, v2 offset:12
379; GFX6-NEXT:    v_mov_b32_e32 v1, s0
380; GFX6-NEXT:    ds_write_b16 v0, v1
381; GFX6-NEXT:    v_mov_b32_e32 v1, s1
382; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
383; GFX6-NEXT:    ds_write_b16 v0, v1 offset:4
384; GFX6-NEXT:    v_mov_b32_e32 v1, s2
385; GFX6-NEXT:    s_lshr_b32 s2, s3, 16
386; GFX6-NEXT:    ds_write_b16 v0, v1 offset:10
387; GFX6-NEXT:    v_mov_b32_e32 v1, s2
388; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
389; GFX6-NEXT:    ds_write_b16 v0, v1 offset:14
390; GFX6-NEXT:    v_mov_b32_e32 v1, s0
391; GFX6-NEXT:    s_lshr_b32 s0, s1, 16
392; GFX6-NEXT:    ds_write_b16 v0, v1 offset:2
393; GFX6-NEXT:    v_mov_b32_e32 v1, s0
394; GFX6-NEXT:    ds_write_b16 v0, v1 offset:6
395; GFX6-NEXT:    s_endpgm
396;
397; GFX10-LABEL: store_lds_v4i32_align2:
398; GFX10:       ; %bb.0:
399; GFX10-NEXT:    s_clause 0x1
400; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
401; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
402; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
403; GFX10-NEXT:    v_mov_b32_e32 v0, s2
404; GFX10-NEXT:    v_mov_b32_e32 v1, s6
405; GFX10-NEXT:    v_mov_b32_e32 v2, s7
406; GFX10-NEXT:    v_mov_b32_e32 v3, s4
407; GFX10-NEXT:    v_mov_b32_e32 v4, s5
408; GFX10-NEXT:    ds_write_b16 v0, v1 offset:8
409; GFX10-NEXT:    ds_write_b16_d16_hi v0, v1 offset:10
410; GFX10-NEXT:    ds_write_b16 v0, v2 offset:12
411; GFX10-NEXT:    ds_write_b16_d16_hi v0, v2 offset:14
412; GFX10-NEXT:    ds_write_b16 v0, v3
413; GFX10-NEXT:    ds_write_b16_d16_hi v0, v3 offset:2
414; GFX10-NEXT:    ds_write_b16 v0, v4 offset:4
415; GFX10-NEXT:    ds_write_b16_d16_hi v0, v4 offset:6
416; GFX10-NEXT:    s_endpgm
417;
418; GFX11-LABEL: store_lds_v4i32_align2:
419; GFX11:       ; %bb.0:
420; GFX11-NEXT:    s_clause 0x1
421; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
422; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
423; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
424; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
425; GFX11-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s0
426; GFX11-NEXT:    v_mov_b32_e32 v4, s1
427; GFX11-NEXT:    ds_store_b16_d16_hi v0, v1 offset:10
428; GFX11-NEXT:    ds_store_b16 v0, v2 offset:12
429; GFX11-NEXT:    ds_store_b16_d16_hi v0, v2 offset:14
430; GFX11-NEXT:    ds_store_b16 v0, v3
431; GFX11-NEXT:    ds_store_b16_d16_hi v0, v3 offset:2
432; GFX11-NEXT:    ds_store_b16 v0, v4 offset:4
433; GFX11-NEXT:    ds_store_b16 v0, v1 offset:8
434; GFX11-NEXT:    ds_store_b16_d16_hi v0, v4 offset:6
435; GFX11-NEXT:    s_endpgm
436  store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 2
437  ret void
438}
439
440define amdgpu_kernel void @store_lds_v4i32_align4(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
441; GFX9-LABEL: store_lds_v4i32_align4:
442; GFX9:       ; %bb.0:
443; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
444; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
445; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
446; GFX9-NEXT:    v_mov_b32_e32 v0, s2
447; GFX9-NEXT:    v_mov_b32_e32 v1, s4
448; GFX9-NEXT:    v_mov_b32_e32 v2, s5
449; GFX9-NEXT:    v_mov_b32_e32 v3, s6
450; GFX9-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1
451; GFX9-NEXT:    v_mov_b32_e32 v1, s7
452; GFX9-NEXT:    ds_write2_b32 v0, v3, v1 offset0:2 offset1:3
453; GFX9-NEXT:    s_endpgm
454;
455; GFX7-LABEL: store_lds_v4i32_align4:
456; GFX7:       ; %bb.0:
457; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
458; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
459; GFX7-NEXT:    s_mov_b32 m0, -1
460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX7-NEXT:    v_mov_b32_e32 v0, s4
462; GFX7-NEXT:    v_mov_b32_e32 v1, s0
463; GFX7-NEXT:    v_mov_b32_e32 v2, s1
464; GFX7-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1
465; GFX7-NEXT:    v_mov_b32_e32 v1, s2
466; GFX7-NEXT:    v_mov_b32_e32 v2, s3
467; GFX7-NEXT:    ds_write2_b32 v0, v1, v2 offset0:2 offset1:3
468; GFX7-NEXT:    s_endpgm
469;
470; GFX6-LABEL: store_lds_v4i32_align4:
471; GFX6:       ; %bb.0:
472; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x0
473; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
474; GFX6-NEXT:    s_mov_b32 m0, -1
475; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
476; GFX6-NEXT:    v_mov_b32_e32 v0, s4
477; GFX6-NEXT:    v_mov_b32_e32 v1, s2
478; GFX6-NEXT:    v_mov_b32_e32 v2, s3
479; GFX6-NEXT:    ds_write2_b32 v0, v1, v2 offset0:2 offset1:3
480; GFX6-NEXT:    v_mov_b32_e32 v1, s0
481; GFX6-NEXT:    v_mov_b32_e32 v2, s1
482; GFX6-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1
483; GFX6-NEXT:    s_endpgm
484;
485; GFX10-LABEL: store_lds_v4i32_align4:
486; GFX10:       ; %bb.0:
487; GFX10-NEXT:    s_clause 0x1
488; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
489; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
490; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
491; GFX10-NEXT:    v_mov_b32_e32 v0, s2
492; GFX10-NEXT:    v_mov_b32_e32 v1, s6
493; GFX10-NEXT:    v_mov_b32_e32 v2, s7
494; GFX10-NEXT:    v_mov_b32_e32 v3, s4
495; GFX10-NEXT:    v_mov_b32_e32 v4, s5
496; GFX10-NEXT:    ds_write2_b32 v0, v1, v2 offset0:2 offset1:3
497; GFX10-NEXT:    ds_write2_b32 v0, v3, v4 offset1:1
498; GFX10-NEXT:    s_endpgm
499;
500; GFX11-LABEL: store_lds_v4i32_align4:
501; GFX11:       ; %bb.0:
502; GFX11-NEXT:    s_clause 0x1
503; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
504; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
505; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
506; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s0
507; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
508; GFX11-NEXT:    v_mov_b32_e32 v4, s3
509; GFX11-NEXT:    ds_store_2addr_b32 v0, v1, v2 offset1:1
510; GFX11-NEXT:    ds_store_2addr_b32 v0, v3, v4 offset0:2 offset1:3
511; GFX11-NEXT:    s_endpgm
512  store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 4
513  ret void
514}
515
516define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
517; GFX9-LABEL: store_lds_v4i32_align8:
518; GFX9:       ; %bb.0:
519; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
520; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
521; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX9-NEXT:    v_mov_b32_e32 v4, s2
523; GFX9-NEXT:    v_mov_b32_e32 v0, s4
524; GFX9-NEXT:    v_mov_b32_e32 v2, s6
525; GFX9-NEXT:    v_mov_b32_e32 v1, s5
526; GFX9-NEXT:    v_mov_b32_e32 v3, s7
527; GFX9-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
528; GFX9-NEXT:    s_endpgm
529;
530; GFX7-LABEL: store_lds_v4i32_align8:
531; GFX7:       ; %bb.0:
532; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
533; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
534; GFX7-NEXT:    s_mov_b32 m0, -1
535; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
536; GFX7-NEXT:    v_mov_b32_e32 v4, s4
537; GFX7-NEXT:    v_mov_b32_e32 v0, s0
538; GFX7-NEXT:    v_mov_b32_e32 v2, s2
539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
540; GFX7-NEXT:    v_mov_b32_e32 v3, s3
541; GFX7-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
542; GFX7-NEXT:    s_endpgm
543;
544; GFX6-LABEL: store_lds_v4i32_align8:
545; GFX6:       ; %bb.0:
546; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4
547; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
548; GFX6-NEXT:    s_mov_b32 m0, -1
549; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
550; GFX6-NEXT:    v_mov_b32_e32 v0, s4
551; GFX6-NEXT:    v_mov_b32_e32 v1, s5
552; GFX6-NEXT:    v_mov_b32_e32 v4, s0
553; GFX6-NEXT:    v_mov_b32_e32 v2, s6
554; GFX6-NEXT:    v_mov_b32_e32 v3, s7
555; GFX6-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
556; GFX6-NEXT:    s_endpgm
557;
558; GFX10-LABEL: store_lds_v4i32_align8:
559; GFX10:       ; %bb.0:
560; GFX10-NEXT:    s_clause 0x1
561; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
562; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
563; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
564; GFX10-NEXT:    v_mov_b32_e32 v0, s4
565; GFX10-NEXT:    v_mov_b32_e32 v1, s5
566; GFX10-NEXT:    v_mov_b32_e32 v4, s2
567; GFX10-NEXT:    v_mov_b32_e32 v2, s6
568; GFX10-NEXT:    v_mov_b32_e32 v3, s7
569; GFX10-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
570; GFX10-NEXT:    s_endpgm
571;
572; GFX11-LABEL: store_lds_v4i32_align8:
573; GFX11:       ; %bb.0:
574; GFX11-NEXT:    s_clause 0x1
575; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
576; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
577; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
578; GFX11-NEXT:    v_mov_b32_e32 v4, s4
579; GFX11-NEXT:    v_mov_b32_e32 v0, s0
580; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
581; GFX11-NEXT:    v_mov_b32_e32 v1, s1
582; GFX11-NEXT:    ds_store_2addr_b64 v4, v[0:1], v[2:3] offset1:1
583; GFX11-NEXT:    s_endpgm
584  store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 8
585  ret void
586}
587
588define amdgpu_kernel void @store_lds_v4i32_align16(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
589; GFX9-LABEL: store_lds_v4i32_align16:
590; GFX9:       ; %bb.0:
591; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
592; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
593; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX9-NEXT:    v_mov_b32_e32 v4, s2
595; GFX9-NEXT:    v_mov_b32_e32 v0, s4
596; GFX9-NEXT:    v_mov_b32_e32 v1, s5
597; GFX9-NEXT:    v_mov_b32_e32 v2, s6
598; GFX9-NEXT:    v_mov_b32_e32 v3, s7
599; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
600; GFX9-NEXT:    s_endpgm
601;
602; GFX7-LABEL: store_lds_v4i32_align16:
603; GFX7:       ; %bb.0:
604; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
605; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x4
606; GFX7-NEXT:    s_mov_b32 m0, -1
607; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
608; GFX7-NEXT:    v_mov_b32_e32 v4, s4
609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
611; GFX7-NEXT:    v_mov_b32_e32 v2, s2
612; GFX7-NEXT:    v_mov_b32_e32 v3, s3
613; GFX7-NEXT:    ds_write_b128 v4, v[0:3]
614; GFX7-NEXT:    s_endpgm
615;
616; GFX6-LABEL: store_lds_v4i32_align16:
617; GFX6:       ; %bb.0:
618; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4
619; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
620; GFX6-NEXT:    s_mov_b32 m0, -1
621; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX6-NEXT:    v_mov_b32_e32 v0, s6
623; GFX6-NEXT:    v_mov_b32_e32 v1, s7
624; GFX6-NEXT:    v_mov_b32_e32 v4, s0
625; GFX6-NEXT:    v_mov_b32_e32 v2, s4
626; GFX6-NEXT:    v_mov_b32_e32 v3, s5
627; GFX6-NEXT:    ds_write2_b64 v4, v[2:3], v[0:1] offset1:1
628; GFX6-NEXT:    s_endpgm
629;
630; GFX10-LABEL: store_lds_v4i32_align16:
631; GFX10:       ; %bb.0:
632; GFX10-NEXT:    s_clause 0x1
633; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
634; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
635; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
636; GFX10-NEXT:    v_mov_b32_e32 v4, s2
637; GFX10-NEXT:    v_mov_b32_e32 v0, s4
638; GFX10-NEXT:    v_mov_b32_e32 v1, s5
639; GFX10-NEXT:    v_mov_b32_e32 v2, s6
640; GFX10-NEXT:    v_mov_b32_e32 v3, s7
641; GFX10-NEXT:    ds_write_b128 v4, v[0:3]
642; GFX10-NEXT:    s_endpgm
643;
644; GFX11-LABEL: store_lds_v4i32_align16:
645; GFX11:       ; %bb.0:
646; GFX11-NEXT:    s_clause 0x1
647; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
648; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x10
649; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
650; GFX11-NEXT:    v_mov_b32_e32 v4, s4
651; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
652; GFX11-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
653; GFX11-NEXT:    ds_store_b128 v4, v[0:3]
654; GFX11-NEXT:    s_endpgm
655  store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 16
656  ret void
657}
658