1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx900 -mattr=-unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,ALIGNED,ALIGNED-SDAG
3; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 -mattr=-unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,ALIGNED,ALIGNED-GISEL
4; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx900 -mattr=+unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,UNALIGNED
5; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 -mattr=+unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,UNALIGNED
6
7define amdgpu_kernel void @ds1align1(i8 addrspace(3)* %in, i8 addrspace(3)* %out) {
8; GCN-LABEL: ds1align1:
9; GCN:       ; %bb.0:
10; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
11; GCN-NEXT:    s_waitcnt lgkmcnt(0)
12; GCN-NEXT:    v_mov_b32_e32 v0, s0
13; GCN-NEXT:    ds_read_u8 v0, v0
14; GCN-NEXT:    v_mov_b32_e32 v1, s1
15; GCN-NEXT:    s_waitcnt lgkmcnt(0)
16; GCN-NEXT:    ds_write_b8 v1, v0
17; GCN-NEXT:    s_endpgm
18  %val = load i8, i8 addrspace(3)* %in, align 1
19  store i8 %val, i8 addrspace(3)* %out, align 1
20  ret void
21}
22
23define amdgpu_kernel void @ds2align1(i16 addrspace(3)* %in, i16 addrspace(3)* %out) {
24; ALIGNED-LABEL: ds2align1:
25; ALIGNED:       ; %bb.0:
26; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
27; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
28; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
29; ALIGNED-NEXT:    ds_read_u8 v1, v0
30; ALIGNED-NEXT:    ds_read_u8 v0, v0 offset:1
31; ALIGNED-NEXT:    v_mov_b32_e32 v2, s1
32; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
33; ALIGNED-NEXT:    ds_write_b8 v2, v1
34; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
35; ALIGNED-NEXT:    ds_write_b8 v2, v0 offset:1
36; ALIGNED-NEXT:    s_endpgm
37;
38; UNALIGNED-LABEL: ds2align1:
39; UNALIGNED:       ; %bb.0:
40; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
41; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
42; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
43; UNALIGNED-NEXT:    ds_read_u16 v0, v0
44; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
45; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
46; UNALIGNED-NEXT:    ds_write_b16 v1, v0
47; UNALIGNED-NEXT:    s_endpgm
48  %val = load i16, i16 addrspace(3)* %in, align 1
49  store i16 %val, i16 addrspace(3)* %out, align 1
50  ret void
51}
52
53define amdgpu_kernel void @ds2align2(i16 addrspace(3)* %in, i16 addrspace(3)* %out) {
54; GCN-LABEL: ds2align2:
55; GCN:       ; %bb.0:
56; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
57; GCN-NEXT:    s_waitcnt lgkmcnt(0)
58; GCN-NEXT:    v_mov_b32_e32 v0, s0
59; GCN-NEXT:    ds_read_u16 v0, v0
60; GCN-NEXT:    v_mov_b32_e32 v1, s1
61; GCN-NEXT:    s_waitcnt lgkmcnt(0)
62; GCN-NEXT:    ds_write_b16 v1, v0
63; GCN-NEXT:    s_endpgm
64  %val = load i16, i16 addrspace(3)* %in, align 2
65  store i16 %val, i16 addrspace(3)* %out, align 2
66  ret void
67}
68
69define amdgpu_kernel void @ds4align1(i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
70; ALIGNED-LABEL: ds4align1:
71; ALIGNED:       ; %bb.0:
72; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
73; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
74; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
75; ALIGNED-NEXT:    ds_read_u8 v1, v0
76; ALIGNED-NEXT:    ds_read_u8 v2, v0 offset:1
77; ALIGNED-NEXT:    ds_read_u8 v3, v0 offset:2
78; ALIGNED-NEXT:    ds_read_u8 v0, v0 offset:3
79; ALIGNED-NEXT:    v_mov_b32_e32 v4, s1
80; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
81; ALIGNED-NEXT:    ds_write_b8 v4, v1
82; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
83; ALIGNED-NEXT:    ds_write_b8 v4, v2 offset:1
84; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
85; ALIGNED-NEXT:    ds_write_b8 v4, v3 offset:2
86; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
87; ALIGNED-NEXT:    ds_write_b8 v4, v0 offset:3
88; ALIGNED-NEXT:    s_endpgm
89;
90; UNALIGNED-LABEL: ds4align1:
91; UNALIGNED:       ; %bb.0:
92; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
93; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
94; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
95; UNALIGNED-NEXT:    ds_read_b32 v0, v0
96; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
97; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
98; UNALIGNED-NEXT:    ds_write_b32 v1, v0
99; UNALIGNED-NEXT:    s_endpgm
100  %val = load i32, i32 addrspace(3)* %in, align 1
101  store i32 %val, i32 addrspace(3)* %out, align 1
102  ret void
103}
104
105define amdgpu_kernel void @ds4align2(i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
106; ALIGNED-LABEL: ds4align2:
107; ALIGNED:       ; %bb.0:
108; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
109; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
110; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
111; ALIGNED-NEXT:    ds_read_u16 v1, v0
112; ALIGNED-NEXT:    ds_read_u16 v0, v0 offset:2
113; ALIGNED-NEXT:    v_mov_b32_e32 v2, s1
114; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
115; ALIGNED-NEXT:    ds_write_b16 v2, v1
116; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
117; ALIGNED-NEXT:    ds_write_b16 v2, v0 offset:2
118; ALIGNED-NEXT:    s_endpgm
119;
120; UNALIGNED-LABEL: ds4align2:
121; UNALIGNED:       ; %bb.0:
122; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
123; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
124; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
125; UNALIGNED-NEXT:    ds_read_b32 v0, v0
126; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
127; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
128; UNALIGNED-NEXT:    ds_write_b32 v1, v0
129; UNALIGNED-NEXT:    s_endpgm
130  %val = load i32, i32 addrspace(3)* %in, align 2
131  store i32 %val, i32 addrspace(3)* %out, align 2
132  ret void
133}
134
135define amdgpu_kernel void @ds4align4(i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
136; GCN-LABEL: ds4align4:
137; GCN:       ; %bb.0:
138; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
139; GCN-NEXT:    s_waitcnt lgkmcnt(0)
140; GCN-NEXT:    v_mov_b32_e32 v0, s0
141; GCN-NEXT:    ds_read_b32 v0, v0
142; GCN-NEXT:    v_mov_b32_e32 v1, s1
143; GCN-NEXT:    s_waitcnt lgkmcnt(0)
144; GCN-NEXT:    ds_write_b32 v1, v0
145; GCN-NEXT:    s_endpgm
146  %val = load i32, i32 addrspace(3)* %in, align 4
147  store i32 %val, i32 addrspace(3)* %out, align 4
148  ret void
149}
150
151define amdgpu_kernel void @ds8align1(<2 x i32> addrspace(3)* %in, <2 x i32> addrspace(3)* %out) {
152; ALIGNED-SDAG-LABEL: ds8align1:
153; ALIGNED-SDAG:       ; %bb.0:
154; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
155; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
156; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
157; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0
158; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:1
159; ALIGNED-SDAG-NEXT:    ds_read_u8 v4, v0 offset:2
160; ALIGNED-SDAG-NEXT:    ds_read_u8 v5, v0 offset:3
161; ALIGNED-SDAG-NEXT:    ds_read_u8 v6, v0 offset:4
162; ALIGNED-SDAG-NEXT:    ds_read_u8 v7, v0 offset:5
163; ALIGNED-SDAG-NEXT:    ds_read_u8 v8, v0 offset:6
164; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:7
165; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
166; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
167; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v4 offset:2
168; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
169; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v5 offset:3
170; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v2
171; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v3 offset:1
172; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
173; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v8 offset:6
174; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
175; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v0 offset:7
176; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v6 offset:4
177; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v7 offset:5
178; ALIGNED-SDAG-NEXT:    s_endpgm
179;
180; ALIGNED-GISEL-LABEL: ds8align1:
181; ALIGNED-GISEL:       ; %bb.0:
182; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
183; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
184; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
185; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0
186; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:1
187; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:2
188; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:3
189; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:4
190; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:5
191; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:6
192; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:7
193; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v1, s1
194; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
195; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v2
196; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
197; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v3 offset:1
198; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
199; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v4 offset:2
200; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
201; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v5 offset:3
202; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
203; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v6 offset:4
204; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
205; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v7 offset:5
206; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
207; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v8 offset:6
208; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
209; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v0 offset:7
210; ALIGNED-GISEL-NEXT:    s_endpgm
211;
212; UNALIGNED-LABEL: ds8align1:
213; UNALIGNED:       ; %bb.0:
214; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
215; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
216; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
217; UNALIGNED-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
218; UNALIGNED-NEXT:    v_mov_b32_e32 v2, s1
219; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
220; UNALIGNED-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
221; UNALIGNED-NEXT:    s_endpgm
222  %val = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 1
223  store <2 x i32> %val, <2 x i32> addrspace(3)* %out, align 1
224  ret void
225}
226
227define amdgpu_kernel void @ds8align2(<2 x i32> addrspace(3)* %in, <2 x i32> addrspace(3)* %out) {
228; ALIGNED-SDAG-LABEL: ds8align2:
229; ALIGNED-SDAG:       ; %bb.0:
230; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
231; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
232; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
233; ALIGNED-SDAG-NEXT:    ds_read_u16 v1, v0
234; ALIGNED-SDAG-NEXT:    ds_read_u16 v2, v0 offset:2
235; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:4
236; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:6
237; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v4, s1
238; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(2)
239; ALIGNED-SDAG-NEXT:    ds_write_b16 v4, v2 offset:2
240; ALIGNED-SDAG-NEXT:    ds_write_b16 v4, v1
241; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(2)
242; ALIGNED-SDAG-NEXT:    ds_write_b16 v4, v0 offset:6
243; ALIGNED-SDAG-NEXT:    ds_write_b16 v4, v3 offset:4
244; ALIGNED-SDAG-NEXT:    s_endpgm
245;
246; ALIGNED-GISEL-LABEL: ds8align2:
247; ALIGNED-GISEL:       ; %bb.0:
248; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
249; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
250; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
251; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
252; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0 offset:2
253; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:4
254; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:6
255; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v4, s1
256; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
257; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v1
258; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
259; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v2 offset:2
260; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
261; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v3 offset:4
262; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
263; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v0 offset:6
264; ALIGNED-GISEL-NEXT:    s_endpgm
265;
266; UNALIGNED-LABEL: ds8align2:
267; UNALIGNED:       ; %bb.0:
268; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
269; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
270; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
271; UNALIGNED-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
272; UNALIGNED-NEXT:    v_mov_b32_e32 v2, s1
273; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
274; UNALIGNED-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
275; UNALIGNED-NEXT:    s_endpgm
276  %val = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 2
277  store <2 x i32> %val, <2 x i32> addrspace(3)* %out, align 2
278  ret void
279}
280
281define amdgpu_kernel void @ds8align4(<2 x i32> addrspace(3)* %in, <2 x i32> addrspace(3)* %out) {
282; GCN-LABEL: ds8align4:
283; GCN:       ; %bb.0:
284; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
285; GCN-NEXT:    s_waitcnt lgkmcnt(0)
286; GCN-NEXT:    v_mov_b32_e32 v0, s0
287; GCN-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
288; GCN-NEXT:    v_mov_b32_e32 v2, s1
289; GCN-NEXT:    s_waitcnt lgkmcnt(0)
290; GCN-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
291; GCN-NEXT:    s_endpgm
292  %val = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4
293  store <2 x i32> %val, <2 x i32> addrspace(3)* %out, align 4
294  ret void
295}
296
297define amdgpu_kernel void @ds8align8(<2 x i32> addrspace(3)* %in, <2 x i32> addrspace(3)* %out) {
298; GCN-LABEL: ds8align8:
299; GCN:       ; %bb.0:
300; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
301; GCN-NEXT:    s_waitcnt lgkmcnt(0)
302; GCN-NEXT:    v_mov_b32_e32 v0, s0
303; GCN-NEXT:    ds_read_b64 v[0:1], v0
304; GCN-NEXT:    v_mov_b32_e32 v2, s1
305; GCN-NEXT:    s_waitcnt lgkmcnt(0)
306; GCN-NEXT:    ds_write_b64 v2, v[0:1]
307; GCN-NEXT:    s_endpgm
308  %val = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 8
309  store <2 x i32> %val, <2 x i32> addrspace(3)* %out, align 8
310  ret void
311}
312
313define amdgpu_kernel void @ds12align1(<3 x i32> addrspace(3)* %in, <3 x i32> addrspace(3)* %out) {
314; ALIGNED-SDAG-LABEL: ds12align1:
315; ALIGNED-SDAG:       ; %bb.0:
316; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
317; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
318; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
319; ALIGNED-SDAG-NEXT:    ds_read_u8 v1, v0
320; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0 offset:1
321; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:2
322; ALIGNED-SDAG-NEXT:    ds_read_u8 v4, v0 offset:3
323; ALIGNED-SDAG-NEXT:    ds_read_u8 v5, v0 offset:4
324; ALIGNED-SDAG-NEXT:    ds_read_u8 v6, v0 offset:5
325; ALIGNED-SDAG-NEXT:    ds_read_u8 v7, v0 offset:6
326; ALIGNED-SDAG-NEXT:    ds_read_u8 v8, v0 offset:7
327; ALIGNED-SDAG-NEXT:    ds_read_u8 v9, v0 offset:8
328; ALIGNED-SDAG-NEXT:    ds_read_u8 v10, v0 offset:9
329; ALIGNED-SDAG-NEXT:    ds_read_u8 v11, v0 offset:10
330; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:11
331; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v12, s1
332; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
333; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v9 offset:8
334; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
335; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v10 offset:9
336; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v3 offset:2
337; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v4 offset:3
338; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v1
339; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v2 offset:1
340; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v5 offset:4
341; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v6 offset:5
342; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v7 offset:6
343; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v8 offset:7
344; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(11)
345; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v11 offset:10
346; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(11)
347; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v0 offset:11
348; ALIGNED-SDAG-NEXT:    s_endpgm
349;
350; ALIGNED-GISEL-LABEL: ds12align1:
351; ALIGNED-GISEL:       ; %bb.0:
352; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
353; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
354; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s0
355; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v2
356; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v2 offset:1
357; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v2 offset:2
358; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v2 offset:3
359; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v2 offset:4
360; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v2 offset:5
361; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v2 offset:6
362; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v2 offset:7
363; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
364; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
365; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
366; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
367; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
368; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
369; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v0, v1, v3
370; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
371; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v6, 8, v5
372; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
373; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
374; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
375; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 24, v8
376; ALIGNED-GISEL-NEXT:    v_or3_b32 v1, v1, v3, v4
377; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v2 offset:8
378; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v2 offset:9
379; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v2 offset:10
380; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v2 offset:11
381; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v6, 8, v0
382; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
383; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v9, s1
384; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v8, 24, v0
385; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v0
386; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v6 offset:1
387; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v7 offset:2
388; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v8 offset:3
389; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
390; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
391; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
392; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v1 offset:4
393; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v0 offset:5
394; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v6 offset:6
395; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v7 offset:7
396; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
397; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v3 offset:8
398; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
399; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v4 offset:9
400; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
401; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v5 offset:10
402; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
403; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v2 offset:11
404; ALIGNED-GISEL-NEXT:    s_endpgm
405;
406; UNALIGNED-LABEL: ds12align1:
407; UNALIGNED:       ; %bb.0:
408; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
409; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
410; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
411; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0
412; UNALIGNED-NEXT:    v_mov_b32_e32 v3, s1
413; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
414; UNALIGNED-NEXT:    ds_write_b96 v3, v[0:2]
415; UNALIGNED-NEXT:    s_endpgm
416  %val = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 1
417  store <3 x i32> %val, <3 x i32> addrspace(3)* %out, align 1
418  ret void
419}
420
421define amdgpu_kernel void @ds12align2(<3 x i32> addrspace(3)* %in, <3 x i32> addrspace(3)* %out) {
422; ALIGNED-SDAG-LABEL: ds12align2:
423; ALIGNED-SDAG:       ; %bb.0:
424; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
425; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
426; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
427; ALIGNED-SDAG-NEXT:    ds_read_u16 v2, v0
428; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:2
429; ALIGNED-SDAG-NEXT:    ds_read_u16 v4, v0 offset:4
430; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:6
431; ALIGNED-SDAG-NEXT:    ds_read_u16 v6, v0 offset:8
432; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:10
433; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
434; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
435; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v6 offset:8
436; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v3 offset:2
437; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v2
438; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v4 offset:4
439; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v5 offset:6
440; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
441; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v0 offset:10
442; ALIGNED-SDAG-NEXT:    s_endpgm
443;
444; ALIGNED-GISEL-LABEL: ds12align2:
445; ALIGNED-GISEL:       ; %bb.0:
446; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
447; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
448; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
449; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
450; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:2
451; ALIGNED-GISEL-NEXT:    ds_read_u16 v4, v0 offset:4
452; ALIGNED-GISEL-NEXT:    ds_read_u16 v5, v0 offset:6
453; ALIGNED-GISEL-NEXT:    ds_read_u16 v6, v0 offset:8
454; ALIGNED-GISEL-NEXT:    ds_read_u16 v7, v0 offset:10
455; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s1
456; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
457; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v3, 16, v1
458; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
459; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
460; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
461; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v0
462; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v3 offset:2
463; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
464; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v1 offset:4
465; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v0 offset:6
466; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
467; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v6 offset:8
468; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
469; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v7 offset:10
470; ALIGNED-GISEL-NEXT:    s_endpgm
471;
472; UNALIGNED-LABEL: ds12align2:
473; UNALIGNED:       ; %bb.0:
474; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
475; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
476; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
477; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0
478; UNALIGNED-NEXT:    v_mov_b32_e32 v3, s1
479; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
480; UNALIGNED-NEXT:    ds_write_b96 v3, v[0:2]
481; UNALIGNED-NEXT:    s_endpgm
482  %val = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 2
483  store <3 x i32> %val, <3 x i32> addrspace(3)* %out, align 2
484  ret void
485}
486
487define amdgpu_kernel void @ds12align4(<3 x i32> addrspace(3)* %in, <3 x i32> addrspace(3)* %out) {
488; ALIGNED-LABEL: ds12align4:
489; ALIGNED:       ; %bb.0:
490; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
491; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
492; ALIGNED-NEXT:    v_mov_b32_e32 v2, s0
493; ALIGNED-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
494; ALIGNED-NEXT:    ds_read_b32 v2, v2 offset:8
495; ALIGNED-NEXT:    v_mov_b32_e32 v3, s1
496; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
497; ALIGNED-NEXT:    ds_write2_b32 v3, v0, v1 offset1:1
498; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
499; ALIGNED-NEXT:    ds_write_b32 v3, v2 offset:8
500; ALIGNED-NEXT:    s_endpgm
501;
502; UNALIGNED-LABEL: ds12align4:
503; UNALIGNED:       ; %bb.0:
504; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
505; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
506; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
507; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0
508; UNALIGNED-NEXT:    v_mov_b32_e32 v3, s1
509; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
510; UNALIGNED-NEXT:    ds_write_b96 v3, v[0:2]
511; UNALIGNED-NEXT:    s_endpgm
512  %val = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 4
513  store <3 x i32> %val, <3 x i32> addrspace(3)* %out, align 4
514  ret void
515}
516
517; TODO: Why does the ALIGNED-SDAG code use ds_write_b64 but not ds_read_b64?
518define amdgpu_kernel void @ds12align8(<3 x i32> addrspace(3)* %in, <3 x i32> addrspace(3)* %out) {
519; ALIGNED-SDAG-LABEL: ds12align8:
520; ALIGNED-SDAG:       ; %bb.0:
521; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
522; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
523; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v2, s0
524; ALIGNED-SDAG-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
525; ALIGNED-SDAG-NEXT:    ds_read_b32 v2, v2 offset:8
526; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v3, s1
527; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
528; ALIGNED-SDAG-NEXT:    ds_write_b32 v3, v2 offset:8
529; ALIGNED-SDAG-NEXT:    ds_write_b64 v3, v[0:1]
530; ALIGNED-SDAG-NEXT:    s_endpgm
531;
532; ALIGNED-GISEL-LABEL: ds12align8:
533; ALIGNED-GISEL:       ; %bb.0:
534; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
535; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
536; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s0
537; ALIGNED-GISEL-NEXT:    ds_read_b64 v[0:1], v2
538; ALIGNED-GISEL-NEXT:    ds_read_b32 v2, v2 offset:8
539; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v3, s1
540; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
541; ALIGNED-GISEL-NEXT:    ds_write_b64 v3, v[0:1]
542; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
543; ALIGNED-GISEL-NEXT:    ds_write_b32 v3, v2 offset:8
544; ALIGNED-GISEL-NEXT:    s_endpgm
545;
546; UNALIGNED-LABEL: ds12align8:
547; UNALIGNED:       ; %bb.0:
548; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
549; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
550; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
551; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0
552; UNALIGNED-NEXT:    v_mov_b32_e32 v3, s1
553; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
554; UNALIGNED-NEXT:    ds_write_b96 v3, v[0:2]
555; UNALIGNED-NEXT:    s_endpgm
556  %val = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 8
557  store <3 x i32> %val, <3 x i32> addrspace(3)* %out, align 8
558  ret void
559}
560
561define amdgpu_kernel void @ds12align16(<3 x i32> addrspace(3)* %in, <3 x i32> addrspace(3)* %out) {
562; GCN-LABEL: ds12align16:
563; GCN:       ; %bb.0:
564; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
565; GCN-NEXT:    s_waitcnt lgkmcnt(0)
566; GCN-NEXT:    v_mov_b32_e32 v0, s0
567; GCN-NEXT:    ds_read_b96 v[0:2], v0
568; GCN-NEXT:    v_mov_b32_e32 v3, s1
569; GCN-NEXT:    s_waitcnt lgkmcnt(0)
570; GCN-NEXT:    ds_write_b96 v3, v[0:2]
571; GCN-NEXT:    s_endpgm
572  %val = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 16
573  store <3 x i32> %val, <3 x i32> addrspace(3)* %out, align 16
574  ret void
575}
576
577define amdgpu_kernel void @ds16align1(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
578; ALIGNED-SDAG-LABEL: ds16align1:
579; ALIGNED-SDAG:       ; %bb.0:
580; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
581; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
582; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
583; ALIGNED-SDAG-NEXT:    ds_read_u8 v1, v0
584; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0 offset:1
585; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:2
586; ALIGNED-SDAG-NEXT:    ds_read_u8 v4, v0 offset:3
587; ALIGNED-SDAG-NEXT:    ds_read_u8 v5, v0 offset:4
588; ALIGNED-SDAG-NEXT:    ds_read_u8 v6, v0 offset:5
589; ALIGNED-SDAG-NEXT:    ds_read_u8 v7, v0 offset:6
590; ALIGNED-SDAG-NEXT:    ds_read_u8 v8, v0 offset:7
591; ALIGNED-SDAG-NEXT:    ds_read_u8 v9, v0 offset:8
592; ALIGNED-SDAG-NEXT:    ds_read_u8 v10, v0 offset:9
593; ALIGNED-SDAG-NEXT:    ds_read_u8 v11, v0 offset:10
594; ALIGNED-SDAG-NEXT:    ds_read_u8 v12, v0 offset:11
595; ALIGNED-SDAG-NEXT:    ds_read_u8 v13, v0 offset:12
596; ALIGNED-SDAG-NEXT:    ds_read_u8 v14, v0 offset:13
597; ALIGNED-SDAG-NEXT:    ds_read_u8 v15, v0 offset:14
598; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:15
599; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v16, s1
600; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
601; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v13 offset:12
602; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
603; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v14 offset:13
604; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v3 offset:2
605; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v4 offset:3
606; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v1
607; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v2 offset:1
608; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v5 offset:4
609; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v6 offset:5
610; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v9 offset:8
611; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v10 offset:9
612; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v7 offset:6
613; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v8 offset:7
614; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v11 offset:10
615; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v12 offset:11
616; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(14)
617; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v15 offset:14
618; ALIGNED-SDAG-NEXT:    ds_write_b8 v16, v0 offset:15
619; ALIGNED-SDAG-NEXT:    s_endpgm
620;
621; ALIGNED-GISEL-LABEL: ds16align1:
622; ALIGNED-GISEL:       ; %bb.0:
623; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
624; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
625; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
626; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
627; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0 offset:1
628; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:2
629; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:3
630; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:4
631; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:5
632; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:6
633; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:7
634; ALIGNED-GISEL-NEXT:    ds_read_u8 v9, v0 offset:8
635; ALIGNED-GISEL-NEXT:    ds_read_u8 v10, v0 offset:9
636; ALIGNED-GISEL-NEXT:    ds_read_u8 v11, v0 offset:10
637; ALIGNED-GISEL-NEXT:    ds_read_u8 v12, v0 offset:11
638; ALIGNED-GISEL-NEXT:    ds_read_u8 v13, v0 offset:12
639; ALIGNED-GISEL-NEXT:    ds_read_u8 v14, v0 offset:13
640; ALIGNED-GISEL-NEXT:    ds_read_u8 v15, v0 offset:14
641; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:15
642; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v16, s1
643; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
644; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v1
645; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v2 offset:1
646; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
647; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v3 offset:2
648; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v4 offset:3
649; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
650; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v5 offset:4
651; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v6 offset:5
652; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
653; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v7 offset:6
654; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v8 offset:7
655; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
656; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v9 offset:8
657; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v10 offset:9
658; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
659; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v11 offset:10
660; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v12 offset:11
661; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
662; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v13 offset:12
663; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v14 offset:13
664; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
665; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v15 offset:14
666; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v0 offset:15
667; ALIGNED-GISEL-NEXT:    s_endpgm
668;
669; UNALIGNED-LABEL: ds16align1:
670; UNALIGNED:       ; %bb.0:
671; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
672; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
673; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
674; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
675; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
676; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
677; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
678; UNALIGNED-NEXT:    s_endpgm
679  %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 1
680  store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 1
681  ret void
682}
683
684define amdgpu_kernel void @ds16align2(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
685; ALIGNED-SDAG-LABEL: ds16align2:
686; ALIGNED-SDAG:       ; %bb.0:
687; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
688; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
689; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
690; ALIGNED-SDAG-NEXT:    ds_read_u16 v2, v0
691; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:2
692; ALIGNED-SDAG-NEXT:    ds_read_u16 v4, v0 offset:4
693; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:6
694; ALIGNED-SDAG-NEXT:    ds_read_u16 v6, v0 offset:8
695; ALIGNED-SDAG-NEXT:    ds_read_u16 v7, v0 offset:10
696; ALIGNED-SDAG-NEXT:    ds_read_u16 v8, v0 offset:12
697; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:14
698; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
699; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
700; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v8 offset:12
701; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v3 offset:2
702; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v2
703; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v4 offset:4
704; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v6 offset:8
705; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v5 offset:6
706; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v7 offset:10
707; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
708; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v0 offset:14
709; ALIGNED-SDAG-NEXT:    s_endpgm
710;
711; ALIGNED-GISEL-LABEL: ds16align2:
712; ALIGNED-GISEL:       ; %bb.0:
713; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
714; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
715; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
716; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0
717; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:2
718; ALIGNED-GISEL-NEXT:    ds_read_u16 v4, v0 offset:4
719; ALIGNED-GISEL-NEXT:    ds_read_u16 v5, v0 offset:6
720; ALIGNED-GISEL-NEXT:    ds_read_u16 v6, v0 offset:8
721; ALIGNED-GISEL-NEXT:    ds_read_u16 v7, v0 offset:10
722; ALIGNED-GISEL-NEXT:    ds_read_u16 v8, v0 offset:12
723; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:14
724; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v1, s1
725; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
726; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v2
727; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
728; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v3 offset:2
729; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
730; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v4 offset:4
731; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
732; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v5 offset:6
733; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
734; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v6 offset:8
735; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
736; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v7 offset:10
737; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
738; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v8 offset:12
739; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
740; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v0 offset:14
741; ALIGNED-GISEL-NEXT:    s_endpgm
742;
743; UNALIGNED-LABEL: ds16align2:
744; UNALIGNED:       ; %bb.0:
745; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
746; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
747; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
748; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
749; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
750; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
751; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
752; UNALIGNED-NEXT:    s_endpgm
753  %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 2
754  store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 2
755  ret void
756}
757
758define amdgpu_kernel void @ds16align4(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
759; ALIGNED-SDAG-LABEL: ds16align4:
760; ALIGNED-SDAG:       ; %bb.0:
761; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
762; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
763; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v2, s0
764; ALIGNED-SDAG-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
765; ALIGNED-SDAG-NEXT:    ds_read2_b32 v[2:3], v2 offset0:2 offset1:3
766; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v4, s1
767; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
768; ALIGNED-SDAG-NEXT:    ds_write2_b32 v4, v2, v3 offset0:2 offset1:3
769; ALIGNED-SDAG-NEXT:    ds_write2_b32 v4, v0, v1 offset1:1
770; ALIGNED-SDAG-NEXT:    s_endpgm
771;
772; ALIGNED-GISEL-LABEL: ds16align4:
773; ALIGNED-GISEL:       ; %bb.0:
774; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
775; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
776; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s0
777; ALIGNED-GISEL-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
778; ALIGNED-GISEL-NEXT:    ds_read2_b32 v[2:3], v2 offset0:2 offset1:3
779; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v4, s1
780; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
781; ALIGNED-GISEL-NEXT:    ds_write2_b32 v4, v0, v1 offset1:1
782; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
783; ALIGNED-GISEL-NEXT:    ds_write2_b32 v4, v2, v3 offset0:2 offset1:3
784; ALIGNED-GISEL-NEXT:    s_endpgm
785;
786; UNALIGNED-LABEL: ds16align4:
787; UNALIGNED:       ; %bb.0:
788; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
789; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
790; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
791; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
792; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
793; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
794; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
795; UNALIGNED-NEXT:    s_endpgm
796  %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 4
797  store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 4
798  ret void
799}
800
801define amdgpu_kernel void @ds16align8(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
802; GCN-LABEL: ds16align8:
803; GCN:       ; %bb.0:
804; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
805; GCN-NEXT:    s_waitcnt lgkmcnt(0)
806; GCN-NEXT:    v_mov_b32_e32 v0, s0
807; GCN-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
808; GCN-NEXT:    v_mov_b32_e32 v4, s1
809; GCN-NEXT:    s_waitcnt lgkmcnt(0)
810; GCN-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
811; GCN-NEXT:    s_endpgm
812  %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 8
813  store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 8
814  ret void
815}
816
817define amdgpu_kernel void @ds16align16(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
818; GCN-LABEL: ds16align16:
819; GCN:       ; %bb.0:
820; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
821; GCN-NEXT:    s_waitcnt lgkmcnt(0)
822; GCN-NEXT:    v_mov_b32_e32 v0, s0
823; GCN-NEXT:    ds_read_b128 v[0:3], v0
824; GCN-NEXT:    v_mov_b32_e32 v4, s1
825; GCN-NEXT:    s_waitcnt lgkmcnt(0)
826; GCN-NEXT:    ds_write_b128 v4, v[0:3]
827; GCN-NEXT:    s_endpgm
828  %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 16
829  store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 16
830  ret void
831}
832