1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
3
4define <4 x half> @shuffle_v4f16_23uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
5; GFX9-LABEL: shuffle_v4f16_23uu:
6; GFX9:       ; %bb.0:
7; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
9; GFX9-NEXT:    s_waitcnt vmcnt(0)
10; GFX9-NEXT:    s_setpc_b64 s[30:31]
11  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
12  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
13  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
14  ret <4 x half> %shuffle
15}
16
17define <4 x half> @shuffle_v4f16_234u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
18; GFX9-LABEL: shuffle_v4f16_234u:
19; GFX9:       ; %bb.0:
20; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
21; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
22; GFX9-NEXT:    global_load_dword v6, v[0:1], off offset:4
23; GFX9-NEXT:    s_waitcnt vmcnt(1)
24; GFX9-NEXT:    v_mov_b32_e32 v1, v4
25; GFX9-NEXT:    s_waitcnt vmcnt(0)
26; GFX9-NEXT:    v_mov_b32_e32 v0, v6
27; GFX9-NEXT:    s_setpc_b64 s[30:31]
28  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
29  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
30  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef>
31  ret <4 x half> %shuffle
32}
33
34define <4 x half> @shuffle_v4f16_u1u3(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
35; GFX9-LABEL: shuffle_v4f16_u1u3:
36; GFX9:       ; %bb.0:
37; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
38; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
39; GFX9-NEXT:    s_waitcnt vmcnt(0)
40; GFX9-NEXT:    s_setpc_b64 s[30:31]
41  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
42  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
43  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
44  ret <4 x half> %shuffle
45}
46
47define <4 x half> @shuffle_v4f16_u3u1(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
48; GFX9-LABEL: shuffle_v4f16_u3u1:
49; GFX9:       ; %bb.0:
50; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
51; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[0:1], off
52; GFX9-NEXT:    s_waitcnt vmcnt(0)
53; GFX9-NEXT:    v_mov_b32_e32 v0, v2
54; GFX9-NEXT:    s_setpc_b64 s[30:31]
55  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
56  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
57  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1>
58  ret <4 x half> %shuffle
59}
60
61define <4 x half> @shuffle_v4f16_u3uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
62; GFX9-LABEL: shuffle_v4f16_u3uu:
63; GFX9:       ; %bb.0:
64; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
65; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
66; GFX9-NEXT:    s_waitcnt vmcnt(0)
67; GFX9-NEXT:    s_setpc_b64 s[30:31]
68  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
69  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
70  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
71  ret <4 x half> %shuffle
72}
73
74define <4 x half> @shuffle_v4f16_3u6u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
75; GFX9-LABEL: shuffle_v4f16_3u6u:
76; GFX9:       ; %bb.0:
77; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
78; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
79; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
80; GFX9-NEXT:    s_waitcnt vmcnt(1)
81; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v5
82; GFX9-NEXT:    s_waitcnt vmcnt(0)
83; GFX9-NEXT:    v_mov_b32_e32 v1, v4
84; GFX9-NEXT:    s_setpc_b64 s[30:31]
85  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
86  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
87  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef>
88  ret <4 x half> %shuffle
89}
90
91define <4 x half> @shuffle_v4f16_3uu7(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
92; GFX9-LABEL: shuffle_v4f16_3uu7:
93; GFX9:       ; %bb.0:
94; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
95; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
96; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
97; GFX9-NEXT:    s_waitcnt vmcnt(1)
98; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v5
99; GFX9-NEXT:    s_waitcnt vmcnt(0)
100; GFX9-NEXT:    v_mov_b32_e32 v1, v4
101; GFX9-NEXT:    s_setpc_b64 s[30:31]
102  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
103  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
104  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7>
105  ret <4 x half> %shuffle
106}
107
108define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
109; GFX9-LABEL: shuffle_v4f16_35u5:
110; GFX9:       ; %bb.0:
111; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
112; GFX9-NEXT:    global_load_dword v4, v[2:3], off
113; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
114; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
115; GFX9-NEXT:    s_waitcnt vmcnt(1)
116; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v4
117; GFX9-NEXT:    s_waitcnt vmcnt(0)
118; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
119; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
120; GFX9-NEXT:    v_mov_b32_e32 v1, v4
121; GFX9-NEXT:    s_setpc_b64 s[30:31]
122  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
123  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
124  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5>
125  ret <4 x half> %shuffle
126}
127
128define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
129; GFX9-LABEL: shuffle_v4f16_357u:
130; GFX9:       ; %bb.0:
131; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
132; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
133; GFX9-NEXT:    global_load_dword v6, v[0:1], off offset:4
134; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
135; GFX9-NEXT:    s_waitcnt vmcnt(1)
136; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
137; GFX9-NEXT:    s_waitcnt vmcnt(0)
138; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
139; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v5
140; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
141; GFX9-NEXT:    s_setpc_b64 s[30:31]
142  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
143  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
144  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef>
145  ret <4 x half> %shuffle
146}
147
148define <4 x half> @shuffle_v4f16_0101(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
149; GFX9-LABEL: shuffle_v4f16_0101:
150; GFX9:       ; %bb.0:
151; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
152; GFX9-NEXT:    global_load_dword v0, v[0:1], off
153; GFX9-NEXT:    s_waitcnt vmcnt(0)
154; GFX9-NEXT:    v_mov_b32_e32 v1, v0
155; GFX9-NEXT:    s_setpc_b64 s[30:31]
156  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
157  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
158  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
159  ret <4 x half> %shuffle
160}
161
162define <4 x half> @shuffle_v4f16_0123(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
163; GFX9-LABEL: shuffle_v4f16_0123:
164; GFX9:       ; %bb.0:
165; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
166; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
167; GFX9-NEXT:    s_waitcnt vmcnt(0)
168; GFX9-NEXT:    s_setpc_b64 s[30:31]
169  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
170  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
171  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
172  ret <4 x half> %shuffle
173}
174
175define <4 x half> @shuffle_v4f16_0145(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
176; GFX9-LABEL: shuffle_v4f16_0145:
177; GFX9:       ; %bb.0:
178; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
179; GFX9-NEXT:    global_load_dword v4, v[0:1], off
180; GFX9-NEXT:    global_load_dword v5, v[2:3], off
181; GFX9-NEXT:    s_waitcnt vmcnt(1)
182; GFX9-NEXT:    v_mov_b32_e32 v0, v4
183; GFX9-NEXT:    s_waitcnt vmcnt(0)
184; GFX9-NEXT:    v_mov_b32_e32 v1, v5
185; GFX9-NEXT:    s_setpc_b64 s[30:31]
186  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
187  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
188  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
189  ret <4 x half> %shuffle
190}
191
192define <4 x half> @shuffle_v4f16_0167(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
193; GFX9-LABEL: shuffle_v4f16_0167:
194; GFX9:       ; %bb.0:
195; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
196; GFX9-NEXT:    global_load_dword v4, v[0:1], off
197; GFX9-NEXT:    global_load_dword v5, v[2:3], off offset:4
198; GFX9-NEXT:    s_waitcnt vmcnt(1)
199; GFX9-NEXT:    v_mov_b32_e32 v0, v4
200; GFX9-NEXT:    s_waitcnt vmcnt(0)
201; GFX9-NEXT:    v_mov_b32_e32 v1, v5
202; GFX9-NEXT:    s_setpc_b64 s[30:31]
203  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
204  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
205  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
206  ret <4 x half> %shuffle
207}
208
209define <4 x half> @shuffle_v4f16_2301(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
210; GFX9-LABEL: shuffle_v4f16_2301:
211; GFX9:       ; %bb.0:
212; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
213; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[0:1], off
214; GFX9-NEXT:    s_waitcnt vmcnt(0)
215; GFX9-NEXT:    v_mov_b32_e32 v0, v2
216; GFX9-NEXT:    s_setpc_b64 s[30:31]
217  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
218  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
219  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
220  ret <4 x half> %shuffle
221}
222
223define <4 x half> @shuffle_v4f16_2323(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
224; GFX9-LABEL: shuffle_v4f16_2323:
225; GFX9:       ; %bb.0:
226; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
227; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
228; GFX9-NEXT:    s_waitcnt vmcnt(0)
229; GFX9-NEXT:    v_mov_b32_e32 v1, v0
230; GFX9-NEXT:    s_setpc_b64 s[30:31]
231  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
232  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
233  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
234  ret <4 x half> %shuffle
235}
236
237define <4 x half> @shuffle_v4f16_2345(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
238; GFX9-LABEL: shuffle_v4f16_2345:
239; GFX9:       ; %bb.0:
240; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
241; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
242; GFX9-NEXT:    global_load_dword v5, v[2:3], off
243; GFX9-NEXT:    s_waitcnt vmcnt(1)
244; GFX9-NEXT:    v_mov_b32_e32 v0, v4
245; GFX9-NEXT:    s_waitcnt vmcnt(0)
246; GFX9-NEXT:    v_mov_b32_e32 v1, v5
247; GFX9-NEXT:    s_setpc_b64 s[30:31]
248  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
249  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
250  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
251  ret <4 x half> %shuffle
252}
253
254define <4 x half> @shuffle_v4f16_2367(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
255; GFX9-LABEL: shuffle_v4f16_2367:
256; GFX9:       ; %bb.0:
257; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
258; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
259; GFX9-NEXT:    global_load_dword v5, v[2:3], off offset:4
260; GFX9-NEXT:    s_waitcnt vmcnt(1)
261; GFX9-NEXT:    v_mov_b32_e32 v0, v4
262; GFX9-NEXT:    s_waitcnt vmcnt(0)
263; GFX9-NEXT:    v_mov_b32_e32 v1, v5
264; GFX9-NEXT:    s_setpc_b64 s[30:31]
265  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
266  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
267  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
268  ret <4 x half> %shuffle
269}
270
271define <4 x half> @shuffle_v4f16_4501(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
272; GFX9-LABEL: shuffle_v4f16_4501:
273; GFX9:       ; %bb.0:
274; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
275; GFX9-NEXT:    global_load_dword v4, v[2:3], off
276; GFX9-NEXT:    global_load_dword v5, v[0:1], off
277; GFX9-NEXT:    s_waitcnt vmcnt(1)
278; GFX9-NEXT:    v_mov_b32_e32 v0, v4
279; GFX9-NEXT:    s_waitcnt vmcnt(0)
280; GFX9-NEXT:    v_mov_b32_e32 v1, v5
281; GFX9-NEXT:    s_setpc_b64 s[30:31]
282  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
283  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
284  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
285  ret <4 x half> %shuffle
286}
287
288define <4 x half> @shuffle_v4f16_4523(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
289; GFX9-LABEL: shuffle_v4f16_4523:
290; GFX9:       ; %bb.0:
291; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
292; GFX9-NEXT:    global_load_dword v4, v[2:3], off
293; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
294; GFX9-NEXT:    s_waitcnt vmcnt(1)
295; GFX9-NEXT:    v_mov_b32_e32 v0, v4
296; GFX9-NEXT:    s_waitcnt vmcnt(0)
297; GFX9-NEXT:    v_mov_b32_e32 v1, v5
298; GFX9-NEXT:    s_setpc_b64 s[30:31]
299  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
300  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
301  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
302  ret <4 x half> %shuffle
303}
304
305define <4 x half> @shuffle_v4f16_4545(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
306; GFX9-LABEL: shuffle_v4f16_4545:
307; GFX9:       ; %bb.0:
308; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
309; GFX9-NEXT:    global_load_dword v0, v[2:3], off
310; GFX9-NEXT:    s_waitcnt vmcnt(0)
311; GFX9-NEXT:    v_mov_b32_e32 v1, v0
312; GFX9-NEXT:    s_setpc_b64 s[30:31]
313  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
314  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
315  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5>
316  ret <4 x half> %shuffle
317}
318
319define <4 x half> @shuffle_v4f16_4567(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
320; GFX9-LABEL: shuffle_v4f16_4567:
321; GFX9:       ; %bb.0:
322; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
323; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
324; GFX9-NEXT:    s_waitcnt vmcnt(0)
325; GFX9-NEXT:    s_setpc_b64 s[30:31]
326  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
327  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
328  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
329  ret <4 x half> %shuffle
330}
331
332define <4 x half> @shuffle_v4f16_6701(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
333; GFX9-LABEL: shuffle_v4f16_6701:
334; GFX9:       ; %bb.0:
335; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
336; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
337; GFX9-NEXT:    global_load_dword v5, v[0:1], off
338; GFX9-NEXT:    s_waitcnt vmcnt(1)
339; GFX9-NEXT:    v_mov_b32_e32 v0, v4
340; GFX9-NEXT:    s_waitcnt vmcnt(0)
341; GFX9-NEXT:    v_mov_b32_e32 v1, v5
342; GFX9-NEXT:    s_setpc_b64 s[30:31]
343  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
344  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
345  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
346  ret <4 x half> %shuffle
347}
348
349define <4 x half> @shuffle_v4f16_6723(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
350; GFX9-LABEL: shuffle_v4f16_6723:
351; GFX9:       ; %bb.0:
352; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
353; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
354; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
355; GFX9-NEXT:    s_waitcnt vmcnt(1)
356; GFX9-NEXT:    v_mov_b32_e32 v0, v4
357; GFX9-NEXT:    s_waitcnt vmcnt(0)
358; GFX9-NEXT:    v_mov_b32_e32 v1, v5
359; GFX9-NEXT:    s_setpc_b64 s[30:31]
360  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
361  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
362  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
363  ret <4 x half> %shuffle
364}
365
366define <4 x half> @shuffle_v4f16_6745(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
367; GFX9-LABEL: shuffle_v4f16_6745:
368; GFX9:       ; %bb.0:
369; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
370; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[2:3], off
371; GFX9-NEXT:    s_waitcnt vmcnt(0)
372; GFX9-NEXT:    v_mov_b32_e32 v0, v2
373; GFX9-NEXT:    s_setpc_b64 s[30:31]
374  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
375  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
376  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5>
377  ret <4 x half> %shuffle
378}
379
380define <4 x half> @shuffle_v4f16_6767(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
381; GFX9-LABEL: shuffle_v4f16_6767:
382; GFX9:       ; %bb.0:
383; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
384; GFX9-NEXT:    global_load_dword v0, v[2:3], off offset:4
385; GFX9-NEXT:    s_waitcnt vmcnt(0)
386; GFX9-NEXT:    v_mov_b32_e32 v1, v0
387; GFX9-NEXT:    s_setpc_b64 s[30:31]
388  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
389  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
390  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7>
391  ret <4 x half> %shuffle
392}
393
394define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
395; GFX9-LABEL: shuffle_v4f16_2356:
396; GFX9:       ; %bb.0:
397; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
398; GFX9-NEXT:    global_load_dwordx2 v[5:6], v[2:3], off
399; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
400; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
401; GFX9-NEXT:    s_waitcnt vmcnt(1)
402; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
403; GFX9-NEXT:    v_lshl_or_b32 v1, v6, 16, v0
404; GFX9-NEXT:    s_waitcnt vmcnt(0)
405; GFX9-NEXT:    v_mov_b32_e32 v0, v4
406; GFX9-NEXT:    s_setpc_b64 s[30:31]
407  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
408  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
409  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
410  ret <4 x half> %shuffle
411}
412
413define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
414; GFX9-LABEL: shuffle_v4f16_5623:
415; GFX9:       ; %bb.0:
416; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
417; GFX9-NEXT:    global_load_dwordx2 v[5:6], v[2:3], off
418; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
419; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
420; GFX9-NEXT:    s_waitcnt vmcnt(1)
421; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
422; GFX9-NEXT:    v_lshl_or_b32 v0, v6, 16, v0
423; GFX9-NEXT:    s_waitcnt vmcnt(0)
424; GFX9-NEXT:    v_mov_b32_e32 v1, v4
425; GFX9-NEXT:    s_setpc_b64 s[30:31]
426  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
427  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
428  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
429  ret <4 x half> %shuffle
430}
431
432define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
433; GFX9-LABEL: shuffle_v4f16_3456:
434; GFX9:       ; %bb.0:
435; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
436; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
437; GFX9-NEXT:    global_load_dword v6, v[0:1], off offset:4
438; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
439; GFX9-NEXT:    s_waitcnt vmcnt(1)
440; GFX9-NEXT:    v_and_b32_sdwa v2, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
441; GFX9-NEXT:    s_waitcnt vmcnt(0)
442; GFX9-NEXT:    v_and_b32_sdwa v1, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
443; GFX9-NEXT:    v_lshl_or_b32 v0, v4, 16, v1
444; GFX9-NEXT:    v_lshl_or_b32 v1, v5, 16, v2
445; GFX9-NEXT:    s_setpc_b64 s[30:31]
446  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
447  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
448  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
449  ret <4 x half> %shuffle
450}
451
452define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
453; GFX9-LABEL: shuffle_v4f16_5634:
454; GFX9:       ; %bb.0:
455; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
456; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
457; GFX9-NEXT:    global_load_dword v6, v[0:1], off offset:4
458; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
459; GFX9-NEXT:    s_waitcnt vmcnt(0)
460; GFX9-NEXT:    v_and_b32_sdwa v1, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
461; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
462; GFX9-NEXT:    v_lshl_or_b32 v1, v4, 16, v1
463; GFX9-NEXT:    v_lshl_or_b32 v0, v5, 16, v0
464; GFX9-NEXT:    s_setpc_b64 s[30:31]
465  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
466  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
467  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4>
468  ret <4 x half> %shuffle
469}
470
471define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
472; GFX9-LABEL: shuffle_v4f16_5734:
473; GFX9:       ; %bb.0:
474; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
475; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
476; GFX9-NEXT:    global_load_dword v6, v[0:1], off offset:4
477; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
478; GFX9-NEXT:    s_waitcnt vmcnt(1)
479; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v5
480; GFX9-NEXT:    s_waitcnt vmcnt(0)
481; GFX9-NEXT:    v_and_b32_sdwa v1, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
482; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
483; GFX9-NEXT:    v_lshl_or_b32 v1, v4, 16, v1
484; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
485; GFX9-NEXT:    s_setpc_b64 s[30:31]
486  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
487  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
488  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4>
489  ret <4 x half> %shuffle
490}
491
492define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) {
493; GFX9-LABEL: shuffle_v4i16_2356:
494; GFX9:       ; %bb.0:
495; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
496; GFX9-NEXT:    global_load_dwordx2 v[5:6], v[2:3], off
497; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
498; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
499; GFX9-NEXT:    s_waitcnt vmcnt(1)
500; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
501; GFX9-NEXT:    v_lshl_or_b32 v1, v6, 16, v0
502; GFX9-NEXT:    s_waitcnt vmcnt(0)
503; GFX9-NEXT:    v_mov_b32_e32 v0, v4
504; GFX9-NEXT:    s_setpc_b64 s[30:31]
505  %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
506  %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
507  %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
508  ret <4 x i16> %shuffle
509}
510
511define <4 x i16> @shuffle_v4i16_0167(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) {
512; GFX9-LABEL: shuffle_v4i16_0167:
513; GFX9:       ; %bb.0:
514; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
515; GFX9-NEXT:    global_load_dword v4, v[0:1], off
516; GFX9-NEXT:    global_load_dword v5, v[2:3], off offset:4
517; GFX9-NEXT:    s_waitcnt vmcnt(1)
518; GFX9-NEXT:    v_mov_b32_e32 v0, v4
519; GFX9-NEXT:    s_waitcnt vmcnt(0)
520; GFX9-NEXT:    v_mov_b32_e32 v1, v5
521; GFX9-NEXT:    s_setpc_b64 s[30:31]
522  %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
523  %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
524  %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
525  ret <4 x i16> %shuffle
526}
527
528define <4 x half> @shuffle_v4f16_0000(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
529; GFX9-LABEL: shuffle_v4f16_0000:
530; GFX9:       ; %bb.0:
531; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
532; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
533; GFX9-NEXT:    s_waitcnt vmcnt(0)
534; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v0
535; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
536; GFX9-NEXT:    v_mov_b32_e32 v1, v0
537; GFX9-NEXT:    s_setpc_b64 s[30:31]
538  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
539  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
540  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer
541  ret <4 x half> %shuffle
542}
543
544define <4 x half> @shuffle_v4f16_1010(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
545; GFX9-LABEL: shuffle_v4f16_1010:
546; GFX9:       ; %bb.0:
547; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
548; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
549; GFX9-NEXT:    s_waitcnt vmcnt(0)
550; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
551; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
552; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
553; GFX9-NEXT:    v_mov_b32_e32 v1, v0
554; GFX9-NEXT:    s_setpc_b64 s[30:31]
555  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
556  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
557  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
558  ret <4 x half> %shuffle
559}
560
561define <4 x half> @shuffle_v4f16_1100(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
562; GFX9-LABEL: shuffle_v4f16_1100:
563; GFX9:       ; %bb.0:
564; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
565; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
566; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
567; GFX9-NEXT:    s_waitcnt vmcnt(0)
568; GFX9-NEXT:    v_and_b32_e32 v1, v2, v0
569; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
570; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
571; GFX9-NEXT:    v_and_b32_e32 v0, v2, v3
572; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
573; GFX9-NEXT:    s_setpc_b64 s[30:31]
574  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
575  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
576  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
577  ret <4 x half> %shuffle
578}
579
580define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
581; GFX9-LABEL: shuffle_v4f16_6161:
582; GFX9:       ; %bb.0:
583; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
584; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
585; GFX9-NEXT:    global_load_dword v5, v[0:1], off
586; GFX9-NEXT:    s_waitcnt vmcnt(1)
587; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v4
588; GFX9-NEXT:    s_waitcnt vmcnt(0)
589; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v5
590; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
591; GFX9-NEXT:    v_mov_b32_e32 v1, v0
592; GFX9-NEXT:    s_setpc_b64 s[30:31]
593  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
594  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
595  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1>
596  ret <4 x half> %shuffle
597}
598
599define <4 x half> @shuffle_v4f16_2333(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
600; GFX9-LABEL: shuffle_v4f16_2333:
601; GFX9:       ; %bb.0:
602; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
603; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
604; GFX9-NEXT:    s_waitcnt vmcnt(0)
605; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
606; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
607; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
608; GFX9-NEXT:    s_setpc_b64 s[30:31]
609  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
610  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
611  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
612  ret <4 x half> %shuffle
613}
614
615define <4 x half> @shuffle_v4f16_6667(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
616; GFX9-LABEL: shuffle_v4f16_6667:
617; GFX9:       ; %bb.0:
618; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
619; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
620; GFX9-NEXT:    s_waitcnt vmcnt(0)
621; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
622; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
623; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
624; GFX9-NEXT:    s_setpc_b64 s[30:31]
625  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
626  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
627  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
628  ret <4 x half> %shuffle
629}
630
631define <4 x half> @shuffle_v8f16_0101(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
632; GFX9-LABEL: shuffle_v8f16_0101:
633; GFX9:       ; %bb.0:
634; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
635; GFX9-NEXT:    global_load_dword v0, v[0:1], off
636; GFX9-NEXT:    s_waitcnt vmcnt(0)
637; GFX9-NEXT:    v_mov_b32_e32 v1, v0
638; GFX9-NEXT:    s_setpc_b64 s[30:31]
639  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
640  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
641  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
642  ret <4 x half> %shuffle
643}
644
645define <4 x half> @shuffle_v8f16_0123(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
646; GFX9-LABEL: shuffle_v8f16_0123:
647; GFX9:       ; %bb.0:
648; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
649; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
650; GFX9-NEXT:    s_waitcnt vmcnt(0)
651; GFX9-NEXT:    s_setpc_b64 s[30:31]
652  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
653  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
654  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
655  ret <4 x half> %shuffle
656}
657
658define <4 x half> @shuffle_v8f16_4589(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
659; GFX9-LABEL: shuffle_v8f16_4589:
660; GFX9:       ; %bb.0:
661; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
662; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:8
663; GFX9-NEXT:    global_load_dword v5, v[2:3], off
664; GFX9-NEXT:    s_waitcnt vmcnt(1)
665; GFX9-NEXT:    v_mov_b32_e32 v0, v4
666; GFX9-NEXT:    s_waitcnt vmcnt(0)
667; GFX9-NEXT:    v_mov_b32_e32 v1, v5
668; GFX9-NEXT:    s_setpc_b64 s[30:31]
669  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
670  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
671  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9>
672  ret <4 x half> %shuffle
673}
674
675define <4 x half> @shuffle_v8f16_10_11_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
676; GFX9-LABEL: shuffle_v8f16_10_11_2_3:
677; GFX9:       ; %bb.0:
678; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
679; GFX9-NEXT:    global_load_dword v4, v[2:3], off offset:4
680; GFX9-NEXT:    global_load_dword v5, v[0:1], off offset:4
681; GFX9-NEXT:    s_waitcnt vmcnt(1)
682; GFX9-NEXT:    v_mov_b32_e32 v0, v4
683; GFX9-NEXT:    s_waitcnt vmcnt(0)
684; GFX9-NEXT:    v_mov_b32_e32 v1, v5
685; GFX9-NEXT:    s_setpc_b64 s[30:31]
686  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
687  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
688  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3>
689  ret <4 x half> %shuffle
690}
691
692define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
693; GFX9-LABEL: shuffle_v8f16_13_14_2_3:
694; GFX9:       ; %bb.0:
695; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
696; GFX9-NEXT:    global_load_dwordx2 v[5:6], v[2:3], off offset:8
697; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
698; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
699; GFX9-NEXT:    s_waitcnt vmcnt(1)
700; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
701; GFX9-NEXT:    v_lshl_or_b32 v0, v6, 16, v0
702; GFX9-NEXT:    s_waitcnt vmcnt(0)
703; GFX9-NEXT:    v_mov_b32_e32 v1, v4
704; GFX9-NEXT:    s_setpc_b64 s[30:31]
705  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
706  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
707  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
708  ret <4 x half> %shuffle
709}
710
711define <4 x half> @shuffle_v3f16_0122(<3 x half> addrspace(1)* %arg0, <3 x half> addrspace(1)* %arg1) {
712; GFX9-LABEL: shuffle_v3f16_0122:
713; GFX9:       ; %bb.0:
714; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
715; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
716; GFX9-NEXT:    s_waitcnt vmcnt(0)
717; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
718; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
719; GFX9-NEXT:    s_setpc_b64 s[30:31]
720  %val0 = load <3 x half>, <3 x half> addrspace(1)* %arg0
721  %val1 = load <3 x half>, <3 x half> addrspace(1)* %arg1
722  %shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
723  ret <4 x half> %shuffle
724}
725
726define <4 x half> @shuffle_v2f16_0122(<2 x half> addrspace(1)* %arg0, <2 x half> addrspace(1)* %arg1) {
727; GFX9-LABEL: shuffle_v2f16_0122:
728; GFX9:       ; %bb.0:
729; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
730; GFX9-NEXT:    global_load_dword v0, v[0:1], off
731; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
732; GFX9-NEXT:    s_waitcnt vmcnt(0)
733; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
734; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
735; GFX9-NEXT:    s_setpc_b64 s[30:31]
736  %val0 = load <2 x half>, <2 x half> addrspace(1)* %arg0
737  %val1 = load <2 x half>, <2 x half> addrspace(1)* %arg1
738  %shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
739  ret <4 x half> %shuffle
740}
741
742define <6 x half> @shuffle_v6f16_452367(<6 x half> addrspace(1)* %arg0, <6 x half> addrspace(1)* %arg1) {
743; GFX9-LABEL: shuffle_v6f16_452367:
744; GFX9:       ; %bb.0:
745; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
746; GFX9-NEXT:    v_mov_b32_e32 v6, v1
747; GFX9-NEXT:    v_mov_b32_e32 v5, v0
748; GFX9-NEXT:    v_mov_b32_e32 v4, v3
749; GFX9-NEXT:    v_mov_b32_e32 v3, v2
750; GFX9-NEXT:    global_load_dwordx3 v[0:2], v[5:6], off
751; GFX9-NEXT:    global_load_dword v7, v[3:4], off
752; GFX9-NEXT:    s_waitcnt vmcnt(1)
753; GFX9-NEXT:    v_mov_b32_e32 v0, v2
754; GFX9-NEXT:    s_waitcnt vmcnt(0)
755; GFX9-NEXT:    v_mov_b32_e32 v2, v7
756; GFX9-NEXT:    s_setpc_b64 s[30:31]
757  %val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0
758  %val1 = load <6 x half>, <6 x half> addrspace(1)* %arg1
759  %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
760  ret <6 x half> %shuffle
761}
762
763define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readonly %A, <4 x half> addrspace(1)* nocapture readonly %B, <4 x half> addrspace(1)* nocapture %C)  {
764; GFX9-LABEL: fma_shuffle:
765; GFX9:       ; %bb.0: ; %entry
766; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
767; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
768; GFX9-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
769; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
770; GFX9-NEXT:    global_load_dwordx2 v[0:1], v6, s[0:1]
771; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[2:3]
772; GFX9-NEXT:    global_load_dwordx2 v[4:5], v6, s[8:9]
773; GFX9-NEXT:    s_waitcnt vmcnt(0)
774; GFX9-NEXT:    v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1]
775; GFX9-NEXT:    v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1]
776; GFX9-NEXT:    v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0]
777; GFX9-NEXT:    v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
778; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[8:9]
779; GFX9-NEXT:    s_endpgm
780entry:
781  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x()
782  %tmp12 = zext i32 %tmp1 to i64
783  %arrayidx = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %A, i64 %tmp12
784  %tmp14 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx, align 8
785  %arrayidx1 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %B, i64 %tmp12
786  %tmp15 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx1, align 8
787  %arrayidx2 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %C, i64 %tmp12
788  %tmp16 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx2, align 8
789  %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> zeroinitializer
790  %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 0, i32 1>
791  %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> undef, <2 x i32> <i32 0, i32 1>
792  %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19)
793  %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 1, i32 1>
794  %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 2, i32 3>
795  %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20)
796  %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
797  %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
798  %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 2, i32 2>
799  %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> undef, <2 x i32> <i32 2, i32 3>
800  %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27)
801  %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 3, i32 3>
802  %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28)
803  %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
804  %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
805  store <4 x half> %tmp32, <4 x half> addrspace(1)* %arrayidx2, align 8
806  ret void
807}
808
809define <4 x half> @shuffle_v4f16_0456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
810; GFX9-LABEL: shuffle_v4f16_0456:
811; GFX9:       ; %bb.0:
812; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
813; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
814; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[2:3], off
815; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
816; GFX9-NEXT:    s_waitcnt vmcnt(1)
817; GFX9-NEXT:    v_and_b32_e32 v1, v0, v4
818; GFX9-NEXT:    s_waitcnt vmcnt(0)
819; GFX9-NEXT:    v_and_b32_sdwa v2, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
820; GFX9-NEXT:    v_lshl_or_b32 v0, v6, 16, v1
821; GFX9-NEXT:    v_lshl_or_b32 v1, v7, 16, v2
822; GFX9-NEXT:    s_setpc_b64 s[30:31]
823  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
824  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
825  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
826  ret <4 x half> %shuffle
827}
828
829define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(<8 x i32> addrspace(4)* %in, <4 x i32> addrspace(1)* %out)  {
830; GFX9-LABEL: shuffle_scalar_load_v8i32_0123:
831; GFX9:       ; %bb.0:
832; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
833; GFX9-NEXT:    v_mov_b32_e32 v4, 0
834; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
836; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
837; GFX9-NEXT:    v_mov_b32_e32 v0, s4
838; GFX9-NEXT:    v_mov_b32_e32 v1, s5
839; GFX9-NEXT:    v_mov_b32_e32 v2, s6
840; GFX9-NEXT:    v_mov_b32_e32 v3, s7
841; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
842; GFX9-NEXT:    s_endpgm
843  %ld8 = load <8 x i32>, <8 x i32> addrspace(4)* %in, align 16
844  %id = shufflevector <8 x i32> %ld8, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
845  store <4 x i32> %id, <4 x i32> addrspace(1)* %out, align 8
846  ret void
847}
848
849declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #0
850declare i32 @llvm.amdgcn.workitem.id.x() #0
851
852attributes #0 = { nounwind readnone speculatable }
853