1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
3
4define <4 x half> @shuffle_v4f16_23uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
5; GFX9-LABEL: shuffle_v4f16_23uu:
6; GFX9:       ; %bb.0:
7; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
9; GFX9-NEXT:    s_waitcnt vmcnt(0)
10; GFX9-NEXT:    s_setpc_b64 s[30:31]
11  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
12  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
13  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
14  ret <4 x half> %shuffle
15}
16
17define <4 x half> @shuffle_v4f16_234u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
18; GFX9-LABEL: shuffle_v4f16_234u:
19; GFX9:       ; %bb.0:
20; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
21; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
22; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
23; GFX9-NEXT:    s_waitcnt vmcnt(1)
24; GFX9-NEXT:    v_mov_b32_e32 v1, v2
25; GFX9-NEXT:    s_waitcnt vmcnt(0)
26; GFX9-NEXT:    s_setpc_b64 s[30:31]
27  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
28  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
29  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef>
30  ret <4 x half> %shuffle
31}
32
33define <4 x half> @shuffle_v4f16_u1u3(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
34; GFX9-LABEL: shuffle_v4f16_u1u3:
35; GFX9:       ; %bb.0:
36; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
37; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
38; GFX9-NEXT:    s_waitcnt vmcnt(0)
39; GFX9-NEXT:    s_setpc_b64 s[30:31]
40  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
41  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
42  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
43  ret <4 x half> %shuffle
44}
45
46define <4 x half> @shuffle_v4f16_u3u1(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
47; GFX9-LABEL: shuffle_v4f16_u3u1:
48; GFX9:       ; %bb.0:
49; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
50; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[0:1], off
51; GFX9-NEXT:    s_waitcnt vmcnt(0)
52; GFX9-NEXT:    v_mov_b32_e32 v0, v2
53; GFX9-NEXT:    s_setpc_b64 s[30:31]
54  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
55  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
56  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1>
57  ret <4 x half> %shuffle
58}
59
60define <4 x half> @shuffle_v4f16_u3uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
61; GFX9-LABEL: shuffle_v4f16_u3uu:
62; GFX9:       ; %bb.0:
63; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
64; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
65; GFX9-NEXT:    s_waitcnt vmcnt(0)
66; GFX9-NEXT:    s_setpc_b64 s[30:31]
67  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
68  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
69  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
70  ret <4 x half> %shuffle
71}
72
73define <4 x half> @shuffle_v4f16_3u6u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
74; GFX9-LABEL: shuffle_v4f16_3u6u:
75; GFX9:       ; %bb.0:
76; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
77; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
78; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
79; GFX9-NEXT:    s_waitcnt vmcnt(1)
80; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
81; GFX9-NEXT:    s_waitcnt vmcnt(0)
82; GFX9-NEXT:    s_setpc_b64 s[30:31]
83  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
84  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
85  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef>
86  ret <4 x half> %shuffle
87}
88
89define <4 x half> @shuffle_v4f16_3uu7(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
90; GFX9-LABEL: shuffle_v4f16_3uu7:
91; GFX9:       ; %bb.0:
92; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
93; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
94; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
95; GFX9-NEXT:    s_waitcnt vmcnt(1)
96; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
97; GFX9-NEXT:    s_waitcnt vmcnt(0)
98; GFX9-NEXT:    s_setpc_b64 s[30:31]
99  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
100  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
101  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7>
102  ret <4 x half> %shuffle
103}
104
105define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
106; GFX9-LABEL: shuffle_v4f16_35u5:
107; GFX9:       ; %bb.0:
108; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
109; GFX9-NEXT:    global_load_dword v2, v[2:3], off
110; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
111; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
112; GFX9-NEXT:    s_waitcnt vmcnt(1)
113; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
114; GFX9-NEXT:    s_waitcnt vmcnt(0)
115; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
116; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
117; GFX9-NEXT:    v_mov_b32_e32 v1, v2
118; GFX9-NEXT:    s_setpc_b64 s[30:31]
119  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
120  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
121  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5>
122  ret <4 x half> %shuffle
123}
124
125define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
126; GFX9-LABEL: shuffle_v4f16_357u:
127; GFX9:       ; %bb.0:
128; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
129; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
130; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
131; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
132; GFX9-NEXT:    s_waitcnt vmcnt(1)
133; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
134; GFX9-NEXT:    s_waitcnt vmcnt(0)
135; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
136; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
137; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
138; GFX9-NEXT:    s_setpc_b64 s[30:31]
139  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
140  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
141  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef>
142  ret <4 x half> %shuffle
143}
144
145define <4 x half> @shuffle_v4f16_0101(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
146; GFX9-LABEL: shuffle_v4f16_0101:
147; GFX9:       ; %bb.0:
148; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
149; GFX9-NEXT:    global_load_dword v0, v[0:1], off
150; GFX9-NEXT:    s_waitcnt vmcnt(0)
151; GFX9-NEXT:    v_mov_b32_e32 v1, v0
152; GFX9-NEXT:    s_setpc_b64 s[30:31]
153  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
154  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
155  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
156  ret <4 x half> %shuffle
157}
158
159define <4 x half> @shuffle_v4f16_0123(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
160; GFX9-LABEL: shuffle_v4f16_0123:
161; GFX9:       ; %bb.0:
162; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
163; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
164; GFX9-NEXT:    s_waitcnt vmcnt(0)
165; GFX9-NEXT:    s_setpc_b64 s[30:31]
166  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
167  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
168  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
169  ret <4 x half> %shuffle
170}
171
172define <4 x half> @shuffle_v4f16_0145(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
173; GFX9-LABEL: shuffle_v4f16_0145:
174; GFX9:       ; %bb.0:
175; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
176; GFX9-NEXT:    global_load_dword v0, v[0:1], off
177; GFX9-NEXT:    global_load_dword v1, v[2:3], off
178; GFX9-NEXT:    s_waitcnt vmcnt(0)
179; GFX9-NEXT:    s_setpc_b64 s[30:31]
180  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
181  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
182  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
183  ret <4 x half> %shuffle
184}
185
186define <4 x half> @shuffle_v4f16_0167(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
187; GFX9-LABEL: shuffle_v4f16_0167:
188; GFX9:       ; %bb.0:
189; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
190; GFX9-NEXT:    global_load_dword v0, v[0:1], off
191; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
192; GFX9-NEXT:    s_waitcnt vmcnt(0)
193; GFX9-NEXT:    s_setpc_b64 s[30:31]
194  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
195  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
196  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
197  ret <4 x half> %shuffle
198}
199
200define <4 x half> @shuffle_v4f16_2301(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
201; GFX9-LABEL: shuffle_v4f16_2301:
202; GFX9:       ; %bb.0:
203; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
204; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[0:1], off
205; GFX9-NEXT:    s_waitcnt vmcnt(0)
206; GFX9-NEXT:    v_mov_b32_e32 v0, v2
207; GFX9-NEXT:    s_setpc_b64 s[30:31]
208  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
209  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
210  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
211  ret <4 x half> %shuffle
212}
213
214define <4 x half> @shuffle_v4f16_2323(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
215; GFX9-LABEL: shuffle_v4f16_2323:
216; GFX9:       ; %bb.0:
217; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
218; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
219; GFX9-NEXT:    s_waitcnt vmcnt(0)
220; GFX9-NEXT:    v_mov_b32_e32 v1, v0
221; GFX9-NEXT:    s_setpc_b64 s[30:31]
222  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
223  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
224  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
225  ret <4 x half> %shuffle
226}
227
228define <4 x half> @shuffle_v4f16_2345(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
229; GFX9-LABEL: shuffle_v4f16_2345:
230; GFX9:       ; %bb.0:
231; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
232; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
233; GFX9-NEXT:    global_load_dword v1, v[2:3], off
234; GFX9-NEXT:    s_waitcnt vmcnt(0)
235; GFX9-NEXT:    s_setpc_b64 s[30:31]
236  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
237  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
238  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
239  ret <4 x half> %shuffle
240}
241
242define <4 x half> @shuffle_v4f16_2367(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
243; GFX9-LABEL: shuffle_v4f16_2367:
244; GFX9:       ; %bb.0:
245; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
246; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
247; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
248; GFX9-NEXT:    s_waitcnt vmcnt(0)
249; GFX9-NEXT:    s_setpc_b64 s[30:31]
250  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
251  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
252  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
253  ret <4 x half> %shuffle
254}
255
256define <4 x half> @shuffle_v4f16_4501(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
257; GFX9-LABEL: shuffle_v4f16_4501:
258; GFX9:       ; %bb.0:
259; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
260; GFX9-NEXT:    global_load_dword v2, v[2:3], off
261; GFX9-NEXT:    global_load_dword v1, v[0:1], off
262; GFX9-NEXT:    s_waitcnt vmcnt(1)
263; GFX9-NEXT:    v_mov_b32_e32 v0, v2
264; GFX9-NEXT:    s_waitcnt vmcnt(0)
265; GFX9-NEXT:    s_setpc_b64 s[30:31]
266  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
267  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
268  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
269  ret <4 x half> %shuffle
270}
271
272define <4 x half> @shuffle_v4f16_4523(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
273; GFX9-LABEL: shuffle_v4f16_4523:
274; GFX9:       ; %bb.0:
275; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
276; GFX9-NEXT:    global_load_dword v2, v[2:3], off
277; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
278; GFX9-NEXT:    s_waitcnt vmcnt(1)
279; GFX9-NEXT:    v_mov_b32_e32 v0, v2
280; GFX9-NEXT:    s_waitcnt vmcnt(0)
281; GFX9-NEXT:    s_setpc_b64 s[30:31]
282  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
283  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
284  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
285  ret <4 x half> %shuffle
286}
287
288define <4 x half> @shuffle_v4f16_4545(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
289; GFX9-LABEL: shuffle_v4f16_4545:
290; GFX9:       ; %bb.0:
291; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
292; GFX9-NEXT:    global_load_dword v0, v[2:3], off
293; GFX9-NEXT:    s_waitcnt vmcnt(0)
294; GFX9-NEXT:    v_mov_b32_e32 v1, v0
295; GFX9-NEXT:    s_setpc_b64 s[30:31]
296  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
297  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
298  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5>
299  ret <4 x half> %shuffle
300}
301
302define <4 x half> @shuffle_v4f16_4567(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
303; GFX9-LABEL: shuffle_v4f16_4567:
304; GFX9:       ; %bb.0:
305; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
306; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
307; GFX9-NEXT:    s_waitcnt vmcnt(0)
308; GFX9-NEXT:    s_setpc_b64 s[30:31]
309  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
310  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
311  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
312  ret <4 x half> %shuffle
313}
314
315define <4 x half> @shuffle_v4f16_6701(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
316; GFX9-LABEL: shuffle_v4f16_6701:
317; GFX9:       ; %bb.0:
318; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
319; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
320; GFX9-NEXT:    global_load_dword v1, v[0:1], off
321; GFX9-NEXT:    s_waitcnt vmcnt(1)
322; GFX9-NEXT:    v_mov_b32_e32 v0, v2
323; GFX9-NEXT:    s_waitcnt vmcnt(0)
324; GFX9-NEXT:    s_setpc_b64 s[30:31]
325  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
326  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
327  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
328  ret <4 x half> %shuffle
329}
330
331define <4 x half> @shuffle_v4f16_6723(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
332; GFX9-LABEL: shuffle_v4f16_6723:
333; GFX9:       ; %bb.0:
334; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
335; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
336; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
337; GFX9-NEXT:    s_waitcnt vmcnt(1)
338; GFX9-NEXT:    v_mov_b32_e32 v0, v2
339; GFX9-NEXT:    s_waitcnt vmcnt(0)
340; GFX9-NEXT:    s_setpc_b64 s[30:31]
341  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
342  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
343  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
344  ret <4 x half> %shuffle
345}
346
347define <4 x half> @shuffle_v4f16_6745(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
348; GFX9-LABEL: shuffle_v4f16_6745:
349; GFX9:       ; %bb.0:
350; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
351; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[2:3], off
352; GFX9-NEXT:    s_waitcnt vmcnt(0)
353; GFX9-NEXT:    v_mov_b32_e32 v0, v2
354; GFX9-NEXT:    s_setpc_b64 s[30:31]
355  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
356  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
357  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5>
358  ret <4 x half> %shuffle
359}
360
361define <4 x half> @shuffle_v4f16_6767(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
362; GFX9-LABEL: shuffle_v4f16_6767:
363; GFX9:       ; %bb.0:
364; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
365; GFX9-NEXT:    global_load_dword v0, v[2:3], off offset:4
366; GFX9-NEXT:    s_waitcnt vmcnt(0)
367; GFX9-NEXT:    v_mov_b32_e32 v1, v0
368; GFX9-NEXT:    s_setpc_b64 s[30:31]
369  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
370  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
371  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7>
372  ret <4 x half> %shuffle
373}
374
375define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
376; GFX9-LABEL: shuffle_v4f16_2356:
377; GFX9:       ; %bb.0:
378; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
379; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
380; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
381; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
382; GFX9-NEXT:    s_waitcnt vmcnt(1)
383; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
384; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
385; GFX9-NEXT:    s_waitcnt vmcnt(0)
386; GFX9-NEXT:    s_setpc_b64 s[30:31]
387  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
388  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
389  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
390  ret <4 x half> %shuffle
391}
392
393define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
394; GFX9-LABEL: shuffle_v4f16_5623:
395; GFX9:       ; %bb.0:
396; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
397; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
398; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
399; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
400; GFX9-NEXT:    s_waitcnt vmcnt(1)
401; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
402; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
403; GFX9-NEXT:    s_waitcnt vmcnt(0)
404; GFX9-NEXT:    s_setpc_b64 s[30:31]
405  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
406  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
407  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
408  ret <4 x half> %shuffle
409}
410
411define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
412; GFX9-LABEL: shuffle_v4f16_3456:
413; GFX9:       ; %bb.0:
414; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
415; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
416; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
417; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
418; GFX9-NEXT:    s_waitcnt vmcnt(1)
419; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
420; GFX9-NEXT:    s_waitcnt vmcnt(0)
421; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
422; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
423; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v4
424; GFX9-NEXT:    s_setpc_b64 s[30:31]
425  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
426  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
427  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
428  ret <4 x half> %shuffle
429}
430
431define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
432; GFX9-LABEL: shuffle_v4f16_5634:
433; GFX9:       ; %bb.0:
434; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
435; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
436; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
437; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
438; GFX9-NEXT:    s_waitcnt vmcnt(1)
439; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
440; GFX9-NEXT:    s_waitcnt vmcnt(0)
441; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
442; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v0
443; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v4
444; GFX9-NEXT:    s_setpc_b64 s[30:31]
445  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
446  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
447  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4>
448  ret <4 x half> %shuffle
449}
450
451define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
452; GFX9-LABEL: shuffle_v4f16_5734:
453; GFX9:       ; %bb.0:
454; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
455; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
456; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
457; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
458; GFX9-NEXT:    s_waitcnt vmcnt(1)
459; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
460; GFX9-NEXT:    s_waitcnt vmcnt(0)
461; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
462; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
463; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v0
464; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v4
465; GFX9-NEXT:    s_setpc_b64 s[30:31]
466  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
467  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
468  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4>
469  ret <4 x half> %shuffle
470}
471
472define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) {
473; GFX9-LABEL: shuffle_v4i16_2356:
474; GFX9:       ; %bb.0:
475; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
476; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
477; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
478; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
479; GFX9-NEXT:    s_waitcnt vmcnt(1)
480; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
481; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
482; GFX9-NEXT:    s_waitcnt vmcnt(0)
483; GFX9-NEXT:    s_setpc_b64 s[30:31]
484  %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
485  %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
486  %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
487  ret <4 x i16> %shuffle
488}
489
490define <4 x i16> @shuffle_v4i16_0167(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) {
491; GFX9-LABEL: shuffle_v4i16_0167:
492; GFX9:       ; %bb.0:
493; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
494; GFX9-NEXT:    global_load_dword v0, v[0:1], off
495; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
496; GFX9-NEXT:    s_waitcnt vmcnt(0)
497; GFX9-NEXT:    s_setpc_b64 s[30:31]
498  %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
499  %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
500  %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
501  ret <4 x i16> %shuffle
502}
503
504define <4 x half> @shuffle_v4f16_0000(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
505; GFX9-LABEL: shuffle_v4f16_0000:
506; GFX9:       ; %bb.0:
507; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
508; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
509; GFX9-NEXT:    s_waitcnt vmcnt(0)
510; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v0
511; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
512; GFX9-NEXT:    v_mov_b32_e32 v1, v0
513; GFX9-NEXT:    s_setpc_b64 s[30:31]
514  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
515  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
516  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer
517  ret <4 x half> %shuffle
518}
519
520define <4 x half> @shuffle_v4f16_1010(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
521; GFX9-LABEL: shuffle_v4f16_1010:
522; GFX9:       ; %bb.0:
523; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
524; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
525; GFX9-NEXT:    s_waitcnt vmcnt(0)
526; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
527; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
528; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
529; GFX9-NEXT:    v_mov_b32_e32 v1, v0
530; GFX9-NEXT:    s_setpc_b64 s[30:31]
531  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
532  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
533  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
534  ret <4 x half> %shuffle
535}
536
537define <4 x half> @shuffle_v4f16_1100(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
538; GFX9-LABEL: shuffle_v4f16_1100:
539; GFX9:       ; %bb.0:
540; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
541; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
542; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
543; GFX9-NEXT:    s_waitcnt vmcnt(0)
544; GFX9-NEXT:    v_and_b32_e32 v1, v2, v0
545; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
546; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
547; GFX9-NEXT:    v_and_b32_e32 v0, v2, v3
548; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
549; GFX9-NEXT:    s_setpc_b64 s[30:31]
550  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
551  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
552  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
553  ret <4 x half> %shuffle
554}
555
556define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
557; GFX9-LABEL: shuffle_v4f16_6161:
558; GFX9:       ; %bb.0:
559; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
560; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
561; GFX9-NEXT:    global_load_dword v0, v[0:1], off
562; GFX9-NEXT:    s_waitcnt vmcnt(1)
563; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v2
564; GFX9-NEXT:    s_waitcnt vmcnt(0)
565; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
566; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
567; GFX9-NEXT:    v_mov_b32_e32 v1, v0
568; GFX9-NEXT:    s_setpc_b64 s[30:31]
569  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
570  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
571  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1>
572  ret <4 x half> %shuffle
573}
574
575define <4 x half> @shuffle_v4f16_2333(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
576; GFX9-LABEL: shuffle_v4f16_2333:
577; GFX9:       ; %bb.0:
578; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
579; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
580; GFX9-NEXT:    s_waitcnt vmcnt(0)
581; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
582; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
583; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
584; GFX9-NEXT:    s_setpc_b64 s[30:31]
585  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
586  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
587  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
588  ret <4 x half> %shuffle
589}
590
591define <4 x half> @shuffle_v4f16_6667(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
592; GFX9-LABEL: shuffle_v4f16_6667:
593; GFX9:       ; %bb.0:
594; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
595; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
596; GFX9-NEXT:    s_waitcnt vmcnt(0)
597; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
598; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
599; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
600; GFX9-NEXT:    s_setpc_b64 s[30:31]
601  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
602  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
603  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
604  ret <4 x half> %shuffle
605}
606
607define <4 x half> @shuffle_v8f16_0101(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
608; GFX9-LABEL: shuffle_v8f16_0101:
609; GFX9:       ; %bb.0:
610; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
611; GFX9-NEXT:    global_load_dword v0, v[0:1], off
612; GFX9-NEXT:    s_waitcnt vmcnt(0)
613; GFX9-NEXT:    v_mov_b32_e32 v1, v0
614; GFX9-NEXT:    s_setpc_b64 s[30:31]
615  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
616  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
617  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
618  ret <4 x half> %shuffle
619}
620
621define <4 x half> @shuffle_v8f16_0123(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
622; GFX9-LABEL: shuffle_v8f16_0123:
623; GFX9:       ; %bb.0:
624; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
625; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
626; GFX9-NEXT:    s_waitcnt vmcnt(0)
627; GFX9-NEXT:    s_setpc_b64 s[30:31]
628  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
629  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
630  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
631  ret <4 x half> %shuffle
632}
633
634define <4 x half> @shuffle_v8f16_4589(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
635; GFX9-LABEL: shuffle_v8f16_4589:
636; GFX9:       ; %bb.0:
637; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
638; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:8
639; GFX9-NEXT:    global_load_dword v1, v[2:3], off
640; GFX9-NEXT:    s_waitcnt vmcnt(0)
641; GFX9-NEXT:    s_setpc_b64 s[30:31]
642  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
643  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
644  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9>
645  ret <4 x half> %shuffle
646}
647
648define <4 x half> @shuffle_v8f16_10_11_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
649; GFX9-LABEL: shuffle_v8f16_10_11_2_3:
650; GFX9:       ; %bb.0:
651; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
652; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
653; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
654; GFX9-NEXT:    s_waitcnt vmcnt(1)
655; GFX9-NEXT:    v_mov_b32_e32 v0, v2
656; GFX9-NEXT:    s_waitcnt vmcnt(0)
657; GFX9-NEXT:    s_setpc_b64 s[30:31]
658  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
659  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
660  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3>
661  ret <4 x half> %shuffle
662}
663
664define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) {
665; GFX9-LABEL: shuffle_v8f16_13_14_2_3:
666; GFX9:       ; %bb.0:
667; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
668; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:8
669; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
670; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
671; GFX9-NEXT:    s_waitcnt vmcnt(1)
672; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
673; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
674; GFX9-NEXT:    s_waitcnt vmcnt(0)
675; GFX9-NEXT:    s_setpc_b64 s[30:31]
676  %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
677  %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
678  %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
679  ret <4 x half> %shuffle
680}
681
682define <4 x half> @shuffle_v3f16_0122(<3 x half> addrspace(1)* %arg0, <3 x half> addrspace(1)* %arg1) {
683; GFX9-LABEL: shuffle_v3f16_0122:
684; GFX9:       ; %bb.0:
685; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
686; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
687; GFX9-NEXT:    s_waitcnt vmcnt(0)
688; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v1
689; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
690; GFX9-NEXT:    s_setpc_b64 s[30:31]
691  %val0 = load <3 x half>, <3 x half> addrspace(1)* %arg0
692  %val1 = load <3 x half>, <3 x half> addrspace(1)* %arg1
693  %shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
694  ret <4 x half> %shuffle
695}
696
697define <4 x half> @shuffle_v2f16_0122(<2 x half> addrspace(1)* %arg0, <2 x half> addrspace(1)* %arg1) {
698; GFX9-LABEL: shuffle_v2f16_0122:
699; GFX9:       ; %bb.0:
700; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
701; GFX9-NEXT:    global_load_dword v0, v[0:1], off
702; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
703; GFX9-NEXT:    s_waitcnt vmcnt(0)
704; GFX9-NEXT:    v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
705; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
706; GFX9-NEXT:    s_setpc_b64 s[30:31]
707  %val0 = load <2 x half>, <2 x half> addrspace(1)* %arg0
708  %val1 = load <2 x half>, <2 x half> addrspace(1)* %arg1
709  %shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
710  ret <4 x half> %shuffle
711}
712
713define <6 x half> @shuffle_v6f16_452367(<6 x half> addrspace(1)* %arg0, <6 x half> addrspace(1)* %arg1) {
714; GFX9-LABEL: shuffle_v6f16_452367:
715; GFX9:       ; %bb.0:
716; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
717; GFX9-NEXT:    v_mov_b32_e32 v4, v3
718; GFX9-NEXT:    v_mov_b32_e32 v3, v2
719; GFX9-NEXT:    global_load_dwordx3 v[0:2], v[0:1], off
720; GFX9-NEXT:    global_load_dword v3, v[3:4], off
721; GFX9-NEXT:    s_waitcnt vmcnt(1)
722; GFX9-NEXT:    v_mov_b32_e32 v0, v2
723; GFX9-NEXT:    s_waitcnt vmcnt(0)
724; GFX9-NEXT:    v_mov_b32_e32 v2, v3
725; GFX9-NEXT:    s_setpc_b64 s[30:31]
726  %val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0
727  %val1 = load <6 x half>, <6 x half> addrspace(1)* %arg1
728  %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
729  ret <6 x half> %shuffle
730}
731
732define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readonly %A, <4 x half> addrspace(1)* nocapture readonly %B, <4 x half> addrspace(1)* nocapture %C)  {
733; GFX9-LABEL: fma_shuffle:
734; GFX9:       ; %bb.0: ; %entry
735; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
736; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
737; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
738; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX9-NEXT:    v_mov_b32_e32 v1, s1
740; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v4
741; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
742; GFX9-NEXT:    v_mov_b32_e32 v3, s3
743; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s2, v4
744; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
745; GFX9-NEXT:    v_mov_b32_e32 v5, s5
746; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s4, v4
747; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
748; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
749; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
750; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[4:5], off
751; GFX9-NEXT:    s_waitcnt vmcnt(0)
752; GFX9-NEXT:    v_pk_fma_f16 v6, v0, v2, v6 op_sel_hi:[0,1,1]
753; GFX9-NEXT:    v_pk_fma_f16 v2, v1, v2, v7 op_sel_hi:[0,1,1]
754; GFX9-NEXT:    v_pk_fma_f16 v0, v0, v3, v6 op_sel:[1,0,0]
755; GFX9-NEXT:    v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0]
756; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
757; GFX9-NEXT:    s_endpgm
758entry:
759  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x()
760  %tmp12 = zext i32 %tmp1 to i64
761  %arrayidx = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %A, i64 %tmp12
762  %tmp14 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx, align 8
763  %arrayidx1 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %B, i64 %tmp12
764  %tmp15 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx1, align 8
765  %arrayidx2 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %C, i64 %tmp12
766  %tmp16 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx2, align 8
767  %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> zeroinitializer
768  %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 0, i32 1>
769  %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> undef, <2 x i32> <i32 0, i32 1>
770  %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19)
771  %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 1, i32 1>
772  %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 2, i32 3>
773  %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20)
774  %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
775  %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
776  %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 2, i32 2>
777  %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> undef, <2 x i32> <i32 2, i32 3>
778  %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27)
779  %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 3, i32 3>
780  %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28)
781  %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
782  %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
783  store <4 x half> %tmp32, <4 x half> addrspace(1)* %arrayidx2, align 8
784  ret void
785}
786
787define <4 x half> @shuffle_v4f16_0456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) {
788; GFX9-LABEL: shuffle_v4f16_0456:
789; GFX9:       ; %bb.0:
790; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
791; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
792; GFX9-NEXT:    s_waitcnt vmcnt(0)
793; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[2:3], off
794; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
795; GFX9-NEXT:    v_and_b32_e32 v0, v3, v0
796; GFX9-NEXT:    s_waitcnt vmcnt(0)
797; GFX9-NEXT:    v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
798; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
799; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v3
800; GFX9-NEXT:    s_setpc_b64 s[30:31]
801  %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
802  %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
803  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
804  ret <4 x half> %shuffle
805}
806
807define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(<8 x i32> addrspace(4)* %in, <4 x i32> addrspace(1)* %out)  {
808; GFX9-LABEL: shuffle_scalar_load_v8i32_0123:
809; GFX9:       ; %bb.0:
810; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
811; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
812; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
813; GFX9-NEXT:    v_mov_b32_e32 v4, s2
814; GFX9-NEXT:    v_mov_b32_e32 v5, s3
815; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
816; GFX9-NEXT:    v_mov_b32_e32 v0, s4
817; GFX9-NEXT:    v_mov_b32_e32 v1, s5
818; GFX9-NEXT:    v_mov_b32_e32 v2, s6
819; GFX9-NEXT:    v_mov_b32_e32 v3, s7
820; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
821; GFX9-NEXT:    s_endpgm
822  %ld8 = load <8 x i32>, <8 x i32> addrspace(4)* %in, align 16
823  %id = shufflevector <8 x i32> %ld8, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
824  store <4 x i32> %id, <4 x i32> addrspace(1)* %out, align 8
825  ret void
826}
827
828declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #0
829declare i32 @llvm.amdgcn.workitem.id.x() #0
830
831attributes #0 = { nounwind readnone speculatable }
832