1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
3; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512bw -mattr=+avx512vl -mattr=+avx512dq| FileCheck %s --check-prefix=VL_BW_DQ
4
5target triple = "x86_64-unknown-unknown"
6
7define <2 x i1> @shuf2i1_1_0(<2 x i1> %a) {
8; AVX512F-LABEL: shuf2i1_1_0:
9; AVX512F:       # BB#0:
10; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
11; AVX512F-NEXT:    retq
12;
13; VL_BW_DQ-LABEL: shuf2i1_1_0:
14; VL_BW_DQ:       # BB#0:
15; VL_BW_DQ-NEXT:    vpsllq $63, %xmm0, %xmm0
16; VL_BW_DQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
17; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm0
18; VL_BW_DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
19; VL_BW_DQ-NEXT:    vpmovq2m %xmm0, %k0
20; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm0
21; VL_BW_DQ-NEXT:    retq
22  %b = shufflevector <2 x i1> %a, <2 x i1> undef, <2 x i32> <i32 1, i32 0>
23  ret <2 x i1> %b
24}
25
26define <2 x i1> @shuf2i1_1_2(<2 x i1> %a) {
27; AVX512F-LABEL: shuf2i1_1_2:
28; AVX512F:       # BB#0:
29; AVX512F-NEXT:    movl $1, %eax
30; AVX512F-NEXT:    vmovq %rax, %xmm1
31; AVX512F-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
32; AVX512F-NEXT:    retq
33;
34; VL_BW_DQ-LABEL: shuf2i1_1_2:
35; VL_BW_DQ:       # BB#0:
36; VL_BW_DQ-NEXT:    vpsllq $63, %xmm0, %xmm0
37; VL_BW_DQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
38; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm0
39; VL_BW_DQ-NEXT:    movb $1, %al
40; VL_BW_DQ-NEXT:    kmovb %eax, %k0
41; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm1
42; VL_BW_DQ-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
43; VL_BW_DQ-NEXT:    vpmovq2m %xmm0, %k0
44; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm0
45; VL_BW_DQ-NEXT:    retq
46  %b = shufflevector <2 x i1> %a, <2 x i1> <i1 1, i1 0>, <2 x i32> <i32 1, i32 2>
47  ret <2 x i1> %b
48}
49
50
51define <4 x i1> @shuf4i1_3_2_10(<4 x i1> %a) {
52; AVX512F-LABEL: shuf4i1_3_2_10:
53; AVX512F:       # BB#0:
54; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,2,1,0]
55; AVX512F-NEXT:    retq
56;
57; VL_BW_DQ-LABEL: shuf4i1_3_2_10:
58; VL_BW_DQ:       # BB#0:
59; VL_BW_DQ-NEXT:    vpslld $31, %xmm0, %xmm0
60; VL_BW_DQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
61; VL_BW_DQ-NEXT:    vpmovm2d %k0, %xmm0
62; VL_BW_DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,2,1,0]
63; VL_BW_DQ-NEXT:    vpmovd2m %xmm0, %k0
64; VL_BW_DQ-NEXT:    vpmovm2d %k0, %xmm0
65; VL_BW_DQ-NEXT:    retq
66  %b = shufflevector <4 x i1> %a, <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
67  ret <4 x i1> %b
68}
69
70define <8 x i1> @shuf8i1_3_6_1_0_3_7_7_0(<8 x i64> %a, <8 x i64> %b, <8 x i64> %a1, <8 x i64> %b1) {
71; AVX512F-LABEL: shuf8i1_3_6_1_0_3_7_7_0:
72; AVX512F:       # BB#0:
73; AVX512F-NEXT:    vpcmpeqq %zmm2, %zmm0, %k1
74; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
75; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [3,6,1,0,3,7,7,0]
76; AVX512F-NEXT:    vpermq %zmm0, %zmm1, %zmm0
77; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
78; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
79; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
80; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
81; AVX512F-NEXT:    retq
82;
83; VL_BW_DQ-LABEL: shuf8i1_3_6_1_0_3_7_7_0:
84; VL_BW_DQ:       # BB#0:
85; VL_BW_DQ-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
86; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
87; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [3,6,1,0,3,7,7,0]
88; VL_BW_DQ-NEXT:    vpermq %zmm0, %zmm1, %zmm0
89; VL_BW_DQ-NEXT:    vpmovq2m %zmm0, %k0
90; VL_BW_DQ-NEXT:    vpmovm2w %k0, %xmm0
91; VL_BW_DQ-NEXT:    retq
92  %a2 = icmp eq <8 x i64> %a, %a1
93  %b2 = icmp eq <8 x i64> %b, %b1
94  %c = shufflevector <8 x i1> %a2, <8 x i1> %b2, <8 x i32> <i32 3, i32 6, i32 1, i32 0, i32 3, i32 7, i32 7, i32 0>
95  ret <8 x i1> %c
96}
97
98define <16 x i1> @shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0(<16 x i32> %a, <16 x i32> %b, <16 x i32> %a1, <16 x i32> %b1) {
99; AVX512F-LABEL: shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0:
100; AVX512F:       # BB#0:
101; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm0, %k1
102; AVX512F-NEXT:    vpcmpeqd %zmm3, %zmm1, %k2
103; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
104; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
105; AVX512F-NEXT:    vmovdqa32 {{.*#+}} zmm2 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0]
106; AVX512F-NEXT:    vpermi2d %zmm0, %zmm1, %zmm2
107; AVX512F-NEXT:    vpslld $31, %zmm2, %zmm0
108; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
109; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
110; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
111; AVX512F-NEXT:    retq
112;
113; VL_BW_DQ-LABEL: shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0:
114; VL_BW_DQ:       # BB#0:
115; VL_BW_DQ-NEXT:    vpcmpeqd %zmm2, %zmm0, %k0
116; VL_BW_DQ-NEXT:    vpcmpeqd %zmm3, %zmm1, %k1
117; VL_BW_DQ-NEXT:    vpmovm2d %k1, %zmm0
118; VL_BW_DQ-NEXT:    vpmovm2d %k0, %zmm1
119; VL_BW_DQ-NEXT:    vmovdqa32 {{.*#+}} zmm2 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0]
120; VL_BW_DQ-NEXT:    vpermi2d %zmm0, %zmm1, %zmm2
121; VL_BW_DQ-NEXT:    vpmovd2m %zmm2, %k0
122; VL_BW_DQ-NEXT:    vpmovm2b %k0, %xmm0
123; VL_BW_DQ-NEXT:    retq
124  %a2 = icmp eq <16 x i32> %a, %a1
125  %b2 = icmp eq <16 x i32> %b, %b1
126  %c = shufflevector <16 x i1> %a2, <16 x i1> %b2, <16 x i32> <i32 3, i32 6, i32 22, i32 12, i32 3, i32 7, i32 7, i32 0, i32 3, i32 6, i32 1, i32 13, i32 3, i32 21, i32 7, i32 0>
127  ret <16 x i1> %c
128}
129
130define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0(<32 x i1> %a) {
131; AVX512F-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0:
132; AVX512F:       # BB#0:
133; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
134; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,6,u,u,u,u,u,u,u,u,u,u,5,u,u,19,22,u,28,19,23,23,16,19,22,17,29,19,u,23,16]
135; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,6,u,12,3,7,7,0,3,6,1,13,3,u,7,0,u,u,22,u,u,u,u,u,u,u,u,u,u,21,u,u]
136; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,0,255,255,0,0,255,0,0,0,0,0,0,0,0,0,0,255,0,0]
137; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
138; AVX512F-NEXT:    retq
139;
140; VL_BW_DQ-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0:
141; VL_BW_DQ:       # BB#0:
142; VL_BW_DQ-NEXT:    vpsllw $7, %ymm0, %ymm0
143; VL_BW_DQ-NEXT:    vpmovb2m %ymm0, %k0
144; VL_BW_DQ-NEXT:    vpmovm2w %k0, %zmm0
145; VL_BW_DQ-NEXT:    vmovdqu16 {{.*#+}} zmm1 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0,3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0]
146; VL_BW_DQ-NEXT:    vpermw %zmm0, %zmm1, %zmm0
147; VL_BW_DQ-NEXT:    vpmovw2m %zmm0, %k0
148; VL_BW_DQ-NEXT:    vpmovm2b %k0, %ymm0
149; VL_BW_DQ-NEXT:    retq
150  %b = shufflevector <32 x i1> %a, <32 x i1> undef, <32 x i32> <i32 3, i32 6, i32 22, i32 12, i32 3, i32 7, i32 7, i32 0, i32 3, i32 6, i32 1, i32 13, i32 3, i32 21, i32 7, i32 0, i32 3, i32 6, i32 22, i32 12, i32 3, i32 7, i32 7, i32 0, i32 3, i32 6, i32 1, i32 13, i32 3, i32 21, i32 7, i32 0>
151  ret <32 x i1> %b
152}
153
154define <8 x i1> @shuf8i1_u_2_u_u_2_u_2_u(i8 %a) {
155; AVX512F-LABEL: shuf8i1_u_2_u_u_2_u_2_u:
156; AVX512F:       # BB#0:
157; AVX512F-NEXT:    kmovw %edi, %k1
158; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
159; AVX512F-NEXT:    vextracti32x4 $1, %zmm0, %xmm0
160; AVX512F-NEXT:    vpbroadcastq %xmm0, %zmm0
161; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
162; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
163; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
164; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
165; AVX512F-NEXT:    retq
166;
167; VL_BW_DQ-LABEL: shuf8i1_u_2_u_u_2_u_2_u:
168; VL_BW_DQ:       # BB#0:
169; VL_BW_DQ-NEXT:    kmovb %edi, %k0
170; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
171; VL_BW_DQ-NEXT:    vextracti64x2 $1, %zmm0, %xmm0
172; VL_BW_DQ-NEXT:    vpbroadcastq %xmm0, %zmm0
173; VL_BW_DQ-NEXT:    vpmovq2m %zmm0, %k0
174; VL_BW_DQ-NEXT:    vpmovm2w %k0, %xmm0
175; VL_BW_DQ-NEXT:    retq
176  %b = bitcast i8 %a to <8 x i1>
177  %c = shufflevector < 8 x i1> %b, <8 x i1>undef, <8 x i32> <i32 undef, i32 2, i32 undef, i32 undef, i32 2, i32 undef, i32 2, i32 undef>
178  ret <8 x i1> %c
179}
180
181define i8 @shuf8i1_10_2_9_u_3_u_2_u(i8 %a) {
182; AVX512F-LABEL: shuf8i1_10_2_9_u_3_u_2_u:
183; AVX512F:       # BB#0:
184; AVX512F-NEXT:    kmovw %edi, %k1
185; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
186; AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
187; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <8,2,10,u,3,u,2,u>
188; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
189; AVX512F-NEXT:    vpsllq $63, %zmm2, %zmm0
190; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
191; AVX512F-NEXT:    kmovw %k0, %eax
192; AVX512F-NEXT:    retq
193;
194; VL_BW_DQ-LABEL: shuf8i1_10_2_9_u_3_u_2_u:
195; VL_BW_DQ:       # BB#0:
196; VL_BW_DQ-NEXT:    kmovb %edi, %k0
197; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
198; VL_BW_DQ-NEXT:    vpxord %zmm1, %zmm1, %zmm1
199; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <8,2,10,u,3,u,2,u>
200; VL_BW_DQ-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
201; VL_BW_DQ-NEXT:    vpmovq2m %zmm2, %k0
202; VL_BW_DQ-NEXT:    kmovb %k0, %eax
203; VL_BW_DQ-NEXT:    retq
204  %b = bitcast i8 %a to <8 x i1>
205  %c = shufflevector < 8 x i1> %b, <8 x i1> zeroinitializer, <8 x i32> <i32 10, i32 2, i32 9, i32 undef, i32 3, i32 undef, i32 2, i32 undef>
206  %d = bitcast <8 x i1> %c to i8
207  ret i8 %d
208}
209
210define i8 @shuf8i1_0_1_4_5_u_u_u_u(i8 %a) {
211; AVX512F-LABEL: shuf8i1_0_1_4_5_u_u_u_u:
212; AVX512F:       # BB#0:
213; AVX512F-NEXT:    kmovw %edi, %k1
214; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
215; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5,0,1,0,1]
216; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
217; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
218; AVX512F-NEXT:    kmovw %k0, %eax
219; AVX512F-NEXT:    retq
220;
221; VL_BW_DQ-LABEL: shuf8i1_0_1_4_5_u_u_u_u:
222; VL_BW_DQ:       # BB#0:
223; VL_BW_DQ-NEXT:    kmovb %edi, %k0
224; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
225; VL_BW_DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5,0,1,0,1]
226; VL_BW_DQ-NEXT:    vpmovq2m %zmm0, %k0
227; VL_BW_DQ-NEXT:    kmovb %k0, %eax
228; VL_BW_DQ-NEXT:    retq
229  %b = bitcast i8 %a to <8 x i1>
230  %c = shufflevector < 8 x i1> %b, <8 x i1> undef, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 undef, i32 undef, i32 undef, i32 undef>
231  %d = bitcast <8 x i1> %c to i8
232  ret i8 %d
233}
234
235define i8 @shuf8i1_9_6_1_0_3_7_7_0(i8 %a) {
236; AVX512F-LABEL: shuf8i1_9_6_1_0_3_7_7_0:
237; AVX512F:       # BB#0:
238; AVX512F-NEXT:    kmovw %edi, %k1
239; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
240; AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
241; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,6,1,0,3,7,7,0]
242; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
243; AVX512F-NEXT:    vpsllq $63, %zmm2, %zmm0
244; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
245; AVX512F-NEXT:    kmovw %k0, %eax
246; AVX512F-NEXT:    retq
247;
248; VL_BW_DQ-LABEL: shuf8i1_9_6_1_0_3_7_7_0:
249; VL_BW_DQ:       # BB#0:
250; VL_BW_DQ-NEXT:    kmovb %edi, %k0
251; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
252; VL_BW_DQ-NEXT:    vpxord %zmm1, %zmm1, %zmm1
253; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,6,1,0,3,7,7,0]
254; VL_BW_DQ-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
255; VL_BW_DQ-NEXT:    vpmovq2m %zmm2, %k0
256; VL_BW_DQ-NEXT:    kmovb %k0, %eax
257; VL_BW_DQ-NEXT:    retq
258  %b = bitcast i8 %a to <8 x i1>
259  %c = shufflevector <8 x i1> %b, <8 x i1> zeroinitializer, <8 x i32> <i32 9, i32 6, i32 1, i32 0, i32 3, i32 7, i32 7, i32 0>
260  %d = bitcast <8 x i1>%c to i8
261  ret i8 %d
262}
263
264define i8 @shuf8i1_9_6_1_10_3_7_7_0(i8 %a) {
265; AVX512F-LABEL: shuf8i1_9_6_1_10_3_7_7_0:
266; AVX512F:       # BB#0:
267; AVX512F-NEXT:    kmovw %edi, %k1
268; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
269; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [9,1,2,10,4,5,6,7]
270; AVX512F-NEXT:    vpxord %zmm2, %zmm2, %zmm2
271; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
272; AVX512F-NEXT:    vpsllq $63, %zmm2, %zmm0
273; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
274; AVX512F-NEXT:    kmovw %k0, %eax
275; AVX512F-NEXT:    retq
276;
277; VL_BW_DQ-LABEL: shuf8i1_9_6_1_10_3_7_7_0:
278; VL_BW_DQ:       # BB#0:
279; VL_BW_DQ-NEXT:    kmovb %edi, %k0
280; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
281; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [9,1,2,10,4,5,6,7]
282; VL_BW_DQ-NEXT:    vpxord %zmm2, %zmm2, %zmm2
283; VL_BW_DQ-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
284; VL_BW_DQ-NEXT:    vpmovq2m %zmm2, %k0
285; VL_BW_DQ-NEXT:    kmovb %k0, %eax
286; VL_BW_DQ-NEXT:    retq
287  %b = bitcast i8 %a to <8 x i1>
288  %c = shufflevector <8 x i1> zeroinitializer, <8 x i1> %b, <8 x i32> <i32 9, i32 6, i32 1, i32 10, i32 3, i32 7, i32 7, i32 0>
289  %d = bitcast <8 x i1>%c to i8
290  ret i8 %d
291}
292
293define i8 @shuf8i1__9_6_1_10_3_7_7_1(i8 %a) {
294; AVX512F-LABEL: shuf8i1__9_6_1_10_3_7_7_1:
295; AVX512F:       # BB#0:
296; AVX512F-NEXT:    kmovw %edi, %k1
297; AVX512F-NEXT:    movb $51, %al
298; AVX512F-NEXT:    kmovw %eax, %k2
299; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
300; AVX512F-NEXT:    vpternlogq $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
301; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [9,6,1,0,3,7,7,1]
302; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
303; AVX512F-NEXT:    vpsllq $63, %zmm2, %zmm0
304; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
305; AVX512F-NEXT:    kmovw %k0, %eax
306; AVX512F-NEXT:    retq
307;
308; VL_BW_DQ-LABEL: shuf8i1__9_6_1_10_3_7_7_1:
309; VL_BW_DQ:       # BB#0:
310; VL_BW_DQ-NEXT:    kmovb %edi, %k0
311; VL_BW_DQ-NEXT:    movb $51, %al
312; VL_BW_DQ-NEXT:    kmovb %eax, %k1
313; VL_BW_DQ-NEXT:    vpmovm2q %k1, %zmm0
314; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm1
315; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [9,6,1,0,3,7,7,1]
316; VL_BW_DQ-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
317; VL_BW_DQ-NEXT:    vpmovq2m %zmm2, %k0
318; VL_BW_DQ-NEXT:    kmovb %k0, %eax
319; VL_BW_DQ-NEXT:    retq
320  %b = bitcast i8 %a to <8 x i1>
321  %c = shufflevector <8 x i1> <i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 0, i1 0>, <8 x i1> %b, <8 x i32> <i32 9, i32 6, i32 1, i32 0, i32 3, i32 7, i32 7, i32 1>
322  %c1 = bitcast <8 x i1>%c to i8
323  ret i8 %c1
324}
325
326define i8 @shuf8i1_9_6_1_10_3_7_7_0_all_ones(<8 x i1> %a) {
327; AVX512F-LABEL: shuf8i1_9_6_1_10_3_7_7_0_all_ones:
328; AVX512F:       # BB#0:
329; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
330; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
331; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
332; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
333; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [9,1,2,3,4,5,6,7]
334; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
335; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
336; AVX512F-NEXT:    vpsllq $63, %zmm2, %zmm0
337; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
338; AVX512F-NEXT:    kmovw %k0, %eax
339; AVX512F-NEXT:    retq
340;
341; VL_BW_DQ-LABEL: shuf8i1_9_6_1_10_3_7_7_0_all_ones:
342; VL_BW_DQ:       # BB#0:
343; VL_BW_DQ-NEXT:    vpsllw $15, %xmm0, %xmm0
344; VL_BW_DQ-NEXT:    vpmovw2m %xmm0, %k0
345; VL_BW_DQ-NEXT:    vpmovm2q %k0, %zmm0
346; VL_BW_DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [9,1,2,3,4,5,6,7]
347; VL_BW_DQ-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
348; VL_BW_DQ-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
349; VL_BW_DQ-NEXT:    vpmovq2m %zmm2, %k0
350; VL_BW_DQ-NEXT:    kmovb %k0, %eax
351; VL_BW_DQ-NEXT:    retq
352  %c = shufflevector <8 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <8 x i1> %a, <8 x i32> <i32 9, i32 6, i32 1, i32 0, i32 3, i32 7, i32 7, i32 0>
353  %c1 = bitcast <8 x i1>%c to i8
354  ret i8 %c1
355}
356
357
358define i16 @shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0(i16 %a) {
359; AVX512F-LABEL: shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0:
360; AVX512F:       # BB#0:
361; AVX512F-NEXT:    kmovw %edi, %k1
362; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
363; AVX512F-NEXT:    vpbroadcastd %xmm0, %zmm0
364; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
365; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
366; AVX512F-NEXT:    kmovw %k0, %eax
367; AVX512F-NEXT:    retq
368;
369; VL_BW_DQ-LABEL: shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0:
370; VL_BW_DQ:       # BB#0:
371; VL_BW_DQ-NEXT:    kmovw %edi, %k0
372; VL_BW_DQ-NEXT:    vpmovm2d %k0, %zmm0
373; VL_BW_DQ-NEXT:    vpbroadcastd %xmm0, %zmm0
374; VL_BW_DQ-NEXT:    vpmovd2m %zmm0, %k0
375; VL_BW_DQ-NEXT:    kmovw %k0, %eax
376; VL_BW_DQ-NEXT:    retq
377  %b = bitcast i16 %a to <16 x i1>
378  %c = shufflevector < 16 x i1> %b, <16 x i1> undef, <16 x i32> zeroinitializer
379  %d = bitcast <16 x i1> %c to i16
380  ret i16 %d
381}
382
383define i64 @shuf64i1_zero(i64 %a) {
384; AVX512F-LABEL: shuf64i1_zero:
385; AVX512F:       # BB#0:
386; AVX512F-NEXT:    pushq %rbp
387; AVX512F-NEXT:  .Lcfi0:
388; AVX512F-NEXT:    .cfi_def_cfa_offset 16
389; AVX512F-NEXT:  .Lcfi1:
390; AVX512F-NEXT:    .cfi_offset %rbp, -16
391; AVX512F-NEXT:    movq %rsp, %rbp
392; AVX512F-NEXT:  .Lcfi2:
393; AVX512F-NEXT:    .cfi_def_cfa_register %rbp
394; AVX512F-NEXT:    andq $-32, %rsp
395; AVX512F-NEXT:    subq $96, %rsp
396; AVX512F-NEXT:    movl %edi, {{[0-9]+}}(%rsp)
397; AVX512F-NEXT:    kmovw {{[0-9]+}}(%rsp), %k1
398; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
399; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
400; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
401; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
402; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
403; AVX512F-NEXT:    vpslld $31, %zmm1, %zmm1
404; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
405; AVX512F-NEXT:    kmovw %k0, {{[0-9]+}}(%rsp)
406; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
407; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
408; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
409; AVX512F-NEXT:    kmovw %k0, (%rsp)
410; AVX512F-NEXT:    movl (%rsp), %ecx
411; AVX512F-NEXT:    movq %rcx, %rax
412; AVX512F-NEXT:    shlq $32, %rax
413; AVX512F-NEXT:    orq %rcx, %rax
414; AVX512F-NEXT:    movq %rbp, %rsp
415; AVX512F-NEXT:    popq %rbp
416; AVX512F-NEXT:    retq
417;
418; VL_BW_DQ-LABEL: shuf64i1_zero:
419; VL_BW_DQ:       # BB#0:
420; VL_BW_DQ-NEXT:    kmovq %rdi, %k0
421; VL_BW_DQ-NEXT:    vpmovm2b %k0, %zmm0
422; VL_BW_DQ-NEXT:    vpbroadcastb %xmm0, %zmm0
423; VL_BW_DQ-NEXT:    vpmovb2m %zmm0, %k0
424; VL_BW_DQ-NEXT:    kmovq %k0, %rax
425; VL_BW_DQ-NEXT:    retq
426  %b = bitcast i64 %a to <64 x i1>
427  %c = shufflevector < 64 x i1> %b, <64 x i1> undef, <64 x i32> zeroinitializer
428  %d = bitcast <64 x i1> %c to i64
429  ret i64 %d
430}
431