1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512cd,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512CDVL
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512cd,-avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512CD
6;
7; Just one 32-bit run to make sure we do reasonable things for i64 tzcnt.
8; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-AVX --check-prefix=X32-AVX2
9
10define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
11; AVX1-LABEL: testv4i64:
12; AVX1:       # BB#0:
13; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
14; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
15; AVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
16; AVX1-NEXT:    vpsubq %xmm0, %xmm2, %xmm3
17; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
18; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
19; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
20; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
21; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
22; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
23; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm5
24; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
25; AVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
26; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
27; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
28; AVX1-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
29; AVX1-NEXT:    vpaddb %xmm5, %xmm1, %xmm1
30; AVX1-NEXT:    vpsadbw %xmm2, %xmm1, %xmm1
31; AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
32; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm3
33; AVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
34; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
35; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
36; AVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
37; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
38; AVX1-NEXT:    vpsadbw %xmm2, %xmm0, %xmm0
39; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
40; AVX1-NEXT:    retq
41;
42; AVX2-LABEL: testv4i64:
43; AVX2:       # BB#0:
44; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
45; AVX2-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
46; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
47; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm2
48; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
49; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
50; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm3
51; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
52; AVX2-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
53; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
54; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
55; AVX2-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
56; AVX2-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
57; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
58; AVX2-NEXT:    retq
59;
60; AVX512CDVL-LABEL: testv4i64:
61; AVX512CDVL:       # BB#0:
62; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
63; AVX512CDVL-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
64; AVX512CDVL-NEXT:    vpandq %ymm2, %ymm0, %ymm0
65; AVX512CDVL-NEXT:    vpsubq {{.*}}(%rip){1to4}, %ymm0, %ymm0
66; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
67; AVX512CDVL-NEXT:    vpandq %ymm2, %ymm0, %ymm3
68; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
69; AVX512CDVL-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
70; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
71; AVX512CDVL-NEXT:    vpandq %ymm2, %ymm0, %ymm0
72; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
73; AVX512CDVL-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
74; AVX512CDVL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
75; AVX512CDVL-NEXT:    retq
76;
77; AVX512CD-LABEL: testv4i64:
78; AVX512CD:       # BB#0:
79; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
80; AVX512CD-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
81; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm0
82; AVX512CD-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm2
83; AVX512CD-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
84; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
85; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm3
86; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
87; AVX512CD-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
88; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
89; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm0
90; AVX512CD-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
91; AVX512CD-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
92; AVX512CD-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
93; AVX512CD-NEXT:    retq
94;
95; X32-AVX-LABEL: testv4i64:
96; X32-AVX:       # BB#0:
97; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
98; X32-AVX-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
99; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
100; X32-AVX-NEXT:    vpsubq {{\.LCPI.*}}, %ymm0, %ymm0
101; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
102; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
103; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
104; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
105; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
106; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
107; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
108; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
109; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
110; X32-AVX-NEXT:    retl
111  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 0)
112  ret <4 x i64> %out
113}
114
115define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
116; AVX1-LABEL: testv4i64u:
117; AVX1:       # BB#0:
118; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
119; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
120; AVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
121; AVX1-NEXT:    vpsubq %xmm0, %xmm2, %xmm3
122; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
123; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
124; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
125; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
126; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
127; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
128; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm5
129; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
130; AVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
131; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
132; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
133; AVX1-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
134; AVX1-NEXT:    vpaddb %xmm5, %xmm1, %xmm1
135; AVX1-NEXT:    vpsadbw %xmm2, %xmm1, %xmm1
136; AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
137; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm3
138; AVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
139; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
140; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
141; AVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
142; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
143; AVX1-NEXT:    vpsadbw %xmm2, %xmm0, %xmm0
144; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
145; AVX1-NEXT:    retq
146;
147; AVX2-LABEL: testv4i64u:
148; AVX2:       # BB#0:
149; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
150; AVX2-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
151; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
152; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm2
153; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
154; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
155; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm3
156; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
157; AVX2-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
158; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
159; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
160; AVX2-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
161; AVX2-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
162; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
163; AVX2-NEXT:    retq
164;
165; AVX512CDVL-LABEL: testv4i64u:
166; AVX512CDVL:       # BB#0:
167; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
168; AVX512CDVL-NEXT:    vpsubq %ymm0, %ymm1, %ymm1
169; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
170; AVX512CDVL-NEXT:    vplzcntq %ymm0, %ymm0
171; AVX512CDVL-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
172; AVX512CDVL-NEXT:    vpsubq %ymm0, %ymm1, %ymm0
173; AVX512CDVL-NEXT:    retq
174;
175; AVX512CD-LABEL: testv4i64u:
176; AVX512CD:       # BB#0:
177; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
178; AVX512CD-NEXT:    vpsubq %ymm0, %ymm1, %ymm1
179; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
180; AVX512CD-NEXT:    vplzcntq %zmm0, %zmm0
181; AVX512CD-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
182; AVX512CD-NEXT:    vpsubq %ymm0, %ymm1, %ymm0
183; AVX512CD-NEXT:    retq
184;
185; X32-AVX-LABEL: testv4i64u:
186; X32-AVX:       # BB#0:
187; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
188; X32-AVX-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
189; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
190; X32-AVX-NEXT:    vpsubq {{\.LCPI.*}}, %ymm0, %ymm0
191; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
192; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
193; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
194; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
195; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
196; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
197; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
198; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
199; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
200; X32-AVX-NEXT:    retl
201  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 -1)
202  ret <4 x i64> %out
203}
204
205define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
206; AVX1-LABEL: testv8i32:
207; AVX1:       # BB#0:
208; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
209; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
210; AVX1-NEXT:    vpsubd %xmm2, %xmm1, %xmm2
211; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm3
212; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
213; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
214; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
215; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,1,1]
216; AVX1-NEXT:    vpsubd %xmm3, %xmm2, %xmm2
217; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
218; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
219; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
220; AVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
221; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
222; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
223; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
224; AVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
225; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
226; AVX1-NEXT:    vpsadbw %xmm1, %xmm5, %xmm5
227; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
228; AVX1-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
229; AVX1-NEXT:    vpackuswb %xmm5, %xmm2, %xmm2
230; AVX1-NEXT:    vpsubd %xmm3, %xmm0, %xmm0
231; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm3
232; AVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
233; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
234; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
235; AVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
236; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
237; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
238; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm3
239; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
240; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
241; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
242; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
243; AVX1-NEXT:    retq
244;
245; AVX2-LABEL: testv8i32:
246; AVX2:       # BB#0:
247; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
248; AVX2-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
249; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
250; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm2
251; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
252; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
253; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm3
254; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
255; AVX2-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
256; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
257; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
258; AVX2-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
259; AVX2-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
260; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
261; AVX2-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
262; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
263; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
264; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
265; AVX2-NEXT:    retq
266;
267; AVX512CDVL-LABEL: testv8i32:
268; AVX512CDVL:       # BB#0:
269; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
270; AVX512CDVL-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
271; AVX512CDVL-NEXT:    vpandd %ymm2, %ymm0, %ymm0
272; AVX512CDVL-NEXT:    vpsubd {{.*}}(%rip){1to8}, %ymm0, %ymm0
273; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
274; AVX512CDVL-NEXT:    vpandq %ymm2, %ymm0, %ymm3
275; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
276; AVX512CDVL-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
277; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
278; AVX512CDVL-NEXT:    vpandq %ymm2, %ymm0, %ymm0
279; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
280; AVX512CDVL-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
281; AVX512CDVL-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
282; AVX512CDVL-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
283; AVX512CDVL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
284; AVX512CDVL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
285; AVX512CDVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
286; AVX512CDVL-NEXT:    retq
287;
288; AVX512CD-LABEL: testv8i32:
289; AVX512CD:       # BB#0:
290; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
291; AVX512CD-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
292; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm0
293; AVX512CD-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm2
294; AVX512CD-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
295; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
296; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm3
297; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
298; AVX512CD-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
299; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
300; AVX512CD-NEXT:    vpand %ymm2, %ymm0, %ymm0
301; AVX512CD-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
302; AVX512CD-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
303; AVX512CD-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
304; AVX512CD-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
305; AVX512CD-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
306; AVX512CD-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
307; AVX512CD-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
308; AVX512CD-NEXT:    retq
309;
310; X32-AVX-LABEL: testv8i32:
311; X32-AVX:       # BB#0:
312; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
313; X32-AVX-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
314; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
315; X32-AVX-NEXT:    vpbroadcastd {{\.LCPI.*}}, %ymm2
316; X32-AVX-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
317; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
318; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
319; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
320; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
321; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
322; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
323; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
324; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
325; X32-AVX-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
326; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
327; X32-AVX-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
328; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
329; X32-AVX-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
330; X32-AVX-NEXT:    retl
331  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 0)
332  ret <8 x i32> %out
333}
334
335define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
336; AVX1-LABEL: testv8i32u:
337; AVX1:       # BB#0:
338; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
339; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
340; AVX1-NEXT:    vpsubd %xmm2, %xmm1, %xmm2
341; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm3
342; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
343; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
344; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
345; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,1,1]
346; AVX1-NEXT:    vpsubd %xmm3, %xmm2, %xmm2
347; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
348; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
349; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
350; AVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
351; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
352; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
353; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
354; AVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
355; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
356; AVX1-NEXT:    vpsadbw %xmm1, %xmm5, %xmm5
357; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
358; AVX1-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
359; AVX1-NEXT:    vpackuswb %xmm5, %xmm2, %xmm2
360; AVX1-NEXT:    vpsubd %xmm3, %xmm0, %xmm0
361; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm3
362; AVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
363; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
364; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
365; AVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
366; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
367; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
368; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm3
369; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
370; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
371; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
372; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
373; AVX1-NEXT:    retq
374;
375; AVX2-LABEL: testv8i32u:
376; AVX2:       # BB#0:
377; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
378; AVX2-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
379; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
380; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm2
381; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
382; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
383; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm3
384; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
385; AVX2-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
386; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
387; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
388; AVX2-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
389; AVX2-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
390; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
391; AVX2-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
392; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
393; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
394; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
395; AVX2-NEXT:    retq
396;
397; AVX512CDVL-LABEL: testv8i32u:
398; AVX512CDVL:       # BB#0:
399; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
400; AVX512CDVL-NEXT:    vpsubd %ymm0, %ymm1, %ymm1
401; AVX512CDVL-NEXT:    vpandd %ymm1, %ymm0, %ymm0
402; AVX512CDVL-NEXT:    vplzcntd %ymm0, %ymm0
403; AVX512CDVL-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
404; AVX512CDVL-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
405; AVX512CDVL-NEXT:    retq
406;
407; AVX512CD-LABEL: testv8i32u:
408; AVX512CD:       # BB#0:
409; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
410; AVX512CD-NEXT:    vpsubd %ymm0, %ymm1, %ymm1
411; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
412; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
413; AVX512CD-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
414; AVX512CD-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
415; AVX512CD-NEXT:    retq
416;
417; X32-AVX-LABEL: testv8i32u:
418; X32-AVX:       # BB#0:
419; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
420; X32-AVX-NEXT:    vpsubd %ymm0, %ymm1, %ymm2
421; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
422; X32-AVX-NEXT:    vpbroadcastd {{\.LCPI.*}}, %ymm2
423; X32-AVX-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
424; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
425; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
426; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
427; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
428; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
429; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
430; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
431; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
432; X32-AVX-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
433; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm2, %ymm2
434; X32-AVX-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
435; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
436; X32-AVX-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
437; X32-AVX-NEXT:    retl
438  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 -1)
439  ret <8 x i32> %out
440}
441
442define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
443; AVX1-LABEL: testv16i16:
444; AVX1:       # BB#0:
445; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
446; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
447; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
448; AVX1-NEXT:    vpsubw %xmm0, %xmm2, %xmm2
449; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
450; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
451; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1]
452; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm2
453; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
454; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm4
455; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
456; AVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
457; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
458; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
459; AVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
460; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
461; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm4
462; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
463; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
464; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
465; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
466; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm1
467; AVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
468; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
469; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
470; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
471; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
472; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
473; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
474; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
475; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
476; AVX1-NEXT:    retq
477;
478; AVX2-LABEL: testv16i16:
479; AVX2:       # BB#0:
480; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
481; AVX2-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
482; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
483; AVX2-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
484; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
485; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
486; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
487; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
488; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
489; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
490; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
491; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
492; AVX2-NEXT:    vpsllw $8, %ymm0, %ymm1
493; AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
494; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
495; AVX2-NEXT:    retq
496;
497; AVX512CDVL-LABEL: testv16i16:
498; AVX512CDVL:       # BB#0:
499; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
500; AVX512CDVL-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
501; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
502; AVX512CDVL-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
503; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
504; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm2
505; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
506; AVX512CDVL-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
507; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
508; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
509; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
510; AVX512CDVL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
511; AVX512CDVL-NEXT:    vpsllw $8, %ymm0, %ymm1
512; AVX512CDVL-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
513; AVX512CDVL-NEXT:    vpsrlw $8, %ymm0, %ymm0
514; AVX512CDVL-NEXT:    retq
515;
516; AVX512CD-LABEL: testv16i16:
517; AVX512CD:       # BB#0:
518; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
519; AVX512CD-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
520; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
521; AVX512CD-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
522; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
523; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm2
524; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
525; AVX512CD-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
526; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
527; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
528; AVX512CD-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
529; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
530; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm1
531; AVX512CD-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
532; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
533; AVX512CD-NEXT:    retq
534;
535; X32-AVX-LABEL: testv16i16:
536; X32-AVX:       # BB#0:
537; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
538; X32-AVX-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
539; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
540; X32-AVX-NEXT:    vpsubw {{\.LCPI.*}}, %ymm0, %ymm0
541; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
542; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm2
543; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
544; X32-AVX-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
545; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
546; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
547; X32-AVX-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
548; X32-AVX-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
549; X32-AVX-NEXT:    vpsllw $8, %ymm0, %ymm1
550; X32-AVX-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
551; X32-AVX-NEXT:    vpsrlw $8, %ymm0, %ymm0
552; X32-AVX-NEXT:    retl
553  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 0)
554  ret <16 x i16> %out
555}
556
557define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
558; AVX1-LABEL: testv16i16u:
559; AVX1:       # BB#0:
560; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
561; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
562; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
563; AVX1-NEXT:    vpsubw %xmm0, %xmm2, %xmm2
564; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
565; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
566; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1]
567; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm2
568; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
569; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm4
570; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
571; AVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
572; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
573; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
574; AVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
575; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
576; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm4
577; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
578; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
579; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
580; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
581; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm1
582; AVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
583; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
584; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
585; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
586; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
587; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
588; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
589; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
590; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
591; AVX1-NEXT:    retq
592;
593; AVX2-LABEL: testv16i16u:
594; AVX2:       # BB#0:
595; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
596; AVX2-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
597; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
598; AVX2-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
599; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
600; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
601; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
602; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
603; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
604; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
605; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
606; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
607; AVX2-NEXT:    vpsllw $8, %ymm0, %ymm1
608; AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
609; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
610; AVX2-NEXT:    retq
611;
612; AVX512CDVL-LABEL: testv16i16u:
613; AVX512CDVL:       # BB#0:
614; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
615; AVX512CDVL-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
616; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
617; AVX512CDVL-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
618; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
619; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm2
620; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
621; AVX512CDVL-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
622; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
623; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
624; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
625; AVX512CDVL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
626; AVX512CDVL-NEXT:    vpsllw $8, %ymm0, %ymm1
627; AVX512CDVL-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
628; AVX512CDVL-NEXT:    vpsrlw $8, %ymm0, %ymm0
629; AVX512CDVL-NEXT:    retq
630;
631; AVX512CD-LABEL: testv16i16u:
632; AVX512CD:       # BB#0:
633; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
634; AVX512CD-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
635; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
636; AVX512CD-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
637; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
638; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm2
639; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
640; AVX512CD-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
641; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
642; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
643; AVX512CD-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
644; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
645; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm1
646; AVX512CD-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
647; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
648; AVX512CD-NEXT:    retq
649;
650; X32-AVX-LABEL: testv16i16u:
651; X32-AVX:       # BB#0:
652; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
653; X32-AVX-NEXT:    vpsubw %ymm0, %ymm1, %ymm1
654; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
655; X32-AVX-NEXT:    vpsubw {{\.LCPI.*}}, %ymm0, %ymm0
656; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
657; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm2
658; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
659; X32-AVX-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
660; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
661; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
662; X32-AVX-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
663; X32-AVX-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
664; X32-AVX-NEXT:    vpsllw $8, %ymm0, %ymm1
665; X32-AVX-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
666; X32-AVX-NEXT:    vpsrlw $8, %ymm0, %ymm0
667; X32-AVX-NEXT:    retl
668  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 -1)
669  ret <16 x i16> %out
670}
671
672define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
673; AVX1-LABEL: testv32i8:
674; AVX1:       # BB#0:
675; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
676; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
677; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
678; AVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm2
679; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
680; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
681; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
682; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
683; AVX1-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
684; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
685; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
686; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
687; AVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
688; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
689; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
690; AVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
691; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
692; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
693; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm2
694; AVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
695; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
696; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
697; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
698; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
699; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
700; AVX1-NEXT:    retq
701;
702; AVX2-LABEL: testv32i8:
703; AVX2:       # BB#0:
704; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
705; AVX2-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
706; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
707; AVX2-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
708; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
709; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
710; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
711; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
712; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
713; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
714; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
715; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
716; AVX2-NEXT:    retq
717;
718; AVX512CDVL-LABEL: testv32i8:
719; AVX512CDVL:       # BB#0:
720; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
721; AVX512CDVL-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
722; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
723; AVX512CDVL-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
724; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
725; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm2
726; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
727; AVX512CDVL-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
728; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
729; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
730; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
731; AVX512CDVL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
732; AVX512CDVL-NEXT:    retq
733;
734; AVX512CD-LABEL: testv32i8:
735; AVX512CD:       # BB#0:
736; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
737; AVX512CD-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
738; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
739; AVX512CD-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
740; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
741; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm2
742; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
743; AVX512CD-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
744; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
745; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
746; AVX512CD-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
747; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
748; AVX512CD-NEXT:    retq
749;
750; X32-AVX-LABEL: testv32i8:
751; X32-AVX:       # BB#0:
752; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
753; X32-AVX-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
754; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
755; X32-AVX-NEXT:    vpsubb {{\.LCPI.*}}, %ymm0, %ymm0
756; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
757; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm2
758; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
759; X32-AVX-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
760; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
761; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
762; X32-AVX-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
763; X32-AVX-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
764; X32-AVX-NEXT:    retl
765  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0)
766  ret <32 x i8> %out
767}
768
769define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
770; AVX1-LABEL: testv32i8u:
771; AVX1:       # BB#0:
772; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
773; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
774; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
775; AVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm2
776; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
777; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
778; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
779; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
780; AVX1-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
781; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
782; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
783; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
784; AVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
785; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
786; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
787; AVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
788; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
789; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
790; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm2
791; AVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
792; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
793; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
794; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
795; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
796; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
797; AVX1-NEXT:    retq
798;
799; AVX2-LABEL: testv32i8u:
800; AVX2:       # BB#0:
801; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
802; AVX2-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
803; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
804; AVX2-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
805; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
806; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
807; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
808; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
809; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
810; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
811; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
812; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
813; AVX2-NEXT:    retq
814;
815; AVX512CDVL-LABEL: testv32i8u:
816; AVX512CDVL:       # BB#0:
817; AVX512CDVL-NEXT:    vpxord %ymm1, %ymm1, %ymm1
818; AVX512CDVL-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
819; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
820; AVX512CDVL-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
821; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
822; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm2
823; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
824; AVX512CDVL-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
825; AVX512CDVL-NEXT:    vpsrlw $4, %ymm0, %ymm0
826; AVX512CDVL-NEXT:    vpandq %ymm1, %ymm0, %ymm0
827; AVX512CDVL-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
828; AVX512CDVL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
829; AVX512CDVL-NEXT:    retq
830;
831; AVX512CD-LABEL: testv32i8u:
832; AVX512CD:       # BB#0:
833; AVX512CD-NEXT:    vpxor %ymm1, %ymm1, %ymm1
834; AVX512CD-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
835; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
836; AVX512CD-NEXT:    vpsubb {{.*}}(%rip), %ymm0, %ymm0
837; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
838; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm2
839; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
840; AVX512CD-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
841; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
842; AVX512CD-NEXT:    vpand %ymm1, %ymm0, %ymm0
843; AVX512CD-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
844; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
845; AVX512CD-NEXT:    retq
846;
847; X32-AVX-LABEL: testv32i8u:
848; X32-AVX:       # BB#0:
849; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
850; X32-AVX-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
851; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
852; X32-AVX-NEXT:    vpsubb {{\.LCPI.*}}, %ymm0, %ymm0
853; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
854; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm2
855; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
856; X32-AVX-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
857; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
858; X32-AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
859; X32-AVX-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
860; X32-AVX-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
861; X32-AVX-NEXT:    retl
862  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1)
863  ret <32 x i8> %out
864}
865
866define <4 x i64> @foldv4i64() nounwind {
867; AVX-LABEL: foldv4i64:
868; AVX:       # BB#0:
869; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
870; AVX-NEXT:    retq
871;
872; X32-AVX-LABEL: foldv4i64:
873; X32-AVX:       # BB#0:
874; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [256,0,4294967295,4294967295,0,0,255,0]
875; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
876; X32-AVX-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
877; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
878; X32-AVX-NEXT:    vpsubq {{\.LCPI.*}}, %ymm0, %ymm0
879; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
880; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
881; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
882; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
883; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
884; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
885; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
886; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
887; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
888; X32-AVX-NEXT:    retl
889  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)
890  ret <4 x i64> %out
891}
892
893define <4 x i64> @foldv4i64u() nounwind {
894; AVX-LABEL: foldv4i64u:
895; AVX:       # BB#0:
896; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
897; AVX-NEXT:    retq
898;
899; X32-AVX-LABEL: foldv4i64u:
900; X32-AVX:       # BB#0:
901; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [256,0,4294967295,4294967295,0,0,255,0]
902; X32-AVX-NEXT:    vpxor %ymm1, %ymm1, %ymm1
903; X32-AVX-NEXT:    vpsubq %ymm0, %ymm1, %ymm2
904; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
905; X32-AVX-NEXT:    vpsubq {{\.LCPI.*}}, %ymm0, %ymm0
906; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
907; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm3
908; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
909; X32-AVX-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
910; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
911; X32-AVX-NEXT:    vpand %ymm2, %ymm0, %ymm0
912; X32-AVX-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
913; X32-AVX-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
914; X32-AVX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0
915; X32-AVX-NEXT:    retl
916  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)
917  ret <4 x i64> %out
918}
919
920define <8 x i32> @foldv8i32() nounwind {
921; AVX-LABEL: foldv8i32:
922; AVX:       # BB#0:
923; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
924; AVX-NEXT:    retq
925;
926; X32-AVX-LABEL: foldv8i32:
927; X32-AVX:       # BB#0:
928; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
929; X32-AVX-NEXT:    retl
930  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)
931  ret <8 x i32> %out
932}
933
934define <8 x i32> @foldv8i32u() nounwind {
935; AVX-LABEL: foldv8i32u:
936; AVX:       # BB#0:
937; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
938; AVX-NEXT:    retq
939;
940; X32-AVX-LABEL: foldv8i32u:
941; X32-AVX:       # BB#0:
942; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
943; X32-AVX-NEXT:    retl
944  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)
945  ret <8 x i32> %out
946}
947
948define <16 x i16> @foldv16i16() nounwind {
949; AVX-LABEL: foldv16i16:
950; AVX:       # BB#0:
951; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
952; AVX-NEXT:    retq
953;
954; X32-AVX-LABEL: foldv16i16:
955; X32-AVX:       # BB#0:
956; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
957; X32-AVX-NEXT:    retl
958  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)
959  ret <16 x i16> %out
960}
961
962define <16 x i16> @foldv16i16u() nounwind {
963; AVX-LABEL: foldv16i16u:
964; AVX:       # BB#0:
965; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
966; AVX-NEXT:    retq
967;
968; X32-AVX-LABEL: foldv16i16u:
969; X32-AVX:       # BB#0:
970; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
971; X32-AVX-NEXT:    retl
972  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)
973  ret <16 x i16> %out
974}
975
976define <32 x i8> @foldv32i8() nounwind {
977; AVX-LABEL: foldv32i8:
978; AVX:       # BB#0:
979; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
980; AVX-NEXT:    retq
981;
982; X32-AVX-LABEL: foldv32i8:
983; X32-AVX:       # BB#0:
984; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
985; X32-AVX-NEXT:    retl
986  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)
987  ret <32 x i8> %out
988}
989
990define <32 x i8> @foldv32i8u() nounwind {
991; AVX-LABEL: foldv32i8u:
992; AVX:       # BB#0:
993; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
994; AVX-NEXT:    retq
995;
996; X32-AVX-LABEL: foldv32i8u:
997; X32-AVX:       # BB#0:
998; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
999; X32-AVX-NEXT:    retl
1000  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)
1001  ret <32 x i8> %out
1002}
1003
1004declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
1005declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
1006declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
1007declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
1008