1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512cd,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512CDVL
9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512cd,-avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512CD
10;
11; Just one 32-bit run to make sure we do reasonable things for i64 tzcnt.
12; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE41
13
14define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
15; SSE2-LABEL: testv2i64:
16; SSE2:       # BB#0:
17; SSE2-NEXT:    pxor %xmm1, %xmm1
18; SSE2-NEXT:    pxor %xmm2, %xmm2
19; SSE2-NEXT:    psubq %xmm0, %xmm2
20; SSE2-NEXT:    pand %xmm0, %xmm2
21; SSE2-NEXT:    psubq {{.*}}(%rip), %xmm2
22; SSE2-NEXT:    movdqa %xmm2, %xmm0
23; SSE2-NEXT:    psrlq $1, %xmm0
24; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
25; SSE2-NEXT:    psubq %xmm0, %xmm2
26; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
27; SSE2-NEXT:    movdqa %xmm2, %xmm3
28; SSE2-NEXT:    pand %xmm0, %xmm3
29; SSE2-NEXT:    psrlq $2, %xmm2
30; SSE2-NEXT:    pand %xmm0, %xmm2
31; SSE2-NEXT:    paddq %xmm3, %xmm2
32; SSE2-NEXT:    movdqa %xmm2, %xmm0
33; SSE2-NEXT:    psrlq $4, %xmm0
34; SSE2-NEXT:    paddq %xmm2, %xmm0
35; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
36; SSE2-NEXT:    psadbw %xmm1, %xmm0
37; SSE2-NEXT:    retq
38;
39; SSE3-LABEL: testv2i64:
40; SSE3:       # BB#0:
41; SSE3-NEXT:    pxor %xmm1, %xmm1
42; SSE3-NEXT:    pxor %xmm2, %xmm2
43; SSE3-NEXT:    psubq %xmm0, %xmm2
44; SSE3-NEXT:    pand %xmm0, %xmm2
45; SSE3-NEXT:    psubq {{.*}}(%rip), %xmm2
46; SSE3-NEXT:    movdqa %xmm2, %xmm0
47; SSE3-NEXT:    psrlq $1, %xmm0
48; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
49; SSE3-NEXT:    psubq %xmm0, %xmm2
50; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
51; SSE3-NEXT:    movdqa %xmm2, %xmm3
52; SSE3-NEXT:    pand %xmm0, %xmm3
53; SSE3-NEXT:    psrlq $2, %xmm2
54; SSE3-NEXT:    pand %xmm0, %xmm2
55; SSE3-NEXT:    paddq %xmm3, %xmm2
56; SSE3-NEXT:    movdqa %xmm2, %xmm0
57; SSE3-NEXT:    psrlq $4, %xmm0
58; SSE3-NEXT:    paddq %xmm2, %xmm0
59; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
60; SSE3-NEXT:    psadbw %xmm1, %xmm0
61; SSE3-NEXT:    retq
62;
63; SSSE3-LABEL: testv2i64:
64; SSSE3:       # BB#0:
65; SSSE3-NEXT:    pxor %xmm1, %xmm1
66; SSSE3-NEXT:    pxor %xmm2, %xmm2
67; SSSE3-NEXT:    psubq %xmm0, %xmm2
68; SSSE3-NEXT:    pand %xmm0, %xmm2
69; SSSE3-NEXT:    psubq {{.*}}(%rip), %xmm2
70; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
71; SSSE3-NEXT:    movdqa %xmm2, %xmm4
72; SSSE3-NEXT:    pand %xmm3, %xmm4
73; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
74; SSSE3-NEXT:    movdqa %xmm0, %xmm5
75; SSSE3-NEXT:    pshufb %xmm4, %xmm5
76; SSSE3-NEXT:    psrlw $4, %xmm2
77; SSSE3-NEXT:    pand %xmm3, %xmm2
78; SSSE3-NEXT:    pshufb %xmm2, %xmm0
79; SSSE3-NEXT:    paddb %xmm5, %xmm0
80; SSSE3-NEXT:    psadbw %xmm1, %xmm0
81; SSSE3-NEXT:    retq
82;
83; SSE41-LABEL: testv2i64:
84; SSE41:       # BB#0:
85; SSE41-NEXT:    pxor %xmm1, %xmm1
86; SSE41-NEXT:    pxor %xmm2, %xmm2
87; SSE41-NEXT:    psubq %xmm0, %xmm2
88; SSE41-NEXT:    pand %xmm0, %xmm2
89; SSE41-NEXT:    psubq {{.*}}(%rip), %xmm2
90; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
91; SSE41-NEXT:    movdqa %xmm2, %xmm4
92; SSE41-NEXT:    pand %xmm3, %xmm4
93; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
94; SSE41-NEXT:    movdqa %xmm0, %xmm5
95; SSE41-NEXT:    pshufb %xmm4, %xmm5
96; SSE41-NEXT:    psrlw $4, %xmm2
97; SSE41-NEXT:    pand %xmm3, %xmm2
98; SSE41-NEXT:    pshufb %xmm2, %xmm0
99; SSE41-NEXT:    paddb %xmm5, %xmm0
100; SSE41-NEXT:    psadbw %xmm1, %xmm0
101; SSE41-NEXT:    retq
102;
103; AVX1-LABEL: testv2i64:
104; AVX1:       # BB#0:
105; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
106; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
107; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
108; AVX1-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
109; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
110; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm3
111; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
112; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
113; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
114; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
115; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
116; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
117; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
118; AVX1-NEXT:    retq
119;
120; AVX2-LABEL: testv2i64:
121; AVX2:       # BB#0:
122; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
123; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
124; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
125; AVX2-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
126; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
127; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm3
128; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
129; AVX2-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
130; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
131; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
132; AVX2-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
133; AVX2-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
134; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
135; AVX2-NEXT:    retq
136;
137; AVX512CDVL-LABEL: testv2i64:
138; AVX512CDVL:       # BB#0:
139; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
140; AVX512CDVL-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
141; AVX512CDVL-NEXT:    vpandq %xmm2, %xmm0, %xmm0
142; AVX512CDVL-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
143; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
144; AVX512CDVL-NEXT:    vpandq %xmm2, %xmm0, %xmm3
145; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
146; AVX512CDVL-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
147; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
148; AVX512CDVL-NEXT:    vpandq %xmm2, %xmm0, %xmm0
149; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
150; AVX512CDVL-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
151; AVX512CDVL-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
152; AVX512CDVL-NEXT:    retq
153;
154; AVX512CD-LABEL: testv2i64:
155; AVX512CD:       # BB#0:
156; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
157; AVX512CD-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
158; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm0
159; AVX512CD-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
160; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
161; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm3
162; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
163; AVX512CD-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
164; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
165; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm0
166; AVX512CD-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
167; AVX512CD-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
168; AVX512CD-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
169; AVX512CD-NEXT:    retq
170;
171; X32-SSE-LABEL: testv2i64:
172; X32-SSE:       # BB#0:
173; X32-SSE-NEXT:    pxor %xmm1, %xmm1
174; X32-SSE-NEXT:    pxor %xmm2, %xmm2
175; X32-SSE-NEXT:    psubq %xmm0, %xmm2
176; X32-SSE-NEXT:    pand %xmm0, %xmm2
177; X32-SSE-NEXT:    psubq {{\.LCPI.*}}, %xmm2
178; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
179; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
180; X32-SSE-NEXT:    pand %xmm3, %xmm4
181; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
182; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
183; X32-SSE-NEXT:    pshufb %xmm4, %xmm5
184; X32-SSE-NEXT:    psrlw $4, %xmm2
185; X32-SSE-NEXT:    pand %xmm3, %xmm2
186; X32-SSE-NEXT:    pshufb %xmm2, %xmm0
187; X32-SSE-NEXT:    paddb %xmm5, %xmm0
188; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
189; X32-SSE-NEXT:    retl
190  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 0)
191  ret <2 x i64> %out
192}
193
194define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
195; SSE2-LABEL: testv2i64u:
196; SSE2:       # BB#0:
197; SSE2-NEXT:    pxor %xmm1, %xmm1
198; SSE2-NEXT:    pxor %xmm2, %xmm2
199; SSE2-NEXT:    psubq %xmm0, %xmm2
200; SSE2-NEXT:    pand %xmm0, %xmm2
201; SSE2-NEXT:    psubq {{.*}}(%rip), %xmm2
202; SSE2-NEXT:    movdqa %xmm2, %xmm0
203; SSE2-NEXT:    psrlq $1, %xmm0
204; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
205; SSE2-NEXT:    psubq %xmm0, %xmm2
206; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
207; SSE2-NEXT:    movdqa %xmm2, %xmm3
208; SSE2-NEXT:    pand %xmm0, %xmm3
209; SSE2-NEXT:    psrlq $2, %xmm2
210; SSE2-NEXT:    pand %xmm0, %xmm2
211; SSE2-NEXT:    paddq %xmm3, %xmm2
212; SSE2-NEXT:    movdqa %xmm2, %xmm0
213; SSE2-NEXT:    psrlq $4, %xmm0
214; SSE2-NEXT:    paddq %xmm2, %xmm0
215; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
216; SSE2-NEXT:    psadbw %xmm1, %xmm0
217; SSE2-NEXT:    retq
218;
219; SSE3-LABEL: testv2i64u:
220; SSE3:       # BB#0:
221; SSE3-NEXT:    pxor %xmm1, %xmm1
222; SSE3-NEXT:    pxor %xmm2, %xmm2
223; SSE3-NEXT:    psubq %xmm0, %xmm2
224; SSE3-NEXT:    pand %xmm0, %xmm2
225; SSE3-NEXT:    psubq {{.*}}(%rip), %xmm2
226; SSE3-NEXT:    movdqa %xmm2, %xmm0
227; SSE3-NEXT:    psrlq $1, %xmm0
228; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
229; SSE3-NEXT:    psubq %xmm0, %xmm2
230; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
231; SSE3-NEXT:    movdqa %xmm2, %xmm3
232; SSE3-NEXT:    pand %xmm0, %xmm3
233; SSE3-NEXT:    psrlq $2, %xmm2
234; SSE3-NEXT:    pand %xmm0, %xmm2
235; SSE3-NEXT:    paddq %xmm3, %xmm2
236; SSE3-NEXT:    movdqa %xmm2, %xmm0
237; SSE3-NEXT:    psrlq $4, %xmm0
238; SSE3-NEXT:    paddq %xmm2, %xmm0
239; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
240; SSE3-NEXT:    psadbw %xmm1, %xmm0
241; SSE3-NEXT:    retq
242;
243; SSSE3-LABEL: testv2i64u:
244; SSSE3:       # BB#0:
245; SSSE3-NEXT:    pxor %xmm1, %xmm1
246; SSSE3-NEXT:    pxor %xmm2, %xmm2
247; SSSE3-NEXT:    psubq %xmm0, %xmm2
248; SSSE3-NEXT:    pand %xmm0, %xmm2
249; SSSE3-NEXT:    psubq {{.*}}(%rip), %xmm2
250; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
251; SSSE3-NEXT:    movdqa %xmm2, %xmm4
252; SSSE3-NEXT:    pand %xmm3, %xmm4
253; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
254; SSSE3-NEXT:    movdqa %xmm0, %xmm5
255; SSSE3-NEXT:    pshufb %xmm4, %xmm5
256; SSSE3-NEXT:    psrlw $4, %xmm2
257; SSSE3-NEXT:    pand %xmm3, %xmm2
258; SSSE3-NEXT:    pshufb %xmm2, %xmm0
259; SSSE3-NEXT:    paddb %xmm5, %xmm0
260; SSSE3-NEXT:    psadbw %xmm1, %xmm0
261; SSSE3-NEXT:    retq
262;
263; SSE41-LABEL: testv2i64u:
264; SSE41:       # BB#0:
265; SSE41-NEXT:    pxor %xmm1, %xmm1
266; SSE41-NEXT:    pxor %xmm2, %xmm2
267; SSE41-NEXT:    psubq %xmm0, %xmm2
268; SSE41-NEXT:    pand %xmm0, %xmm2
269; SSE41-NEXT:    psubq {{.*}}(%rip), %xmm2
270; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
271; SSE41-NEXT:    movdqa %xmm2, %xmm4
272; SSE41-NEXT:    pand %xmm3, %xmm4
273; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
274; SSE41-NEXT:    movdqa %xmm0, %xmm5
275; SSE41-NEXT:    pshufb %xmm4, %xmm5
276; SSE41-NEXT:    psrlw $4, %xmm2
277; SSE41-NEXT:    pand %xmm3, %xmm2
278; SSE41-NEXT:    pshufb %xmm2, %xmm0
279; SSE41-NEXT:    paddb %xmm5, %xmm0
280; SSE41-NEXT:    psadbw %xmm1, %xmm0
281; SSE41-NEXT:    retq
282;
283; AVX1-LABEL: testv2i64u:
284; AVX1:       # BB#0:
285; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
286; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
287; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
288; AVX1-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
289; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
290; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm3
291; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
292; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
293; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
294; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
295; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
296; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
297; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
298; AVX1-NEXT:    retq
299;
300; AVX2-LABEL: testv2i64u:
301; AVX2:       # BB#0:
302; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
303; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm2
304; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
305; AVX2-NEXT:    vpsubq {{.*}}(%rip), %xmm0, %xmm0
306; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
307; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm3
308; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
309; AVX2-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
310; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
311; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
312; AVX2-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
313; AVX2-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
314; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
315; AVX2-NEXT:    retq
316;
317; AVX512CDVL-LABEL: testv2i64u:
318; AVX512CDVL:       # BB#0:
319; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
320; AVX512CDVL-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
321; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
322; AVX512CDVL-NEXT:    vplzcntq %xmm0, %xmm0
323; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm1 = [63,63]
324; AVX512CDVL-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
325; AVX512CDVL-NEXT:    retq
326;
327; AVX512CD-LABEL: testv2i64u:
328; AVX512CD:       # BB#0:
329; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
330; AVX512CD-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
331; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
332; AVX512CD-NEXT:    vplzcntq %zmm0, %zmm0
333; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm1 = [63,63]
334; AVX512CD-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
335; AVX512CD-NEXT:    retq
336;
337; X32-SSE-LABEL: testv2i64u:
338; X32-SSE:       # BB#0:
339; X32-SSE-NEXT:    pxor %xmm1, %xmm1
340; X32-SSE-NEXT:    pxor %xmm2, %xmm2
341; X32-SSE-NEXT:    psubq %xmm0, %xmm2
342; X32-SSE-NEXT:    pand %xmm0, %xmm2
343; X32-SSE-NEXT:    psubq {{\.LCPI.*}}, %xmm2
344; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
345; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
346; X32-SSE-NEXT:    pand %xmm3, %xmm4
347; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
348; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
349; X32-SSE-NEXT:    pshufb %xmm4, %xmm5
350; X32-SSE-NEXT:    psrlw $4, %xmm2
351; X32-SSE-NEXT:    pand %xmm3, %xmm2
352; X32-SSE-NEXT:    pshufb %xmm2, %xmm0
353; X32-SSE-NEXT:    paddb %xmm5, %xmm0
354; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
355; X32-SSE-NEXT:    retl
356  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 -1)
357  ret <2 x i64> %out
358}
359
360define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
361; SSE2-LABEL: testv4i32:
362; SSE2:       # BB#0:
363; SSE2-NEXT:    pxor %xmm1, %xmm1
364; SSE2-NEXT:    pxor %xmm2, %xmm2
365; SSE2-NEXT:    psubd %xmm0, %xmm2
366; SSE2-NEXT:    pand %xmm0, %xmm2
367; SSE2-NEXT:    psubd {{.*}}(%rip), %xmm2
368; SSE2-NEXT:    movdqa %xmm2, %xmm0
369; SSE2-NEXT:    psrld $1, %xmm0
370; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
371; SSE2-NEXT:    psubd %xmm0, %xmm2
372; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459]
373; SSE2-NEXT:    movdqa %xmm2, %xmm3
374; SSE2-NEXT:    pand %xmm0, %xmm3
375; SSE2-NEXT:    psrld $2, %xmm2
376; SSE2-NEXT:    pand %xmm0, %xmm2
377; SSE2-NEXT:    paddd %xmm3, %xmm2
378; SSE2-NEXT:    movdqa %xmm2, %xmm0
379; SSE2-NEXT:    psrld $4, %xmm0
380; SSE2-NEXT:    paddd %xmm2, %xmm0
381; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
382; SSE2-NEXT:    movdqa %xmm0, %xmm2
383; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
384; SSE2-NEXT:    psadbw %xmm1, %xmm2
385; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
386; SSE2-NEXT:    psadbw %xmm1, %xmm0
387; SSE2-NEXT:    packuswb %xmm2, %xmm0
388; SSE2-NEXT:    retq
389;
390; SSE3-LABEL: testv4i32:
391; SSE3:       # BB#0:
392; SSE3-NEXT:    pxor %xmm1, %xmm1
393; SSE3-NEXT:    pxor %xmm2, %xmm2
394; SSE3-NEXT:    psubd %xmm0, %xmm2
395; SSE3-NEXT:    pand %xmm0, %xmm2
396; SSE3-NEXT:    psubd {{.*}}(%rip), %xmm2
397; SSE3-NEXT:    movdqa %xmm2, %xmm0
398; SSE3-NEXT:    psrld $1, %xmm0
399; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
400; SSE3-NEXT:    psubd %xmm0, %xmm2
401; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459]
402; SSE3-NEXT:    movdqa %xmm2, %xmm3
403; SSE3-NEXT:    pand %xmm0, %xmm3
404; SSE3-NEXT:    psrld $2, %xmm2
405; SSE3-NEXT:    pand %xmm0, %xmm2
406; SSE3-NEXT:    paddd %xmm3, %xmm2
407; SSE3-NEXT:    movdqa %xmm2, %xmm0
408; SSE3-NEXT:    psrld $4, %xmm0
409; SSE3-NEXT:    paddd %xmm2, %xmm0
410; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
411; SSE3-NEXT:    movdqa %xmm0, %xmm2
412; SSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
413; SSE3-NEXT:    psadbw %xmm1, %xmm2
414; SSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
415; SSE3-NEXT:    psadbw %xmm1, %xmm0
416; SSE3-NEXT:    packuswb %xmm2, %xmm0
417; SSE3-NEXT:    retq
418;
419; SSSE3-LABEL: testv4i32:
420; SSSE3:       # BB#0:
421; SSSE3-NEXT:    pxor %xmm1, %xmm1
422; SSSE3-NEXT:    pxor %xmm2, %xmm2
423; SSSE3-NEXT:    psubd %xmm0, %xmm2
424; SSSE3-NEXT:    pand %xmm0, %xmm2
425; SSSE3-NEXT:    psubd {{.*}}(%rip), %xmm2
426; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
427; SSSE3-NEXT:    movdqa %xmm2, %xmm4
428; SSSE3-NEXT:    pand %xmm3, %xmm4
429; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
430; SSSE3-NEXT:    movdqa %xmm0, %xmm5
431; SSSE3-NEXT:    pshufb %xmm4, %xmm5
432; SSSE3-NEXT:    psrlw $4, %xmm2
433; SSSE3-NEXT:    pand %xmm3, %xmm2
434; SSSE3-NEXT:    pshufb %xmm2, %xmm0
435; SSSE3-NEXT:    paddb %xmm5, %xmm0
436; SSSE3-NEXT:    movdqa %xmm0, %xmm2
437; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
438; SSSE3-NEXT:    psadbw %xmm1, %xmm2
439; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
440; SSSE3-NEXT:    psadbw %xmm1, %xmm0
441; SSSE3-NEXT:    packuswb %xmm2, %xmm0
442; SSSE3-NEXT:    retq
443;
444; SSE41-LABEL: testv4i32:
445; SSE41:       # BB#0:
446; SSE41-NEXT:    pxor %xmm1, %xmm1
447; SSE41-NEXT:    pxor %xmm2, %xmm2
448; SSE41-NEXT:    psubd %xmm0, %xmm2
449; SSE41-NEXT:    pand %xmm0, %xmm2
450; SSE41-NEXT:    psubd {{.*}}(%rip), %xmm2
451; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
452; SSE41-NEXT:    movdqa %xmm2, %xmm3
453; SSE41-NEXT:    pand %xmm0, %xmm3
454; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
455; SSE41-NEXT:    movdqa %xmm4, %xmm5
456; SSE41-NEXT:    pshufb %xmm3, %xmm5
457; SSE41-NEXT:    psrlw $4, %xmm2
458; SSE41-NEXT:    pand %xmm0, %xmm2
459; SSE41-NEXT:    pshufb %xmm2, %xmm4
460; SSE41-NEXT:    paddb %xmm5, %xmm4
461; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
462; SSE41-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
463; SSE41-NEXT:    psadbw %xmm1, %xmm4
464; SSE41-NEXT:    psadbw %xmm1, %xmm0
465; SSE41-NEXT:    packuswb %xmm4, %xmm0
466; SSE41-NEXT:    retq
467;
468; AVX1-LABEL: testv4i32:
469; AVX1:       # BB#0:
470; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
471; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
472; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
473; AVX1-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
474; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
475; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm3
476; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
477; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
478; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
479; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
480; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
481; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
482; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
483; AVX1-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
484; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
485; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
486; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
487; AVX1-NEXT:    retq
488;
489; AVX2-LABEL: testv4i32:
490; AVX2:       # BB#0:
491; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
492; AVX2-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
493; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
494; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
495; AVX2-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
496; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
497; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm3
498; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
499; AVX2-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
500; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
501; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
502; AVX2-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
503; AVX2-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
504; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
505; AVX2-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
506; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
507; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
508; AVX2-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
509; AVX2-NEXT:    retq
510;
511; AVX512CDVL-LABEL: testv4i32:
512; AVX512CDVL:       # BB#0:
513; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
514; AVX512CDVL-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
515; AVX512CDVL-NEXT:    vpandd %xmm2, %xmm0, %xmm0
516; AVX512CDVL-NEXT:    vpsubd {{.*}}(%rip){1to4}, %xmm0, %xmm0
517; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
518; AVX512CDVL-NEXT:    vpandq %xmm2, %xmm0, %xmm3
519; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
520; AVX512CDVL-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
521; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
522; AVX512CDVL-NEXT:    vpandq %xmm2, %xmm0, %xmm0
523; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
524; AVX512CDVL-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
525; AVX512CDVL-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
526; AVX512CDVL-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
527; AVX512CDVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
528; AVX512CDVL-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
529; AVX512CDVL-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
530; AVX512CDVL-NEXT:    retq
531;
532; AVX512CD-LABEL: testv4i32:
533; AVX512CD:       # BB#0:
534; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
535; AVX512CD-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
536; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm0
537; AVX512CD-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
538; AVX512CD-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
539; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
540; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm3
541; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
542; AVX512CD-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
543; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
544; AVX512CD-NEXT:    vpand %xmm2, %xmm0, %xmm0
545; AVX512CD-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
546; AVX512CD-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
547; AVX512CD-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
548; AVX512CD-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
549; AVX512CD-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
550; AVX512CD-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
551; AVX512CD-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
552; AVX512CD-NEXT:    retq
553;
554; X32-SSE-LABEL: testv4i32:
555; X32-SSE:       # BB#0:
556; X32-SSE-NEXT:    pxor %xmm1, %xmm1
557; X32-SSE-NEXT:    pxor %xmm2, %xmm2
558; X32-SSE-NEXT:    psubd %xmm0, %xmm2
559; X32-SSE-NEXT:    pand %xmm0, %xmm2
560; X32-SSE-NEXT:    psubd {{\.LCPI.*}}, %xmm2
561; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
562; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
563; X32-SSE-NEXT:    pand %xmm0, %xmm3
564; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
565; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
566; X32-SSE-NEXT:    pshufb %xmm3, %xmm5
567; X32-SSE-NEXT:    psrlw $4, %xmm2
568; X32-SSE-NEXT:    pand %xmm0, %xmm2
569; X32-SSE-NEXT:    pshufb %xmm2, %xmm4
570; X32-SSE-NEXT:    paddb %xmm5, %xmm4
571; X32-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
572; X32-SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
573; X32-SSE-NEXT:    psadbw %xmm1, %xmm4
574; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
575; X32-SSE-NEXT:    packuswb %xmm4, %xmm0
576; X32-SSE-NEXT:    retl
577  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 0)
578  ret <4 x i32> %out
579}
580
581define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
582; SSE2-LABEL: testv4i32u:
583; SSE2:       # BB#0:
584; SSE2-NEXT:    pxor %xmm1, %xmm1
585; SSE2-NEXT:    pxor %xmm2, %xmm2
586; SSE2-NEXT:    psubd %xmm0, %xmm2
587; SSE2-NEXT:    pand %xmm0, %xmm2
588; SSE2-NEXT:    psubd {{.*}}(%rip), %xmm2
589; SSE2-NEXT:    movdqa %xmm2, %xmm0
590; SSE2-NEXT:    psrld $1, %xmm0
591; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
592; SSE2-NEXT:    psubd %xmm0, %xmm2
593; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459]
594; SSE2-NEXT:    movdqa %xmm2, %xmm3
595; SSE2-NEXT:    pand %xmm0, %xmm3
596; SSE2-NEXT:    psrld $2, %xmm2
597; SSE2-NEXT:    pand %xmm0, %xmm2
598; SSE2-NEXT:    paddd %xmm3, %xmm2
599; SSE2-NEXT:    movdqa %xmm2, %xmm0
600; SSE2-NEXT:    psrld $4, %xmm0
601; SSE2-NEXT:    paddd %xmm2, %xmm0
602; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
603; SSE2-NEXT:    movdqa %xmm0, %xmm2
604; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
605; SSE2-NEXT:    psadbw %xmm1, %xmm2
606; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
607; SSE2-NEXT:    psadbw %xmm1, %xmm0
608; SSE2-NEXT:    packuswb %xmm2, %xmm0
609; SSE2-NEXT:    retq
610;
611; SSE3-LABEL: testv4i32u:
612; SSE3:       # BB#0:
613; SSE3-NEXT:    pxor %xmm1, %xmm1
614; SSE3-NEXT:    pxor %xmm2, %xmm2
615; SSE3-NEXT:    psubd %xmm0, %xmm2
616; SSE3-NEXT:    pand %xmm0, %xmm2
617; SSE3-NEXT:    psubd {{.*}}(%rip), %xmm2
618; SSE3-NEXT:    movdqa %xmm2, %xmm0
619; SSE3-NEXT:    psrld $1, %xmm0
620; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
621; SSE3-NEXT:    psubd %xmm0, %xmm2
622; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459]
623; SSE3-NEXT:    movdqa %xmm2, %xmm3
624; SSE3-NEXT:    pand %xmm0, %xmm3
625; SSE3-NEXT:    psrld $2, %xmm2
626; SSE3-NEXT:    pand %xmm0, %xmm2
627; SSE3-NEXT:    paddd %xmm3, %xmm2
628; SSE3-NEXT:    movdqa %xmm2, %xmm0
629; SSE3-NEXT:    psrld $4, %xmm0
630; SSE3-NEXT:    paddd %xmm2, %xmm0
631; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
632; SSE3-NEXT:    movdqa %xmm0, %xmm2
633; SSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
634; SSE3-NEXT:    psadbw %xmm1, %xmm2
635; SSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
636; SSE3-NEXT:    psadbw %xmm1, %xmm0
637; SSE3-NEXT:    packuswb %xmm2, %xmm0
638; SSE3-NEXT:    retq
639;
640; SSSE3-LABEL: testv4i32u:
641; SSSE3:       # BB#0:
642; SSSE3-NEXT:    pxor %xmm1, %xmm1
643; SSSE3-NEXT:    pxor %xmm2, %xmm2
644; SSSE3-NEXT:    psubd %xmm0, %xmm2
645; SSSE3-NEXT:    pand %xmm0, %xmm2
646; SSSE3-NEXT:    psubd {{.*}}(%rip), %xmm2
647; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
648; SSSE3-NEXT:    movdqa %xmm2, %xmm4
649; SSSE3-NEXT:    pand %xmm3, %xmm4
650; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
651; SSSE3-NEXT:    movdqa %xmm0, %xmm5
652; SSSE3-NEXT:    pshufb %xmm4, %xmm5
653; SSSE3-NEXT:    psrlw $4, %xmm2
654; SSSE3-NEXT:    pand %xmm3, %xmm2
655; SSSE3-NEXT:    pshufb %xmm2, %xmm0
656; SSSE3-NEXT:    paddb %xmm5, %xmm0
657; SSSE3-NEXT:    movdqa %xmm0, %xmm2
658; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
659; SSSE3-NEXT:    psadbw %xmm1, %xmm2
660; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
661; SSSE3-NEXT:    psadbw %xmm1, %xmm0
662; SSSE3-NEXT:    packuswb %xmm2, %xmm0
663; SSSE3-NEXT:    retq
664;
665; SSE41-LABEL: testv4i32u:
666; SSE41:       # BB#0:
667; SSE41-NEXT:    pxor %xmm1, %xmm1
668; SSE41-NEXT:    pxor %xmm2, %xmm2
669; SSE41-NEXT:    psubd %xmm0, %xmm2
670; SSE41-NEXT:    pand %xmm0, %xmm2
671; SSE41-NEXT:    psubd {{.*}}(%rip), %xmm2
672; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
673; SSE41-NEXT:    movdqa %xmm2, %xmm3
674; SSE41-NEXT:    pand %xmm0, %xmm3
675; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
676; SSE41-NEXT:    movdqa %xmm4, %xmm5
677; SSE41-NEXT:    pshufb %xmm3, %xmm5
678; SSE41-NEXT:    psrlw $4, %xmm2
679; SSE41-NEXT:    pand %xmm0, %xmm2
680; SSE41-NEXT:    pshufb %xmm2, %xmm4
681; SSE41-NEXT:    paddb %xmm5, %xmm4
682; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
683; SSE41-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
684; SSE41-NEXT:    psadbw %xmm1, %xmm4
685; SSE41-NEXT:    psadbw %xmm1, %xmm0
686; SSE41-NEXT:    packuswb %xmm4, %xmm0
687; SSE41-NEXT:    retq
688;
689; AVX1-LABEL: testv4i32u:
690; AVX1:       # BB#0:
691; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
692; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
693; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
694; AVX1-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
695; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
696; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm3
697; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
698; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
699; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
700; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
701; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
702; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
703; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
704; AVX1-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
705; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
706; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
707; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
708; AVX1-NEXT:    retq
709;
710; AVX2-LABEL: testv4i32u:
711; AVX2:       # BB#0:
712; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
713; AVX2-NEXT:    vpsubd %xmm0, %xmm1, %xmm2
714; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
715; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
716; AVX2-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
717; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
718; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm3
719; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
720; AVX2-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
721; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
722; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
723; AVX2-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
724; AVX2-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
725; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
726; AVX2-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
727; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
728; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
729; AVX2-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
730; AVX2-NEXT:    retq
731;
732; AVX512CDVL-LABEL: testv4i32u:
733; AVX512CDVL:       # BB#0:
734; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
735; AVX512CDVL-NEXT:    vpsubd %xmm0, %xmm1, %xmm1
736; AVX512CDVL-NEXT:    vpandd %xmm1, %xmm0, %xmm0
737; AVX512CDVL-NEXT:    vplzcntd %xmm0, %xmm0
738; AVX512CDVL-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
739; AVX512CDVL-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
740; AVX512CDVL-NEXT:    retq
741;
742; AVX512CD-LABEL: testv4i32u:
743; AVX512CD:       # BB#0:
744; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
745; AVX512CD-NEXT:    vpsubd %xmm0, %xmm1, %xmm1
746; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
747; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
748; AVX512CD-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
749; AVX512CD-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
750; AVX512CD-NEXT:    retq
751;
752; X32-SSE-LABEL: testv4i32u:
753; X32-SSE:       # BB#0:
754; X32-SSE-NEXT:    pxor %xmm1, %xmm1
755; X32-SSE-NEXT:    pxor %xmm2, %xmm2
756; X32-SSE-NEXT:    psubd %xmm0, %xmm2
757; X32-SSE-NEXT:    pand %xmm0, %xmm2
758; X32-SSE-NEXT:    psubd {{\.LCPI.*}}, %xmm2
759; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
760; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
761; X32-SSE-NEXT:    pand %xmm0, %xmm3
762; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
763; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
764; X32-SSE-NEXT:    pshufb %xmm3, %xmm5
765; X32-SSE-NEXT:    psrlw $4, %xmm2
766; X32-SSE-NEXT:    pand %xmm0, %xmm2
767; X32-SSE-NEXT:    pshufb %xmm2, %xmm4
768; X32-SSE-NEXT:    paddb %xmm5, %xmm4
769; X32-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
770; X32-SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
771; X32-SSE-NEXT:    psadbw %xmm1, %xmm4
772; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
773; X32-SSE-NEXT:    packuswb %xmm4, %xmm0
774; X32-SSE-NEXT:    retl
775  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 -1)
776  ret <4 x i32> %out
777}
778
779define <8 x i16> @testv8i16(<8 x i16> %in) nounwind {
780; SSE2-LABEL: testv8i16:
781; SSE2:       # BB#0:
782; SSE2-NEXT:    pxor %xmm1, %xmm1
783; SSE2-NEXT:    psubw %xmm0, %xmm1
784; SSE2-NEXT:    pand %xmm0, %xmm1
785; SSE2-NEXT:    psubw {{.*}}(%rip), %xmm1
786; SSE2-NEXT:    movdqa %xmm1, %xmm0
787; SSE2-NEXT:    psrlw $1, %xmm0
788; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
789; SSE2-NEXT:    psubw %xmm0, %xmm1
790; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107]
791; SSE2-NEXT:    movdqa %xmm1, %xmm2
792; SSE2-NEXT:    pand %xmm0, %xmm2
793; SSE2-NEXT:    psrlw $2, %xmm1
794; SSE2-NEXT:    pand %xmm0, %xmm1
795; SSE2-NEXT:    paddw %xmm2, %xmm1
796; SSE2-NEXT:    movdqa %xmm1, %xmm2
797; SSE2-NEXT:    psrlw $4, %xmm2
798; SSE2-NEXT:    paddw %xmm1, %xmm2
799; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
800; SSE2-NEXT:    movdqa %xmm2, %xmm0
801; SSE2-NEXT:    psllw $8, %xmm0
802; SSE2-NEXT:    paddb %xmm2, %xmm0
803; SSE2-NEXT:    psrlw $8, %xmm0
804; SSE2-NEXT:    retq
805;
806; SSE3-LABEL: testv8i16:
807; SSE3:       # BB#0:
808; SSE3-NEXT:    pxor %xmm1, %xmm1
809; SSE3-NEXT:    psubw %xmm0, %xmm1
810; SSE3-NEXT:    pand %xmm0, %xmm1
811; SSE3-NEXT:    psubw {{.*}}(%rip), %xmm1
812; SSE3-NEXT:    movdqa %xmm1, %xmm0
813; SSE3-NEXT:    psrlw $1, %xmm0
814; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
815; SSE3-NEXT:    psubw %xmm0, %xmm1
816; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107]
817; SSE3-NEXT:    movdqa %xmm1, %xmm2
818; SSE3-NEXT:    pand %xmm0, %xmm2
819; SSE3-NEXT:    psrlw $2, %xmm1
820; SSE3-NEXT:    pand %xmm0, %xmm1
821; SSE3-NEXT:    paddw %xmm2, %xmm1
822; SSE3-NEXT:    movdqa %xmm1, %xmm2
823; SSE3-NEXT:    psrlw $4, %xmm2
824; SSE3-NEXT:    paddw %xmm1, %xmm2
825; SSE3-NEXT:    pand {{.*}}(%rip), %xmm2
826; SSE3-NEXT:    movdqa %xmm2, %xmm0
827; SSE3-NEXT:    psllw $8, %xmm0
828; SSE3-NEXT:    paddb %xmm2, %xmm0
829; SSE3-NEXT:    psrlw $8, %xmm0
830; SSE3-NEXT:    retq
831;
832; SSSE3-LABEL: testv8i16:
833; SSSE3:       # BB#0:
834; SSSE3-NEXT:    pxor %xmm1, %xmm1
835; SSSE3-NEXT:    psubw %xmm0, %xmm1
836; SSSE3-NEXT:    pand %xmm0, %xmm1
837; SSSE3-NEXT:    psubw {{.*}}(%rip), %xmm1
838; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
839; SSSE3-NEXT:    movdqa %xmm1, %xmm2
840; SSSE3-NEXT:    pand %xmm0, %xmm2
841; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
842; SSSE3-NEXT:    movdqa %xmm3, %xmm4
843; SSSE3-NEXT:    pshufb %xmm2, %xmm4
844; SSSE3-NEXT:    psrlw $4, %xmm1
845; SSSE3-NEXT:    pand %xmm0, %xmm1
846; SSSE3-NEXT:    pshufb %xmm1, %xmm3
847; SSSE3-NEXT:    paddb %xmm4, %xmm3
848; SSSE3-NEXT:    movdqa %xmm3, %xmm0
849; SSSE3-NEXT:    psllw $8, %xmm0
850; SSSE3-NEXT:    paddb %xmm3, %xmm0
851; SSSE3-NEXT:    psrlw $8, %xmm0
852; SSSE3-NEXT:    retq
853;
854; SSE41-LABEL: testv8i16:
855; SSE41:       # BB#0:
856; SSE41-NEXT:    pxor %xmm1, %xmm1
857; SSE41-NEXT:    psubw %xmm0, %xmm1
858; SSE41-NEXT:    pand %xmm0, %xmm1
859; SSE41-NEXT:    psubw {{.*}}(%rip), %xmm1
860; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
861; SSE41-NEXT:    movdqa %xmm1, %xmm2
862; SSE41-NEXT:    pand %xmm0, %xmm2
863; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
864; SSE41-NEXT:    movdqa %xmm3, %xmm4
865; SSE41-NEXT:    pshufb %xmm2, %xmm4
866; SSE41-NEXT:    psrlw $4, %xmm1
867; SSE41-NEXT:    pand %xmm0, %xmm1
868; SSE41-NEXT:    pshufb %xmm1, %xmm3
869; SSE41-NEXT:    paddb %xmm4, %xmm3
870; SSE41-NEXT:    movdqa %xmm3, %xmm0
871; SSE41-NEXT:    psllw $8, %xmm0
872; SSE41-NEXT:    paddb %xmm3, %xmm0
873; SSE41-NEXT:    psrlw $8, %xmm0
874; SSE41-NEXT:    retq
875;
876; AVX1-LABEL: testv8i16:
877; AVX1:       # BB#0:
878; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
879; AVX1-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
880; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
881; AVX1-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
882; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
883; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
884; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
885; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
886; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
887; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
888; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
889; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
890; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
891; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
892; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
893; AVX1-NEXT:    retq
894;
895; AVX2-LABEL: testv8i16:
896; AVX2:       # BB#0:
897; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
898; AVX2-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
899; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
900; AVX2-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
901; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
902; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
903; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
904; AVX2-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
905; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
906; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
907; AVX2-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
908; AVX2-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
909; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm1
910; AVX2-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
911; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm0
912; AVX2-NEXT:    retq
913;
914; AVX512CDVL-LABEL: testv8i16:
915; AVX512CDVL:       # BB#0:
916; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
917; AVX512CDVL-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
918; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
919; AVX512CDVL-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
920; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
921; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm2
922; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
923; AVX512CDVL-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
924; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
925; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
926; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
927; AVX512CDVL-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
928; AVX512CDVL-NEXT:    vpsllw $8, %xmm0, %xmm1
929; AVX512CDVL-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
930; AVX512CDVL-NEXT:    vpsrlw $8, %xmm0, %xmm0
931; AVX512CDVL-NEXT:    retq
932;
933; AVX512CD-LABEL: testv8i16:
934; AVX512CD:       # BB#0:
935; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
936; AVX512CD-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
937; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
938; AVX512CD-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
939; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
940; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm2
941; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
942; AVX512CD-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
943; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
944; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
945; AVX512CD-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
946; AVX512CD-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
947; AVX512CD-NEXT:    vpsllw $8, %xmm0, %xmm1
948; AVX512CD-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
949; AVX512CD-NEXT:    vpsrlw $8, %xmm0, %xmm0
950; AVX512CD-NEXT:    retq
951;
952; X32-SSE-LABEL: testv8i16:
953; X32-SSE:       # BB#0:
954; X32-SSE-NEXT:    pxor %xmm1, %xmm1
955; X32-SSE-NEXT:    psubw %xmm0, %xmm1
956; X32-SSE-NEXT:    pand %xmm0, %xmm1
957; X32-SSE-NEXT:    psubw {{\.LCPI.*}}, %xmm1
958; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
959; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
960; X32-SSE-NEXT:    pand %xmm0, %xmm2
961; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
962; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
963; X32-SSE-NEXT:    pshufb %xmm2, %xmm4
964; X32-SSE-NEXT:    psrlw $4, %xmm1
965; X32-SSE-NEXT:    pand %xmm0, %xmm1
966; X32-SSE-NEXT:    pshufb %xmm1, %xmm3
967; X32-SSE-NEXT:    paddb %xmm4, %xmm3
968; X32-SSE-NEXT:    movdqa %xmm3, %xmm0
969; X32-SSE-NEXT:    psllw $8, %xmm0
970; X32-SSE-NEXT:    paddb %xmm3, %xmm0
971; X32-SSE-NEXT:    psrlw $8, %xmm0
972; X32-SSE-NEXT:    retl
973  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 0)
974  ret <8 x i16> %out
975}
976
977define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind {
978; SSE2-LABEL: testv8i16u:
979; SSE2:       # BB#0:
980; SSE2-NEXT:    pxor %xmm1, %xmm1
981; SSE2-NEXT:    psubw %xmm0, %xmm1
982; SSE2-NEXT:    pand %xmm0, %xmm1
983; SSE2-NEXT:    psubw {{.*}}(%rip), %xmm1
984; SSE2-NEXT:    movdqa %xmm1, %xmm0
985; SSE2-NEXT:    psrlw $1, %xmm0
986; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
987; SSE2-NEXT:    psubw %xmm0, %xmm1
988; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107]
989; SSE2-NEXT:    movdqa %xmm1, %xmm2
990; SSE2-NEXT:    pand %xmm0, %xmm2
991; SSE2-NEXT:    psrlw $2, %xmm1
992; SSE2-NEXT:    pand %xmm0, %xmm1
993; SSE2-NEXT:    paddw %xmm2, %xmm1
994; SSE2-NEXT:    movdqa %xmm1, %xmm2
995; SSE2-NEXT:    psrlw $4, %xmm2
996; SSE2-NEXT:    paddw %xmm1, %xmm2
997; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
998; SSE2-NEXT:    movdqa %xmm2, %xmm0
999; SSE2-NEXT:    psllw $8, %xmm0
1000; SSE2-NEXT:    paddb %xmm2, %xmm0
1001; SSE2-NEXT:    psrlw $8, %xmm0
1002; SSE2-NEXT:    retq
1003;
1004; SSE3-LABEL: testv8i16u:
1005; SSE3:       # BB#0:
1006; SSE3-NEXT:    pxor %xmm1, %xmm1
1007; SSE3-NEXT:    psubw %xmm0, %xmm1
1008; SSE3-NEXT:    pand %xmm0, %xmm1
1009; SSE3-NEXT:    psubw {{.*}}(%rip), %xmm1
1010; SSE3-NEXT:    movdqa %xmm1, %xmm0
1011; SSE3-NEXT:    psrlw $1, %xmm0
1012; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
1013; SSE3-NEXT:    psubw %xmm0, %xmm1
1014; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107]
1015; SSE3-NEXT:    movdqa %xmm1, %xmm2
1016; SSE3-NEXT:    pand %xmm0, %xmm2
1017; SSE3-NEXT:    psrlw $2, %xmm1
1018; SSE3-NEXT:    pand %xmm0, %xmm1
1019; SSE3-NEXT:    paddw %xmm2, %xmm1
1020; SSE3-NEXT:    movdqa %xmm1, %xmm2
1021; SSE3-NEXT:    psrlw $4, %xmm2
1022; SSE3-NEXT:    paddw %xmm1, %xmm2
1023; SSE3-NEXT:    pand {{.*}}(%rip), %xmm2
1024; SSE3-NEXT:    movdqa %xmm2, %xmm0
1025; SSE3-NEXT:    psllw $8, %xmm0
1026; SSE3-NEXT:    paddb %xmm2, %xmm0
1027; SSE3-NEXT:    psrlw $8, %xmm0
1028; SSE3-NEXT:    retq
1029;
1030; SSSE3-LABEL: testv8i16u:
1031; SSSE3:       # BB#0:
1032; SSSE3-NEXT:    pxor %xmm1, %xmm1
1033; SSSE3-NEXT:    psubw %xmm0, %xmm1
1034; SSSE3-NEXT:    pand %xmm0, %xmm1
1035; SSSE3-NEXT:    psubw {{.*}}(%rip), %xmm1
1036; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1037; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1038; SSSE3-NEXT:    pand %xmm0, %xmm2
1039; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1040; SSSE3-NEXT:    movdqa %xmm3, %xmm4
1041; SSSE3-NEXT:    pshufb %xmm2, %xmm4
1042; SSSE3-NEXT:    psrlw $4, %xmm1
1043; SSSE3-NEXT:    pand %xmm0, %xmm1
1044; SSSE3-NEXT:    pshufb %xmm1, %xmm3
1045; SSSE3-NEXT:    paddb %xmm4, %xmm3
1046; SSSE3-NEXT:    movdqa %xmm3, %xmm0
1047; SSSE3-NEXT:    psllw $8, %xmm0
1048; SSSE3-NEXT:    paddb %xmm3, %xmm0
1049; SSSE3-NEXT:    psrlw $8, %xmm0
1050; SSSE3-NEXT:    retq
1051;
1052; SSE41-LABEL: testv8i16u:
1053; SSE41:       # BB#0:
1054; SSE41-NEXT:    pxor %xmm1, %xmm1
1055; SSE41-NEXT:    psubw %xmm0, %xmm1
1056; SSE41-NEXT:    pand %xmm0, %xmm1
1057; SSE41-NEXT:    psubw {{.*}}(%rip), %xmm1
1058; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1059; SSE41-NEXT:    movdqa %xmm1, %xmm2
1060; SSE41-NEXT:    pand %xmm0, %xmm2
1061; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1062; SSE41-NEXT:    movdqa %xmm3, %xmm4
1063; SSE41-NEXT:    pshufb %xmm2, %xmm4
1064; SSE41-NEXT:    psrlw $4, %xmm1
1065; SSE41-NEXT:    pand %xmm0, %xmm1
1066; SSE41-NEXT:    pshufb %xmm1, %xmm3
1067; SSE41-NEXT:    paddb %xmm4, %xmm3
1068; SSE41-NEXT:    movdqa %xmm3, %xmm0
1069; SSE41-NEXT:    psllw $8, %xmm0
1070; SSE41-NEXT:    paddb %xmm3, %xmm0
1071; SSE41-NEXT:    psrlw $8, %xmm0
1072; SSE41-NEXT:    retq
1073;
1074; AVX1-LABEL: testv8i16u:
1075; AVX1:       # BB#0:
1076; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1077; AVX1-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
1078; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1079; AVX1-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
1080; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1081; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
1082; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1083; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1084; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
1085; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1086; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1087; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1088; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
1089; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
1090; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1091; AVX1-NEXT:    retq
1092;
1093; AVX2-LABEL: testv8i16u:
1094; AVX2:       # BB#0:
1095; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1096; AVX2-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
1097; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1098; AVX2-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
1099; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1100; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
1101; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1102; AVX2-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1103; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
1104; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1105; AVX2-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1106; AVX2-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1107; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm1
1108; AVX2-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
1109; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm0
1110; AVX2-NEXT:    retq
1111;
1112; AVX512CDVL-LABEL: testv8i16u:
1113; AVX512CDVL:       # BB#0:
1114; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
1115; AVX512CDVL-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
1116; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1117; AVX512CDVL-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
1118; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1119; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm2
1120; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1121; AVX512CDVL-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1122; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
1123; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1124; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1125; AVX512CDVL-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1126; AVX512CDVL-NEXT:    vpsllw $8, %xmm0, %xmm1
1127; AVX512CDVL-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
1128; AVX512CDVL-NEXT:    vpsrlw $8, %xmm0, %xmm0
1129; AVX512CDVL-NEXT:    retq
1130;
1131; AVX512CD-LABEL: testv8i16u:
1132; AVX512CD:       # BB#0:
1133; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1134; AVX512CD-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
1135; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1136; AVX512CD-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
1137; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1138; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm2
1139; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1140; AVX512CD-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1141; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
1142; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1143; AVX512CD-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1144; AVX512CD-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1145; AVX512CD-NEXT:    vpsllw $8, %xmm0, %xmm1
1146; AVX512CD-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
1147; AVX512CD-NEXT:    vpsrlw $8, %xmm0, %xmm0
1148; AVX512CD-NEXT:    retq
1149;
1150; X32-SSE-LABEL: testv8i16u:
1151; X32-SSE:       # BB#0:
1152; X32-SSE-NEXT:    pxor %xmm1, %xmm1
1153; X32-SSE-NEXT:    psubw %xmm0, %xmm1
1154; X32-SSE-NEXT:    pand %xmm0, %xmm1
1155; X32-SSE-NEXT:    psubw {{\.LCPI.*}}, %xmm1
1156; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1157; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
1158; X32-SSE-NEXT:    pand %xmm0, %xmm2
1159; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1160; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
1161; X32-SSE-NEXT:    pshufb %xmm2, %xmm4
1162; X32-SSE-NEXT:    psrlw $4, %xmm1
1163; X32-SSE-NEXT:    pand %xmm0, %xmm1
1164; X32-SSE-NEXT:    pshufb %xmm1, %xmm3
1165; X32-SSE-NEXT:    paddb %xmm4, %xmm3
1166; X32-SSE-NEXT:    movdqa %xmm3, %xmm0
1167; X32-SSE-NEXT:    psllw $8, %xmm0
1168; X32-SSE-NEXT:    paddb %xmm3, %xmm0
1169; X32-SSE-NEXT:    psrlw $8, %xmm0
1170; X32-SSE-NEXT:    retl
1171  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 -1)
1172  ret <8 x i16> %out
1173}
1174
1175define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
1176; SSE2-LABEL: testv16i8:
1177; SSE2:       # BB#0:
1178; SSE2-NEXT:    pxor %xmm1, %xmm1
1179; SSE2-NEXT:    psubb %xmm0, %xmm1
1180; SSE2-NEXT:    pand %xmm0, %xmm1
1181; SSE2-NEXT:    psubb {{.*}}(%rip), %xmm1
1182; SSE2-NEXT:    movdqa %xmm1, %xmm0
1183; SSE2-NEXT:    psrlw $1, %xmm0
1184; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1185; SSE2-NEXT:    psubb %xmm0, %xmm1
1186; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
1187; SSE2-NEXT:    movdqa %xmm1, %xmm2
1188; SSE2-NEXT:    pand %xmm0, %xmm2
1189; SSE2-NEXT:    psrlw $2, %xmm1
1190; SSE2-NEXT:    pand %xmm0, %xmm1
1191; SSE2-NEXT:    paddb %xmm2, %xmm1
1192; SSE2-NEXT:    movdqa %xmm1, %xmm0
1193; SSE2-NEXT:    psrlw $4, %xmm0
1194; SSE2-NEXT:    paddb %xmm1, %xmm0
1195; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1196; SSE2-NEXT:    retq
1197;
1198; SSE3-LABEL: testv16i8:
1199; SSE3:       # BB#0:
1200; SSE3-NEXT:    pxor %xmm1, %xmm1
1201; SSE3-NEXT:    psubb %xmm0, %xmm1
1202; SSE3-NEXT:    pand %xmm0, %xmm1
1203; SSE3-NEXT:    psubb {{.*}}(%rip), %xmm1
1204; SSE3-NEXT:    movdqa %xmm1, %xmm0
1205; SSE3-NEXT:    psrlw $1, %xmm0
1206; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
1207; SSE3-NEXT:    psubb %xmm0, %xmm1
1208; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
1209; SSE3-NEXT:    movdqa %xmm1, %xmm2
1210; SSE3-NEXT:    pand %xmm0, %xmm2
1211; SSE3-NEXT:    psrlw $2, %xmm1
1212; SSE3-NEXT:    pand %xmm0, %xmm1
1213; SSE3-NEXT:    paddb %xmm2, %xmm1
1214; SSE3-NEXT:    movdqa %xmm1, %xmm0
1215; SSE3-NEXT:    psrlw $4, %xmm0
1216; SSE3-NEXT:    paddb %xmm1, %xmm0
1217; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
1218; SSE3-NEXT:    retq
1219;
1220; SSSE3-LABEL: testv16i8:
1221; SSSE3:       # BB#0:
1222; SSSE3-NEXT:    pxor %xmm1, %xmm1
1223; SSSE3-NEXT:    psubb %xmm0, %xmm1
1224; SSSE3-NEXT:    pand %xmm0, %xmm1
1225; SSSE3-NEXT:    psubb {{.*}}(%rip), %xmm1
1226; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1227; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1228; SSSE3-NEXT:    pand %xmm2, %xmm3
1229; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1230; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1231; SSSE3-NEXT:    pshufb %xmm3, %xmm4
1232; SSSE3-NEXT:    psrlw $4, %xmm1
1233; SSSE3-NEXT:    pand %xmm2, %xmm1
1234; SSSE3-NEXT:    pshufb %xmm1, %xmm0
1235; SSSE3-NEXT:    paddb %xmm4, %xmm0
1236; SSSE3-NEXT:    retq
1237;
1238; SSE41-LABEL: testv16i8:
1239; SSE41:       # BB#0:
1240; SSE41-NEXT:    pxor %xmm1, %xmm1
1241; SSE41-NEXT:    psubb %xmm0, %xmm1
1242; SSE41-NEXT:    pand %xmm0, %xmm1
1243; SSE41-NEXT:    psubb {{.*}}(%rip), %xmm1
1244; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1245; SSE41-NEXT:    movdqa %xmm1, %xmm3
1246; SSE41-NEXT:    pand %xmm2, %xmm3
1247; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1248; SSE41-NEXT:    movdqa %xmm0, %xmm4
1249; SSE41-NEXT:    pshufb %xmm3, %xmm4
1250; SSE41-NEXT:    psrlw $4, %xmm1
1251; SSE41-NEXT:    pand %xmm2, %xmm1
1252; SSE41-NEXT:    pshufb %xmm1, %xmm0
1253; SSE41-NEXT:    paddb %xmm4, %xmm0
1254; SSE41-NEXT:    retq
1255;
1256; AVX1-LABEL: testv16i8:
1257; AVX1:       # BB#0:
1258; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1259; AVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1260; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1261; AVX1-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1262; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1263; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
1264; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1265; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1266; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
1267; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1268; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1269; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1270; AVX1-NEXT:    retq
1271;
1272; AVX2-LABEL: testv16i8:
1273; AVX2:       # BB#0:
1274; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1275; AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1276; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1277; AVX2-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1278; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1279; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
1280; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1281; AVX2-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1282; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
1283; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1284; AVX2-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1285; AVX2-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1286; AVX2-NEXT:    retq
1287;
1288; AVX512CDVL-LABEL: testv16i8:
1289; AVX512CDVL:       # BB#0:
1290; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
1291; AVX512CDVL-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1292; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1293; AVX512CDVL-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1294; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1295; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm2
1296; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1297; AVX512CDVL-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1298; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
1299; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1300; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1301; AVX512CDVL-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1302; AVX512CDVL-NEXT:    retq
1303;
1304; AVX512CD-LABEL: testv16i8:
1305; AVX512CD:       # BB#0:
1306; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1307; AVX512CD-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1308; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1309; AVX512CD-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1310; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1311; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm2
1312; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1313; AVX512CD-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1314; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
1315; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1316; AVX512CD-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1317; AVX512CD-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1318; AVX512CD-NEXT:    retq
1319;
1320; X32-SSE-LABEL: testv16i8:
1321; X32-SSE:       # BB#0:
1322; X32-SSE-NEXT:    pxor %xmm1, %xmm1
1323; X32-SSE-NEXT:    psubb %xmm0, %xmm1
1324; X32-SSE-NEXT:    pand %xmm0, %xmm1
1325; X32-SSE-NEXT:    psubb {{\.LCPI.*}}, %xmm1
1326; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1327; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
1328; X32-SSE-NEXT:    pand %xmm2, %xmm3
1329; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1330; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
1331; X32-SSE-NEXT:    pshufb %xmm3, %xmm4
1332; X32-SSE-NEXT:    psrlw $4, %xmm1
1333; X32-SSE-NEXT:    pand %xmm2, %xmm1
1334; X32-SSE-NEXT:    pshufb %xmm1, %xmm0
1335; X32-SSE-NEXT:    paddb %xmm4, %xmm0
1336; X32-SSE-NEXT:    retl
1337  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0)
1338  ret <16 x i8> %out
1339}
1340
1341define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
1342; SSE2-LABEL: testv16i8u:
1343; SSE2:       # BB#0:
1344; SSE2-NEXT:    pxor %xmm1, %xmm1
1345; SSE2-NEXT:    psubb %xmm0, %xmm1
1346; SSE2-NEXT:    pand %xmm0, %xmm1
1347; SSE2-NEXT:    psubb {{.*}}(%rip), %xmm1
1348; SSE2-NEXT:    movdqa %xmm1, %xmm0
1349; SSE2-NEXT:    psrlw $1, %xmm0
1350; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1351; SSE2-NEXT:    psubb %xmm0, %xmm1
1352; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
1353; SSE2-NEXT:    movdqa %xmm1, %xmm2
1354; SSE2-NEXT:    pand %xmm0, %xmm2
1355; SSE2-NEXT:    psrlw $2, %xmm1
1356; SSE2-NEXT:    pand %xmm0, %xmm1
1357; SSE2-NEXT:    paddb %xmm2, %xmm1
1358; SSE2-NEXT:    movdqa %xmm1, %xmm0
1359; SSE2-NEXT:    psrlw $4, %xmm0
1360; SSE2-NEXT:    paddb %xmm1, %xmm0
1361; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1362; SSE2-NEXT:    retq
1363;
1364; SSE3-LABEL: testv16i8u:
1365; SSE3:       # BB#0:
1366; SSE3-NEXT:    pxor %xmm1, %xmm1
1367; SSE3-NEXT:    psubb %xmm0, %xmm1
1368; SSE3-NEXT:    pand %xmm0, %xmm1
1369; SSE3-NEXT:    psubb {{.*}}(%rip), %xmm1
1370; SSE3-NEXT:    movdqa %xmm1, %xmm0
1371; SSE3-NEXT:    psrlw $1, %xmm0
1372; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
1373; SSE3-NEXT:    psubb %xmm0, %xmm1
1374; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
1375; SSE3-NEXT:    movdqa %xmm1, %xmm2
1376; SSE3-NEXT:    pand %xmm0, %xmm2
1377; SSE3-NEXT:    psrlw $2, %xmm1
1378; SSE3-NEXT:    pand %xmm0, %xmm1
1379; SSE3-NEXT:    paddb %xmm2, %xmm1
1380; SSE3-NEXT:    movdqa %xmm1, %xmm0
1381; SSE3-NEXT:    psrlw $4, %xmm0
1382; SSE3-NEXT:    paddb %xmm1, %xmm0
1383; SSE3-NEXT:    pand {{.*}}(%rip), %xmm0
1384; SSE3-NEXT:    retq
1385;
1386; SSSE3-LABEL: testv16i8u:
1387; SSSE3:       # BB#0:
1388; SSSE3-NEXT:    pxor %xmm1, %xmm1
1389; SSSE3-NEXT:    psubb %xmm0, %xmm1
1390; SSSE3-NEXT:    pand %xmm0, %xmm1
1391; SSSE3-NEXT:    psubb {{.*}}(%rip), %xmm1
1392; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1393; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1394; SSSE3-NEXT:    pand %xmm2, %xmm3
1395; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1396; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1397; SSSE3-NEXT:    pshufb %xmm3, %xmm4
1398; SSSE3-NEXT:    psrlw $4, %xmm1
1399; SSSE3-NEXT:    pand %xmm2, %xmm1
1400; SSSE3-NEXT:    pshufb %xmm1, %xmm0
1401; SSSE3-NEXT:    paddb %xmm4, %xmm0
1402; SSSE3-NEXT:    retq
1403;
1404; SSE41-LABEL: testv16i8u:
1405; SSE41:       # BB#0:
1406; SSE41-NEXT:    pxor %xmm1, %xmm1
1407; SSE41-NEXT:    psubb %xmm0, %xmm1
1408; SSE41-NEXT:    pand %xmm0, %xmm1
1409; SSE41-NEXT:    psubb {{.*}}(%rip), %xmm1
1410; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1411; SSE41-NEXT:    movdqa %xmm1, %xmm3
1412; SSE41-NEXT:    pand %xmm2, %xmm3
1413; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1414; SSE41-NEXT:    movdqa %xmm0, %xmm4
1415; SSE41-NEXT:    pshufb %xmm3, %xmm4
1416; SSE41-NEXT:    psrlw $4, %xmm1
1417; SSE41-NEXT:    pand %xmm2, %xmm1
1418; SSE41-NEXT:    pshufb %xmm1, %xmm0
1419; SSE41-NEXT:    paddb %xmm4, %xmm0
1420; SSE41-NEXT:    retq
1421;
1422; AVX1-LABEL: testv16i8u:
1423; AVX1:       # BB#0:
1424; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1425; AVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1426; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1427; AVX1-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1428; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1429; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
1430; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1431; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1432; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
1433; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
1434; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1435; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1436; AVX1-NEXT:    retq
1437;
1438; AVX2-LABEL: testv16i8u:
1439; AVX2:       # BB#0:
1440; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1441; AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1442; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1443; AVX2-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1444; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1445; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
1446; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1447; AVX2-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1448; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
1449; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
1450; AVX2-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1451; AVX2-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1452; AVX2-NEXT:    retq
1453;
1454; AVX512CDVL-LABEL: testv16i8u:
1455; AVX512CDVL:       # BB#0:
1456; AVX512CDVL-NEXT:    vpxord %xmm1, %xmm1, %xmm1
1457; AVX512CDVL-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1458; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1459; AVX512CDVL-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1460; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1461; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm2
1462; AVX512CDVL-NEXT:    vmovdqa64 {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1463; AVX512CDVL-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1464; AVX512CDVL-NEXT:    vpsrlw $4, %xmm0, %xmm0
1465; AVX512CDVL-NEXT:    vpandq %xmm1, %xmm0, %xmm0
1466; AVX512CDVL-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1467; AVX512CDVL-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1468; AVX512CDVL-NEXT:    retq
1469;
1470; AVX512CD-LABEL: testv16i8u:
1471; AVX512CD:       # BB#0:
1472; AVX512CD-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1473; AVX512CD-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
1474; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1475; AVX512CD-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
1476; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1477; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm2
1478; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1479; AVX512CD-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
1480; AVX512CD-NEXT:    vpsrlw $4, %xmm0, %xmm0
1481; AVX512CD-NEXT:    vpand %xmm1, %xmm0, %xmm0
1482; AVX512CD-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
1483; AVX512CD-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
1484; AVX512CD-NEXT:    retq
1485;
1486; X32-SSE-LABEL: testv16i8u:
1487; X32-SSE:       # BB#0:
1488; X32-SSE-NEXT:    pxor %xmm1, %xmm1
1489; X32-SSE-NEXT:    psubb %xmm0, %xmm1
1490; X32-SSE-NEXT:    pand %xmm0, %xmm1
1491; X32-SSE-NEXT:    psubb {{\.LCPI.*}}, %xmm1
1492; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1493; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
1494; X32-SSE-NEXT:    pand %xmm2, %xmm3
1495; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1496; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
1497; X32-SSE-NEXT:    pshufb %xmm3, %xmm4
1498; X32-SSE-NEXT:    psrlw $4, %xmm1
1499; X32-SSE-NEXT:    pand %xmm2, %xmm1
1500; X32-SSE-NEXT:    pshufb %xmm1, %xmm0
1501; X32-SSE-NEXT:    paddb %xmm4, %xmm0
1502; X32-SSE-NEXT:    retl
1503  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1)
1504  ret <16 x i8> %out
1505}
1506
1507define <2 x i64> @foldv2i64() nounwind {
1508; SSE-LABEL: foldv2i64:
1509; SSE:       # BB#0:
1510; SSE-NEXT:    movl $8, %eax
1511; SSE-NEXT:    movd %rax, %xmm0
1512; SSE-NEXT:    retq
1513;
1514; AVX-LABEL: foldv2i64:
1515; AVX:       # BB#0:
1516; AVX-NEXT:    movl $8, %eax
1517; AVX-NEXT:    vmovq %rax, %xmm0
1518; AVX-NEXT:    retq
1519;
1520; X32-SSE-LABEL: foldv2i64:
1521; X32-SSE:       # BB#0:
1522; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295]
1523; X32-SSE-NEXT:    pxor %xmm1, %xmm1
1524; X32-SSE-NEXT:    pxor %xmm2, %xmm2
1525; X32-SSE-NEXT:    psubq %xmm0, %xmm2
1526; X32-SSE-NEXT:    pand %xmm0, %xmm2
1527; X32-SSE-NEXT:    psubq {{\.LCPI.*}}, %xmm2
1528; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1529; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
1530; X32-SSE-NEXT:    pand %xmm3, %xmm4
1531; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1532; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
1533; X32-SSE-NEXT:    pshufb %xmm4, %xmm5
1534; X32-SSE-NEXT:    psrlw $4, %xmm2
1535; X32-SSE-NEXT:    pand %xmm3, %xmm2
1536; X32-SSE-NEXT:    pshufb %xmm2, %xmm0
1537; X32-SSE-NEXT:    paddb %xmm5, %xmm0
1538; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
1539; X32-SSE-NEXT:    retl
1540  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
1541  ret <2 x i64> %out
1542}
1543
1544define <2 x i64> @foldv2i64u() nounwind {
1545; SSE-LABEL: foldv2i64u:
1546; SSE:       # BB#0:
1547; SSE-NEXT:    movl $8, %eax
1548; SSE-NEXT:    movd %rax, %xmm0
1549; SSE-NEXT:    retq
1550;
1551; AVX-LABEL: foldv2i64u:
1552; AVX:       # BB#0:
1553; AVX-NEXT:    movl $8, %eax
1554; AVX-NEXT:    vmovq %rax, %xmm0
1555; AVX-NEXT:    retq
1556;
1557; X32-SSE-LABEL: foldv2i64u:
1558; X32-SSE:       # BB#0:
1559; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295]
1560; X32-SSE-NEXT:    pxor %xmm1, %xmm1
1561; X32-SSE-NEXT:    pxor %xmm2, %xmm2
1562; X32-SSE-NEXT:    psubq %xmm0, %xmm2
1563; X32-SSE-NEXT:    pand %xmm0, %xmm2
1564; X32-SSE-NEXT:    psubq {{\.LCPI.*}}, %xmm2
1565; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
1566; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
1567; X32-SSE-NEXT:    pand %xmm3, %xmm4
1568; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
1569; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
1570; X32-SSE-NEXT:    pshufb %xmm4, %xmm5
1571; X32-SSE-NEXT:    psrlw $4, %xmm2
1572; X32-SSE-NEXT:    pand %xmm3, %xmm2
1573; X32-SSE-NEXT:    pshufb %xmm2, %xmm0
1574; X32-SSE-NEXT:    paddb %xmm5, %xmm0
1575; X32-SSE-NEXT:    psadbw %xmm1, %xmm0
1576; X32-SSE-NEXT:    retl
1577  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
1578  ret <2 x i64> %out
1579}
1580
1581define <4 x i32> @foldv4i32() nounwind {
1582; SSE-LABEL: foldv4i32:
1583; SSE:       # BB#0:
1584; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
1585; SSE-NEXT:    retq
1586;
1587; AVX-LABEL: foldv4i32:
1588; AVX:       # BB#0:
1589; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
1590; AVX-NEXT:    retq
1591;
1592; X32-SSE-LABEL: foldv4i32:
1593; X32-SSE:       # BB#0:
1594; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
1595; X32-SSE-NEXT:    retl
1596  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 0)
1597  ret <4 x i32> %out
1598}
1599
1600define <4 x i32> @foldv4i32u() nounwind {
1601; SSE-LABEL: foldv4i32u:
1602; SSE:       # BB#0:
1603; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
1604; SSE-NEXT:    retq
1605;
1606; AVX-LABEL: foldv4i32u:
1607; AVX:       # BB#0:
1608; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
1609; AVX-NEXT:    retq
1610;
1611; X32-SSE-LABEL: foldv4i32u:
1612; X32-SSE:       # BB#0:
1613; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
1614; X32-SSE-NEXT:    retl
1615  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 -1)
1616  ret <4 x i32> %out
1617}
1618
1619define <8 x i16> @foldv8i16() nounwind {
1620; SSE-LABEL: foldv8i16:
1621; SSE:       # BB#0:
1622; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1623; SSE-NEXT:    retq
1624;
1625; AVX-LABEL: foldv8i16:
1626; AVX:       # BB#0:
1627; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1628; AVX-NEXT:    retq
1629;
1630; X32-SSE-LABEL: foldv8i16:
1631; X32-SSE:       # BB#0:
1632; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1633; X32-SSE-NEXT:    retl
1634  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 0)
1635  ret <8 x i16> %out
1636}
1637
1638define <8 x i16> @foldv8i16u() nounwind {
1639; SSE-LABEL: foldv8i16u:
1640; SSE:       # BB#0:
1641; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1642; SSE-NEXT:    retq
1643;
1644; AVX-LABEL: foldv8i16u:
1645; AVX:       # BB#0:
1646; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1647; AVX-NEXT:    retq
1648;
1649; X32-SSE-LABEL: foldv8i16u:
1650; X32-SSE:       # BB#0:
1651; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
1652; X32-SSE-NEXT:    retl
1653  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 -1)
1654  ret <8 x i16> %out
1655}
1656
1657define <16 x i8> @foldv16i8() nounwind {
1658; SSE-LABEL: foldv16i8:
1659; SSE:       # BB#0:
1660; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1661; SSE-NEXT:    retq
1662;
1663; AVX-LABEL: foldv16i8:
1664; AVX:       # BB#0:
1665; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1666; AVX-NEXT:    retq
1667;
1668; X32-SSE-LABEL: foldv16i8:
1669; X32-SSE:       # BB#0:
1670; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1671; X32-SSE-NEXT:    retl
1672  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 0)
1673  ret <16 x i8> %out
1674}
1675
1676define <16 x i8> @foldv16i8u() nounwind {
1677; SSE-LABEL: foldv16i8u:
1678; SSE:       # BB#0:
1679; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1680; SSE-NEXT:    retq
1681;
1682; AVX-LABEL: foldv16i8u:
1683; AVX:       # BB#0:
1684; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1685; AVX-NEXT:    retq
1686;
1687; X32-SSE-LABEL: foldv16i8u:
1688; X32-SSE:       # BB#0:
1689; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
1690; X32-SSE-NEXT:    retl
1691  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 -1)
1692  ret <16 x i8> %out
1693}
1694
1695declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
1696declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
1697declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
1698declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
1699