1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
8
9define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
10; SSE2-LABEL: testv2i64:
11; SSE2:       # BB#0:
12; SSE2-NEXT:    movdqa %xmm0, %xmm1
13; SSE2-NEXT:    psrlq $1, %xmm1
14; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
15; SSE2-NEXT:    psubq %xmm1, %xmm0
16; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
17; SSE2-NEXT:    movdqa %xmm0, %xmm2
18; SSE2-NEXT:    pand %xmm1, %xmm2
19; SSE2-NEXT:    psrlq $2, %xmm0
20; SSE2-NEXT:    pand %xmm1, %xmm0
21; SSE2-NEXT:    paddq %xmm2, %xmm0
22; SSE2-NEXT:    movdqa %xmm0, %xmm1
23; SSE2-NEXT:    psrlq $4, %xmm1
24; SSE2-NEXT:    paddq %xmm0, %xmm1
25; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
26; SSE2-NEXT:    pxor %xmm0, %xmm0
27; SSE2-NEXT:    psadbw %xmm0, %xmm1
28; SSE2-NEXT:    movdqa %xmm1, %xmm0
29; SSE2-NEXT:    retq
30;
31; SSE3-LABEL: testv2i64:
32; SSE3:       # BB#0:
33; SSE3-NEXT:    movdqa %xmm0, %xmm1
34; SSE3-NEXT:    psrlq $1, %xmm1
35; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
36; SSE3-NEXT:    psubq %xmm1, %xmm0
37; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
38; SSE3-NEXT:    movdqa %xmm0, %xmm2
39; SSE3-NEXT:    pand %xmm1, %xmm2
40; SSE3-NEXT:    psrlq $2, %xmm0
41; SSE3-NEXT:    pand %xmm1, %xmm0
42; SSE3-NEXT:    paddq %xmm2, %xmm0
43; SSE3-NEXT:    movdqa %xmm0, %xmm1
44; SSE3-NEXT:    psrlq $4, %xmm1
45; SSE3-NEXT:    paddq %xmm0, %xmm1
46; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
47; SSE3-NEXT:    pxor %xmm0, %xmm0
48; SSE3-NEXT:    psadbw %xmm0, %xmm1
49; SSE3-NEXT:    movdqa %xmm1, %xmm0
50; SSE3-NEXT:    retq
51;
52; SSSE3-LABEL: testv2i64:
53; SSSE3:       # BB#0:
54; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
55; SSSE3-NEXT:    movdqa %xmm0, %xmm2
56; SSSE3-NEXT:    pand %xmm1, %xmm2
57; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
58; SSSE3-NEXT:    movdqa %xmm3, %xmm4
59; SSSE3-NEXT:    pshufb %xmm2, %xmm4
60; SSSE3-NEXT:    psrlw $4, %xmm0
61; SSSE3-NEXT:    pand %xmm1, %xmm0
62; SSSE3-NEXT:    pshufb %xmm0, %xmm3
63; SSSE3-NEXT:    paddb %xmm4, %xmm3
64; SSSE3-NEXT:    pxor %xmm0, %xmm0
65; SSSE3-NEXT:    psadbw %xmm3, %xmm0
66; SSSE3-NEXT:    retq
67;
68; SSE41-LABEL: testv2i64:
69; SSE41:       # BB#0:
70; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
71; SSE41-NEXT:    movdqa %xmm0, %xmm2
72; SSE41-NEXT:    pand %xmm1, %xmm2
73; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
74; SSE41-NEXT:    movdqa %xmm3, %xmm4
75; SSE41-NEXT:    pshufb %xmm2, %xmm4
76; SSE41-NEXT:    psrlw $4, %xmm0
77; SSE41-NEXT:    pand %xmm1, %xmm0
78; SSE41-NEXT:    pshufb %xmm0, %xmm3
79; SSE41-NEXT:    paddb %xmm4, %xmm3
80; SSE41-NEXT:    pxor %xmm0, %xmm0
81; SSE41-NEXT:    psadbw %xmm3, %xmm0
82; SSE41-NEXT:    retq
83;
84; AVX-LABEL: testv2i64:
85; AVX:       # BB#0:
86; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
87; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
88; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
89; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
90; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
91; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
92; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
93; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
94; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
95; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
96; AVX-NEXT:    retq
97  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %in)
98  ret <2 x i64> %out
99}
100
101define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
102; SSE2-LABEL: testv4i32:
103; SSE2:       # BB#0:
104; SSE2-NEXT:    movdqa %xmm0, %xmm1
105; SSE2-NEXT:    psrld $1, %xmm1
106; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
107; SSE2-NEXT:    psubd %xmm1, %xmm0
108; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
109; SSE2-NEXT:    movdqa %xmm0, %xmm2
110; SSE2-NEXT:    pand %xmm1, %xmm2
111; SSE2-NEXT:    psrld $2, %xmm0
112; SSE2-NEXT:    pand %xmm1, %xmm0
113; SSE2-NEXT:    paddd %xmm2, %xmm0
114; SSE2-NEXT:    movdqa %xmm0, %xmm1
115; SSE2-NEXT:    psrld $4, %xmm1
116; SSE2-NEXT:    paddd %xmm0, %xmm1
117; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
118; SSE2-NEXT:    pxor %xmm0, %xmm0
119; SSE2-NEXT:    movdqa %xmm1, %xmm2
120; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
121; SSE2-NEXT:    psadbw %xmm0, %xmm2
122; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
123; SSE2-NEXT:    psadbw %xmm0, %xmm1
124; SSE2-NEXT:    packuswb %xmm2, %xmm1
125; SSE2-NEXT:    movdqa %xmm1, %xmm0
126; SSE2-NEXT:    retq
127;
128; SSE3-LABEL: testv4i32:
129; SSE3:       # BB#0:
130; SSE3-NEXT:    movdqa %xmm0, %xmm1
131; SSE3-NEXT:    psrld $1, %xmm1
132; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
133; SSE3-NEXT:    psubd %xmm1, %xmm0
134; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
135; SSE3-NEXT:    movdqa %xmm0, %xmm2
136; SSE3-NEXT:    pand %xmm1, %xmm2
137; SSE3-NEXT:    psrld $2, %xmm0
138; SSE3-NEXT:    pand %xmm1, %xmm0
139; SSE3-NEXT:    paddd %xmm2, %xmm0
140; SSE3-NEXT:    movdqa %xmm0, %xmm1
141; SSE3-NEXT:    psrld $4, %xmm1
142; SSE3-NEXT:    paddd %xmm0, %xmm1
143; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
144; SSE3-NEXT:    pxor %xmm0, %xmm0
145; SSE3-NEXT:    movdqa %xmm1, %xmm2
146; SSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
147; SSE3-NEXT:    psadbw %xmm0, %xmm2
148; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
149; SSE3-NEXT:    psadbw %xmm0, %xmm1
150; SSE3-NEXT:    packuswb %xmm2, %xmm1
151; SSE3-NEXT:    movdqa %xmm1, %xmm0
152; SSE3-NEXT:    retq
153;
154; SSSE3-LABEL: testv4i32:
155; SSSE3:       # BB#0:
156; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
157; SSSE3-NEXT:    movdqa %xmm0, %xmm3
158; SSSE3-NEXT:    pand %xmm2, %xmm3
159; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
160; SSSE3-NEXT:    movdqa %xmm1, %xmm4
161; SSSE3-NEXT:    pshufb %xmm3, %xmm4
162; SSSE3-NEXT:    psrlw $4, %xmm0
163; SSSE3-NEXT:    pand %xmm2, %xmm0
164; SSSE3-NEXT:    pshufb %xmm0, %xmm1
165; SSSE3-NEXT:    paddb %xmm4, %xmm1
166; SSSE3-NEXT:    pxor %xmm0, %xmm0
167; SSSE3-NEXT:    movdqa %xmm1, %xmm2
168; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
169; SSSE3-NEXT:    psadbw %xmm0, %xmm2
170; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
171; SSSE3-NEXT:    psadbw %xmm0, %xmm1
172; SSSE3-NEXT:    packuswb %xmm2, %xmm1
173; SSSE3-NEXT:    movdqa %xmm1, %xmm0
174; SSSE3-NEXT:    retq
175;
176; SSE41-LABEL: testv4i32:
177; SSE41:       # BB#0:
178; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
179; SSE41-NEXT:    movdqa %xmm0, %xmm2
180; SSE41-NEXT:    pand %xmm1, %xmm2
181; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
182; SSE41-NEXT:    movdqa %xmm3, %xmm4
183; SSE41-NEXT:    pshufb %xmm2, %xmm4
184; SSE41-NEXT:    psrlw $4, %xmm0
185; SSE41-NEXT:    pand %xmm1, %xmm0
186; SSE41-NEXT:    pshufb %xmm0, %xmm3
187; SSE41-NEXT:    paddb %xmm4, %xmm3
188; SSE41-NEXT:    pxor %xmm1, %xmm1
189; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero
190; SSE41-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
191; SSE41-NEXT:    psadbw %xmm1, %xmm3
192; SSE41-NEXT:    psadbw %xmm1, %xmm0
193; SSE41-NEXT:    packuswb %xmm3, %xmm0
194; SSE41-NEXT:    retq
195;
196; AVX-LABEL: testv4i32:
197; AVX:       # BB#0:
198; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
199; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
200; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
201; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
202; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
203; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
204; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
205; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
206; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
207; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
208; AVX-NEXT:    vpsadbw %xmm1, %xmm2, %xmm2
209; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
210; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
211; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
212; AVX-NEXT:    retq
213  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %in)
214  ret <4 x i32> %out
215}
216
217define <8 x i16> @testv8i16(<8 x i16> %in) nounwind {
218; SSE2-LABEL: testv8i16:
219; SSE2:       # BB#0:
220; SSE2-NEXT:    movdqa %xmm0, %xmm1
221; SSE2-NEXT:    psrlw $1, %xmm1
222; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
223; SSE2-NEXT:    psubw %xmm1, %xmm0
224; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
225; SSE2-NEXT:    movdqa %xmm0, %xmm2
226; SSE2-NEXT:    pand %xmm1, %xmm2
227; SSE2-NEXT:    psrlw $2, %xmm0
228; SSE2-NEXT:    pand %xmm1, %xmm0
229; SSE2-NEXT:    paddw %xmm2, %xmm0
230; SSE2-NEXT:    movdqa %xmm0, %xmm1
231; SSE2-NEXT:    psrlw $4, %xmm1
232; SSE2-NEXT:    paddw %xmm0, %xmm1
233; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
234; SSE2-NEXT:    movdqa %xmm1, %xmm0
235; SSE2-NEXT:    psllw $8, %xmm0
236; SSE2-NEXT:    paddb %xmm1, %xmm0
237; SSE2-NEXT:    psrlw $8, %xmm0
238; SSE2-NEXT:    retq
239;
240; SSE3-LABEL: testv8i16:
241; SSE3:       # BB#0:
242; SSE3-NEXT:    movdqa %xmm0, %xmm1
243; SSE3-NEXT:    psrlw $1, %xmm1
244; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
245; SSE3-NEXT:    psubw %xmm1, %xmm0
246; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
247; SSE3-NEXT:    movdqa %xmm0, %xmm2
248; SSE3-NEXT:    pand %xmm1, %xmm2
249; SSE3-NEXT:    psrlw $2, %xmm0
250; SSE3-NEXT:    pand %xmm1, %xmm0
251; SSE3-NEXT:    paddw %xmm2, %xmm0
252; SSE3-NEXT:    movdqa %xmm0, %xmm1
253; SSE3-NEXT:    psrlw $4, %xmm1
254; SSE3-NEXT:    paddw %xmm0, %xmm1
255; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
256; SSE3-NEXT:    movdqa %xmm1, %xmm0
257; SSE3-NEXT:    psllw $8, %xmm0
258; SSE3-NEXT:    paddb %xmm1, %xmm0
259; SSE3-NEXT:    psrlw $8, %xmm0
260; SSE3-NEXT:    retq
261;
262; SSSE3-LABEL: testv8i16:
263; SSSE3:       # BB#0:
264; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
265; SSSE3-NEXT:    movdqa %xmm0, %xmm2
266; SSSE3-NEXT:    pand %xmm1, %xmm2
267; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
268; SSSE3-NEXT:    movdqa %xmm3, %xmm4
269; SSSE3-NEXT:    pshufb %xmm2, %xmm4
270; SSSE3-NEXT:    psrlw $4, %xmm0
271; SSSE3-NEXT:    pand %xmm1, %xmm0
272; SSSE3-NEXT:    pshufb %xmm0, %xmm3
273; SSSE3-NEXT:    paddb %xmm4, %xmm3
274; SSSE3-NEXT:    movdqa %xmm3, %xmm0
275; SSSE3-NEXT:    psllw $8, %xmm0
276; SSSE3-NEXT:    paddb %xmm3, %xmm0
277; SSSE3-NEXT:    psrlw $8, %xmm0
278; SSSE3-NEXT:    retq
279;
280; SSE41-LABEL: testv8i16:
281; SSE41:       # BB#0:
282; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
283; SSE41-NEXT:    movdqa %xmm0, %xmm2
284; SSE41-NEXT:    pand %xmm1, %xmm2
285; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
286; SSE41-NEXT:    movdqa %xmm3, %xmm4
287; SSE41-NEXT:    pshufb %xmm2, %xmm4
288; SSE41-NEXT:    psrlw $4, %xmm0
289; SSE41-NEXT:    pand %xmm1, %xmm0
290; SSE41-NEXT:    pshufb %xmm0, %xmm3
291; SSE41-NEXT:    paddb %xmm4, %xmm3
292; SSE41-NEXT:    movdqa %xmm3, %xmm0
293; SSE41-NEXT:    psllw $8, %xmm0
294; SSE41-NEXT:    paddb %xmm3, %xmm0
295; SSE41-NEXT:    psrlw $8, %xmm0
296; SSE41-NEXT:    retq
297;
298; AVX-LABEL: testv8i16:
299; AVX:       # BB#0:
300; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
301; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
302; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
303; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
304; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
305; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
306; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
307; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
308; AVX-NEXT:    vpsllw $8, %xmm0, %xmm1
309; AVX-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
310; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
311; AVX-NEXT:    retq
312  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %in)
313  ret <8 x i16> %out
314}
315
316define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
317; SSE2-LABEL: testv16i8:
318; SSE2:       # BB#0:
319; SSE2-NEXT:    movdqa %xmm0, %xmm1
320; SSE2-NEXT:    psrlw $1, %xmm1
321; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
322; SSE2-NEXT:    psubb %xmm1, %xmm0
323; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
324; SSE2-NEXT:    movdqa %xmm0, %xmm2
325; SSE2-NEXT:    pand %xmm1, %xmm2
326; SSE2-NEXT:    psrlw $2, %xmm0
327; SSE2-NEXT:    pand %xmm1, %xmm0
328; SSE2-NEXT:    paddb %xmm2, %xmm0
329; SSE2-NEXT:    movdqa %xmm0, %xmm1
330; SSE2-NEXT:    psrlw $4, %xmm1
331; SSE2-NEXT:    paddb %xmm0, %xmm1
332; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
333; SSE2-NEXT:    movdqa %xmm1, %xmm0
334; SSE2-NEXT:    retq
335;
336; SSE3-LABEL: testv16i8:
337; SSE3:       # BB#0:
338; SSE3-NEXT:    movdqa %xmm0, %xmm1
339; SSE3-NEXT:    psrlw $1, %xmm1
340; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
341; SSE3-NEXT:    psubb %xmm1, %xmm0
342; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
343; SSE3-NEXT:    movdqa %xmm0, %xmm2
344; SSE3-NEXT:    pand %xmm1, %xmm2
345; SSE3-NEXT:    psrlw $2, %xmm0
346; SSE3-NEXT:    pand %xmm1, %xmm0
347; SSE3-NEXT:    paddb %xmm2, %xmm0
348; SSE3-NEXT:    movdqa %xmm0, %xmm1
349; SSE3-NEXT:    psrlw $4, %xmm1
350; SSE3-NEXT:    paddb %xmm0, %xmm1
351; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
352; SSE3-NEXT:    movdqa %xmm1, %xmm0
353; SSE3-NEXT:    retq
354;
355; SSSE3-LABEL: testv16i8:
356; SSSE3:       # BB#0:
357; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
358; SSSE3-NEXT:    movdqa %xmm0, %xmm3
359; SSSE3-NEXT:    pand %xmm2, %xmm3
360; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
361; SSSE3-NEXT:    movdqa %xmm1, %xmm4
362; SSSE3-NEXT:    pshufb %xmm3, %xmm4
363; SSSE3-NEXT:    psrlw $4, %xmm0
364; SSSE3-NEXT:    pand %xmm2, %xmm0
365; SSSE3-NEXT:    pshufb %xmm0, %xmm1
366; SSSE3-NEXT:    paddb %xmm4, %xmm1
367; SSSE3-NEXT:    movdqa %xmm1, %xmm0
368; SSSE3-NEXT:    retq
369;
370; SSE41-LABEL: testv16i8:
371; SSE41:       # BB#0:
372; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
373; SSE41-NEXT:    movdqa %xmm0, %xmm3
374; SSE41-NEXT:    pand %xmm2, %xmm3
375; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
376; SSE41-NEXT:    movdqa %xmm1, %xmm4
377; SSE41-NEXT:    pshufb %xmm3, %xmm4
378; SSE41-NEXT:    psrlw $4, %xmm0
379; SSE41-NEXT:    pand %xmm2, %xmm0
380; SSE41-NEXT:    pshufb %xmm0, %xmm1
381; SSE41-NEXT:    paddb %xmm4, %xmm1
382; SSE41-NEXT:    movdqa %xmm1, %xmm0
383; SSE41-NEXT:    retq
384;
385; AVX-LABEL: testv16i8:
386; AVX:       # BB#0:
387; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
388; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
389; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
390; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
391; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
392; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
393; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
394; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
395; AVX-NEXT:    retq
396  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %in)
397  ret <16 x i8> %out
398}
399
400define <2 x i64> @foldv2i64() nounwind {
401; SSE-LABEL: foldv2i64:
402; SSE:       # BB#0:
403; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,64]
404; SSE-NEXT:    retq
405;
406; AVX-LABEL: foldv2i64:
407; AVX:       # BB#0:
408; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,64]
409; AVX-NEXT:    retq
410  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> <i64 256, i64 -1>)
411  ret <2 x i64> %out
412}
413
414define <4 x i32> @foldv4i32() nounwind {
415; SSE-LABEL: foldv4i32:
416; SSE:       # BB#0:
417; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,32,0,8]
418; SSE-NEXT:    retq
419;
420; AVX-LABEL: foldv4i32:
421; AVX:       # BB#0:
422; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,32,0,8]
423; AVX-NEXT:    retq
424  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>)
425  ret <4 x i32> %out
426}
427
428define <8 x i16> @foldv8i16() nounwind {
429; SSE-LABEL: foldv8i16:
430; SSE:       # BB#0:
431; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
432; SSE-NEXT:    retq
433;
434; AVX-LABEL: foldv8i16:
435; AVX:       # BB#0:
436; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
437; AVX-NEXT:    retq
438  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>)
439  ret <8 x i16> %out
440}
441
442define <16 x i8> @foldv16i8() nounwind {
443; SSE-LABEL: foldv16i8:
444; SSE:       # BB#0:
445; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
446; SSE-NEXT:    retq
447;
448; AVX-LABEL: foldv16i8:
449; AVX:       # BB#0:
450; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
451; AVX-NEXT:    retq
452  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>)
453  ret <16 x i8> %out
454}
455
456declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
457declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
458declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
459declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
460