1; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
2; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=-bmi,+sse2 | FileCheck %s -check-prefixes=SSE2,NOBMI
3; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+sse2 | FileCheck %s -check-prefixes=SSE2,BMI
4; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+sse4.2 | FileCheck %s -check-prefixes=BMI,SSE42
5; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx | FileCheck %s -check-prefixes=BMI,AVX1
6; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx2 | FileCheck %s -check-prefixes=BMI,AVX2
7; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx512f | FileCheck %s -check-prefixes=BMI,AVX512,AVX512F
8; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=BMI,AVX512,AVX512BW
9; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=BMI,AVX512,AVX512BW
10; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx512vl,+avx512vpopcntdq | FileCheck %s -check-prefixes=BMI,AVX512,AVX512VPOPCNT
11; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+bmi,+avx512vl,+avx512bitalg | FileCheck %s -check-prefixes=BMI,AVX512,AVX512BITALG
12
13; Verify the cost of scalar trailing zero count instructions.
14
15declare i64 @llvm.cttz.i64(i64, i1)
16declare i32 @llvm.cttz.i32(i32, i1)
17declare i16 @llvm.cttz.i16(i16, i1)
18declare  i8 @llvm.cttz.i8(i8, i1)
19
20define i64 @var_cttz_i64(i64 %a) {
21; NOBMI-LABEL: 'var_cttz_i64'
22; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
23; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
24;
25; BMI-LABEL: 'var_cttz_i64'
26; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
27; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
28;
29  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 0)
30  ret i64 %cttz
31}
32
33define i64 @var_cttz_i64u(i64 %a) {
34; NOBMI-LABEL: 'var_cttz_i64u'
35; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
36; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
37;
38; BMI-LABEL: 'var_cttz_i64u'
39; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
40; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
41;
42  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 1)
43  ret i64 %cttz
44}
45
46define i32 @var_cttz_i32(i32 %a) {
47; NOBMI-LABEL: 'var_cttz_i32'
48; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
49; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
50;
51; BMI-LABEL: 'var_cttz_i32'
52; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
53; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
54;
55  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 0)
56  ret i32 %cttz
57}
58
59define i32 @var_cttz_i32u(i32 %a) {
60; NOBMI-LABEL: 'var_cttz_i32u'
61; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
62; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
63;
64; BMI-LABEL: 'var_cttz_i32u'
65; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
66; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
67;
68  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 1)
69  ret i32 %cttz
70}
71
72define i16 @var_cttz_i16(i16 %a) {
73; NOBMI-LABEL: 'var_cttz_i16'
74; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
75; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
76;
77; BMI-LABEL: 'var_cttz_i16'
78; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
79; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
80;
81  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 0)
82  ret i16 %cttz
83}
84
85define i16 @var_cttz_i16u(i16 %a) {
86; NOBMI-LABEL: 'var_cttz_i16u'
87; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
88; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
89;
90; BMI-LABEL: 'var_cttz_i16u'
91; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
92; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
93;
94  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 1)
95  ret i16 %cttz
96}
97
98define i8 @var_cttz_i8(i8 %a) {
99; NOBMI-LABEL: 'var_cttz_i8'
100; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
101; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
102;
103; BMI-LABEL: 'var_cttz_i8'
104; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
105; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
106;
107  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 0)
108  ret i8 %cttz
109}
110
111define i8 @var_cttz_i8u(i8 %a) {
112; NOBMI-LABEL: 'var_cttz_i8u'
113; NOBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
114; NOBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
115;
116; BMI-LABEL: 'var_cttz_i8u'
117; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
118; BMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
119;
120  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 1)
121  ret i8 %cttz
122}
123
124; Verify the cost of vector trailing zero count instructions.
125
126declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
127declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
128declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
129declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
130
131declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
132declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
133declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
134declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
135
136declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
137declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
138declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)
139declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)
140
141define <2 x i64> @var_cttz_v2i64(<2 x i64> %a) {
142; SSE2-LABEL: 'var_cttz_v2i64'
143; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
144; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
145;
146; SSE42-LABEL: 'var_cttz_v2i64'
147; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
148; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
149;
150; AVX1-LABEL: 'var_cttz_v2i64'
151; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
152; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
153;
154; AVX2-LABEL: 'var_cttz_v2i64'
155; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
156; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
157;
158; AVX512-LABEL: 'var_cttz_v2i64'
159; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
160; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
161;
162  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 0)
163  ret <2 x i64> %cttz
164}
165
166define <2 x i64> @var_cttz_v2i64u(<2 x i64> %a) {
167; SSE2-LABEL: 'var_cttz_v2i64u'
168; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
169; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
170;
171; SSE42-LABEL: 'var_cttz_v2i64u'
172; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
173; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
174;
175; AVX1-LABEL: 'var_cttz_v2i64u'
176; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
177; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
178;
179; AVX2-LABEL: 'var_cttz_v2i64u'
180; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
181; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
182;
183; AVX512-LABEL: 'var_cttz_v2i64u'
184; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
185; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %cttz
186;
187  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 1)
188  ret <2 x i64> %cttz
189}
190
191define <4 x i64> @var_cttz_v4i64(<4 x i64> %a) {
192; SSE2-LABEL: 'var_cttz_v4i64'
193; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
194; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
195;
196; SSE42-LABEL: 'var_cttz_v4i64'
197; SSE42-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
198; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
199;
200; AVX1-LABEL: 'var_cttz_v4i64'
201; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
202; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
203;
204; AVX2-LABEL: 'var_cttz_v4i64'
205; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
206; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
207;
208; AVX512-LABEL: 'var_cttz_v4i64'
209; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
210; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
211;
212  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 0)
213  ret <4 x i64> %cttz
214}
215
216define <4 x i64> @var_cttz_v4i64u(<4 x i64> %a) {
217; SSE2-LABEL: 'var_cttz_v4i64u'
218; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
219; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
220;
221; SSE42-LABEL: 'var_cttz_v4i64u'
222; SSE42-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
223; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
224;
225; AVX1-LABEL: 'var_cttz_v4i64u'
226; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
227; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
228;
229; AVX2-LABEL: 'var_cttz_v4i64u'
230; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
231; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
232;
233; AVX512-LABEL: 'var_cttz_v4i64u'
234; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
235; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %cttz
236;
237  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 1)
238  ret <4 x i64> %cttz
239}
240
241define <8 x i64> @var_cttz_v8i64(<8 x i64> %a) {
242; SSE2-LABEL: 'var_cttz_v8i64'
243; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
244; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
245;
246; SSE42-LABEL: 'var_cttz_v8i64'
247; SSE42-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
248; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
249;
250; AVX1-LABEL: 'var_cttz_v8i64'
251; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
252; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
253;
254; AVX2-LABEL: 'var_cttz_v8i64'
255; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
256; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
257;
258; AVX512F-LABEL: 'var_cttz_v8i64'
259; AVX512F-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
260; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
261;
262; AVX512BW-LABEL: 'var_cttz_v8i64'
263; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
264; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
265;
266; AVX512VPOPCNT-LABEL: 'var_cttz_v8i64'
267; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
268; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
269;
270; AVX512BITALG-LABEL: 'var_cttz_v8i64'
271; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
272; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
273;
274  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 0)
275  ret <8 x i64> %cttz
276}
277
278define <8 x i64> @var_cttz_v8i64u(<8 x i64> %a) {
279; SSE2-LABEL: 'var_cttz_v8i64u'
280; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
281; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
282;
283; SSE42-LABEL: 'var_cttz_v8i64u'
284; SSE42-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
285; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
286;
287; AVX1-LABEL: 'var_cttz_v8i64u'
288; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
289; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
290;
291; AVX2-LABEL: 'var_cttz_v8i64u'
292; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
293; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
294;
295; AVX512F-LABEL: 'var_cttz_v8i64u'
296; AVX512F-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
297; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
298;
299; AVX512BW-LABEL: 'var_cttz_v8i64u'
300; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
301; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
302;
303; AVX512VPOPCNT-LABEL: 'var_cttz_v8i64u'
304; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
305; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
306;
307; AVX512BITALG-LABEL: 'var_cttz_v8i64u'
308; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
309; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %cttz
310;
311  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 1)
312  ret <8 x i64> %cttz
313}
314
315define <4 x i32> @var_cttz_v4i32(<4 x i32> %a) {
316; SSE2-LABEL: 'var_cttz_v4i32'
317; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
318; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
319;
320; SSE42-LABEL: 'var_cttz_v4i32'
321; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
322; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
323;
324; AVX1-LABEL: 'var_cttz_v4i32'
325; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
326; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
327;
328; AVX2-LABEL: 'var_cttz_v4i32'
329; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
330; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
331;
332; AVX512-LABEL: 'var_cttz_v4i32'
333; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
334; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
335;
336  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 0)
337  ret <4 x i32> %cttz
338}
339
340define <4 x i32> @var_cttz_v4i32u(<4 x i32> %a) {
341; SSE2-LABEL: 'var_cttz_v4i32u'
342; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
343; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
344;
345; SSE42-LABEL: 'var_cttz_v4i32u'
346; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
347; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
348;
349; AVX1-LABEL: 'var_cttz_v4i32u'
350; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
351; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
352;
353; AVX2-LABEL: 'var_cttz_v4i32u'
354; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
355; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
356;
357; AVX512-LABEL: 'var_cttz_v4i32u'
358; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
359; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %cttz
360;
361  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 1)
362  ret <4 x i32> %cttz
363}
364
365define <8 x i32> @var_cttz_v8i32(<8 x i32> %a) {
366; SSE2-LABEL: 'var_cttz_v8i32'
367; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
368; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
369;
370; SSE42-LABEL: 'var_cttz_v8i32'
371; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
372; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
373;
374; AVX1-LABEL: 'var_cttz_v8i32'
375; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
376; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
377;
378; AVX2-LABEL: 'var_cttz_v8i32'
379; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
380; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
381;
382; AVX512-LABEL: 'var_cttz_v8i32'
383; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
384; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
385;
386  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 0)
387  ret <8 x i32> %cttz
388}
389
390define <8 x i32> @var_cttz_v8i32u(<8 x i32> %a) {
391; SSE2-LABEL: 'var_cttz_v8i32u'
392; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
393; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
394;
395; SSE42-LABEL: 'var_cttz_v8i32u'
396; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
397; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
398;
399; AVX1-LABEL: 'var_cttz_v8i32u'
400; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
401; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
402;
403; AVX2-LABEL: 'var_cttz_v8i32u'
404; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
405; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
406;
407; AVX512-LABEL: 'var_cttz_v8i32u'
408; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
409; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %cttz
410;
411  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 1)
412  ret <8 x i32> %cttz
413}
414
415define <16 x i32> @var_cttz_v16i32(<16 x i32> %a) {
416; SSE2-LABEL: 'var_cttz_v16i32'
417; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
418; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
419;
420; SSE42-LABEL: 'var_cttz_v16i32'
421; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
422; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
423;
424; AVX1-LABEL: 'var_cttz_v16i32'
425; AVX1-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
426; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
427;
428; AVX2-LABEL: 'var_cttz_v16i32'
429; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
430; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
431;
432; AVX512F-LABEL: 'var_cttz_v16i32'
433; AVX512F-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
434; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
435;
436; AVX512BW-LABEL: 'var_cttz_v16i32'
437; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
438; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
439;
440; AVX512VPOPCNT-LABEL: 'var_cttz_v16i32'
441; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
442; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
443;
444; AVX512BITALG-LABEL: 'var_cttz_v16i32'
445; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
446; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
447;
448  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 0)
449  ret <16 x i32> %cttz
450}
451
452define <16 x i32> @var_cttz_v16i32u(<16 x i32> %a) {
453; SSE2-LABEL: 'var_cttz_v16i32u'
454; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
455; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
456;
457; SSE42-LABEL: 'var_cttz_v16i32u'
458; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
459; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
460;
461; AVX1-LABEL: 'var_cttz_v16i32u'
462; AVX1-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
463; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
464;
465; AVX2-LABEL: 'var_cttz_v16i32u'
466; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
467; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
468;
469; AVX512F-LABEL: 'var_cttz_v16i32u'
470; AVX512F-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
471; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
472;
473; AVX512BW-LABEL: 'var_cttz_v16i32u'
474; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
475; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
476;
477; AVX512VPOPCNT-LABEL: 'var_cttz_v16i32u'
478; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
479; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
480;
481; AVX512BITALG-LABEL: 'var_cttz_v16i32u'
482; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
483; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %cttz
484;
485  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 1)
486  ret <16 x i32> %cttz
487}
488
489define <8 x i16> @var_cttz_v8i16(<8 x i16> %a) {
490; SSE2-LABEL: 'var_cttz_v8i16'
491; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
492; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
493;
494; SSE42-LABEL: 'var_cttz_v8i16'
495; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
496; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
497;
498; AVX1-LABEL: 'var_cttz_v8i16'
499; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
500; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
501;
502; AVX2-LABEL: 'var_cttz_v8i16'
503; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
504; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
505;
506; AVX512-LABEL: 'var_cttz_v8i16'
507; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
508; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
509;
510  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 0)
511  ret <8 x i16> %cttz
512}
513
514define <8 x i16> @var_cttz_v8i16u(<8 x i16> %a) {
515; SSE2-LABEL: 'var_cttz_v8i16u'
516; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
517; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
518;
519; SSE42-LABEL: 'var_cttz_v8i16u'
520; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
521; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
522;
523; AVX1-LABEL: 'var_cttz_v8i16u'
524; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
525; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
526;
527; AVX2-LABEL: 'var_cttz_v8i16u'
528; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
529; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
530;
531; AVX512-LABEL: 'var_cttz_v8i16u'
532; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
533; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %cttz
534;
535  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 1)
536  ret <8 x i16> %cttz
537}
538
539define <16 x i16> @var_cttz_v16i16(<16 x i16> %a) {
540; SSE2-LABEL: 'var_cttz_v16i16'
541; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
542; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
543;
544; SSE42-LABEL: 'var_cttz_v16i16'
545; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
546; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
547;
548; AVX1-LABEL: 'var_cttz_v16i16'
549; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
550; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
551;
552; AVX2-LABEL: 'var_cttz_v16i16'
553; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
554; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
555;
556; AVX512-LABEL: 'var_cttz_v16i16'
557; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
558; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
559;
560  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 0)
561  ret <16 x i16> %cttz
562}
563
564define <16 x i16> @var_cttz_v16i16u(<16 x i16> %a) {
565; SSE2-LABEL: 'var_cttz_v16i16u'
566; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
567; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
568;
569; SSE42-LABEL: 'var_cttz_v16i16u'
570; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
571; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
572;
573; AVX1-LABEL: 'var_cttz_v16i16u'
574; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
575; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
576;
577; AVX2-LABEL: 'var_cttz_v16i16u'
578; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
579; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
580;
581; AVX512-LABEL: 'var_cttz_v16i16u'
582; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
583; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %cttz
584;
585  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 1)
586  ret <16 x i16> %cttz
587}
588
589define <32 x i16> @var_cttz_v32i16(<32 x i16> %a) {
590; SSE2-LABEL: 'var_cttz_v32i16'
591; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
592; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
593;
594; SSE42-LABEL: 'var_cttz_v32i16'
595; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
596; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
597;
598; AVX1-LABEL: 'var_cttz_v32i16'
599; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
600; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
601;
602; AVX2-LABEL: 'var_cttz_v32i16'
603; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
604; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
605;
606; AVX512F-LABEL: 'var_cttz_v32i16'
607; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
608; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
609;
610; AVX512BW-LABEL: 'var_cttz_v32i16'
611; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
612; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
613;
614; AVX512VPOPCNT-LABEL: 'var_cttz_v32i16'
615; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
616; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
617;
618; AVX512BITALG-LABEL: 'var_cttz_v32i16'
619; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
620; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
621;
622  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 0)
623  ret <32 x i16> %cttz
624}
625
626define <32 x i16> @var_cttz_v32i16u(<32 x i16> %a) {
627; SSE2-LABEL: 'var_cttz_v32i16u'
628; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
629; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
630;
631; SSE42-LABEL: 'var_cttz_v32i16u'
632; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
633; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
634;
635; AVX1-LABEL: 'var_cttz_v32i16u'
636; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
637; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
638;
639; AVX2-LABEL: 'var_cttz_v32i16u'
640; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
641; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
642;
643; AVX512F-LABEL: 'var_cttz_v32i16u'
644; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
645; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
646;
647; AVX512BW-LABEL: 'var_cttz_v32i16u'
648; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
649; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
650;
651; AVX512VPOPCNT-LABEL: 'var_cttz_v32i16u'
652; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
653; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
654;
655; AVX512BITALG-LABEL: 'var_cttz_v32i16u'
656; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
657; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %cttz
658;
659  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 1)
660  ret <32 x i16> %cttz
661}
662
663define <16 x i8> @var_cttz_v16i8(<16 x i8> %a) {
664; SSE2-LABEL: 'var_cttz_v16i8'
665; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
666; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
667;
668; SSE42-LABEL: 'var_cttz_v16i8'
669; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
670; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
671;
672; AVX1-LABEL: 'var_cttz_v16i8'
673; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
674; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
675;
676; AVX2-LABEL: 'var_cttz_v16i8'
677; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
678; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
679;
680; AVX512-LABEL: 'var_cttz_v16i8'
681; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
682; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
683;
684  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 0)
685  ret <16 x i8> %cttz
686}
687
688define <16 x i8> @var_cttz_v16i8u(<16 x i8> %a) {
689; SSE2-LABEL: 'var_cttz_v16i8u'
690; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
691; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
692;
693; SSE42-LABEL: 'var_cttz_v16i8u'
694; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
695; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
696;
697; AVX1-LABEL: 'var_cttz_v16i8u'
698; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
699; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
700;
701; AVX2-LABEL: 'var_cttz_v16i8u'
702; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
703; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
704;
705; AVX512-LABEL: 'var_cttz_v16i8u'
706; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
707; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %cttz
708;
709  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 1)
710  ret <16 x i8> %cttz
711}
712
713define <32 x i8> @var_cttz_v32i8(<32 x i8> %a) {
714; SSE2-LABEL: 'var_cttz_v32i8'
715; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
716; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
717;
718; SSE42-LABEL: 'var_cttz_v32i8'
719; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
720; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
721;
722; AVX1-LABEL: 'var_cttz_v32i8'
723; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
724; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
725;
726; AVX2-LABEL: 'var_cttz_v32i8'
727; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
728; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
729;
730; AVX512-LABEL: 'var_cttz_v32i8'
731; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
732; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
733;
734  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 0)
735  ret <32 x i8> %cttz
736}
737
738define <32 x i8> @var_cttz_v32i8u(<32 x i8> %a) {
739; SSE2-LABEL: 'var_cttz_v32i8u'
740; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
741; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
742;
743; SSE42-LABEL: 'var_cttz_v32i8u'
744; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
745; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
746;
747; AVX1-LABEL: 'var_cttz_v32i8u'
748; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
749; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
750;
751; AVX2-LABEL: 'var_cttz_v32i8u'
752; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
753; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
754;
755; AVX512-LABEL: 'var_cttz_v32i8u'
756; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
757; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %cttz
758;
759  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 1)
760  ret <32 x i8> %cttz
761}
762
763define <64 x i8> @var_cttz_v64i8(<64 x i8> %a) {
764; SSE2-LABEL: 'var_cttz_v64i8'
765; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
766; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
767;
768; SSE42-LABEL: 'var_cttz_v64i8'
769; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
770; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
771;
772; AVX1-LABEL: 'var_cttz_v64i8'
773; AVX1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
774; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
775;
776; AVX2-LABEL: 'var_cttz_v64i8'
777; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
778; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
779;
780; AVX512F-LABEL: 'var_cttz_v64i8'
781; AVX512F-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
782; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
783;
784; AVX512BW-LABEL: 'var_cttz_v64i8'
785; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
786; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
787;
788; AVX512VPOPCNT-LABEL: 'var_cttz_v64i8'
789; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
790; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
791;
792; AVX512BITALG-LABEL: 'var_cttz_v64i8'
793; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
794; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
795;
796  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 0)
797  ret <64 x i8> %cttz
798}
799
800define <64 x i8> @var_cttz_v64i8u(<64 x i8> %a) {
801; SSE2-LABEL: 'var_cttz_v64i8u'
802; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
803; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
804;
805; SSE42-LABEL: 'var_cttz_v64i8u'
806; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
807; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
808;
809; AVX1-LABEL: 'var_cttz_v64i8u'
810; AVX1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
811; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
812;
813; AVX2-LABEL: 'var_cttz_v64i8u'
814; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
815; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
816;
817; AVX512F-LABEL: 'var_cttz_v64i8u'
818; AVX512F-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
819; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
820;
821; AVX512BW-LABEL: 'var_cttz_v64i8u'
822; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
823; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
824;
825; AVX512VPOPCNT-LABEL: 'var_cttz_v64i8u'
826; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
827; AVX512VPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
828;
829; AVX512BITALG-LABEL: 'var_cttz_v64i8u'
830; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
831; AVX512BITALG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %cttz
832;
833  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 1)
834  ret <64 x i8> %cttz
835}
836