1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512VL --check-prefix=AVX512BWVL
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ
9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512VL --check-prefix=AVX512DQVL
10
11;
12; vXi64
13;
14
15define i64 @test_v2i64(<2 x i64> %a0) {
16; SSE-LABEL: test_v2i64:
17; SSE:       # %bb.0:
18; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
19; SSE-NEXT:    movdqa %xmm0, %xmm2
20; SSE-NEXT:    psrlq $32, %xmm2
21; SSE-NEXT:    pmuludq %xmm1, %xmm2
22; SSE-NEXT:    movdqa %xmm0, %xmm3
23; SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
24; SSE-NEXT:    pmuludq %xmm0, %xmm3
25; SSE-NEXT:    paddq %xmm2, %xmm3
26; SSE-NEXT:    psllq $32, %xmm3
27; SSE-NEXT:    pmuludq %xmm1, %xmm0
28; SSE-NEXT:    paddq %xmm3, %xmm0
29; SSE-NEXT:    movq %xmm0, %rax
30; SSE-NEXT:    retq
31;
32; AVX-LABEL: test_v2i64:
33; AVX:       # %bb.0:
34; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
35; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm2
36; AVX-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
37; AVX-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
38; AVX-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
39; AVX-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
40; AVX-NEXT:    vpsllq $32, %xmm2, %xmm2
41; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
42; AVX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
43; AVX-NEXT:    vmovq %xmm0, %rax
44; AVX-NEXT:    retq
45;
46; AVX512BW-LABEL: test_v2i64:
47; AVX512BW:       # %bb.0:
48; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
49; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
50; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
51; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
52; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
53; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
54; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
55; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
56; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
57; AVX512BW-NEXT:    vmovq %xmm0, %rax
58; AVX512BW-NEXT:    retq
59;
60; AVX512BWVL-LABEL: test_v2i64:
61; AVX512BWVL:       # %bb.0:
62; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
63; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
64; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
65; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
66; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
67; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
68; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
69; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
70; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
71; AVX512BWVL-NEXT:    vmovq %xmm0, %rax
72; AVX512BWVL-NEXT:    retq
73;
74; AVX512DQ-LABEL: test_v2i64:
75; AVX512DQ:       # %bb.0:
76; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
77; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
78; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
79; AVX512DQ-NEXT:    vmovq %xmm0, %rax
80; AVX512DQ-NEXT:    vzeroupper
81; AVX512DQ-NEXT:    retq
82;
83; AVX512DQVL-LABEL: test_v2i64:
84; AVX512DQVL:       # %bb.0:
85; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
86; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
87; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
88; AVX512DQVL-NEXT:    retq
89  %1 = call i64 @llvm.experimental.vector.reduce.mul.v2i64(<2 x i64> %a0)
90  ret i64 %1
91}
92
93define i64 @test_v4i64(<4 x i64> %a0) {
94; SSE-LABEL: test_v4i64:
95; SSE:       # %bb.0:
96; SSE-NEXT:    movdqa %xmm0, %xmm2
97; SSE-NEXT:    psrlq $32, %xmm2
98; SSE-NEXT:    pmuludq %xmm1, %xmm2
99; SSE-NEXT:    movdqa %xmm1, %xmm3
100; SSE-NEXT:    psrlq $32, %xmm3
101; SSE-NEXT:    pmuludq %xmm0, %xmm3
102; SSE-NEXT:    paddq %xmm2, %xmm3
103; SSE-NEXT:    psllq $32, %xmm3
104; SSE-NEXT:    pmuludq %xmm1, %xmm0
105; SSE-NEXT:    paddq %xmm3, %xmm0
106; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
107; SSE-NEXT:    movdqa %xmm0, %xmm2
108; SSE-NEXT:    psrlq $32, %xmm2
109; SSE-NEXT:    pmuludq %xmm1, %xmm2
110; SSE-NEXT:    movdqa %xmm0, %xmm3
111; SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
112; SSE-NEXT:    pmuludq %xmm0, %xmm3
113; SSE-NEXT:    paddq %xmm2, %xmm3
114; SSE-NEXT:    psllq $32, %xmm3
115; SSE-NEXT:    pmuludq %xmm1, %xmm0
116; SSE-NEXT:    paddq %xmm3, %xmm0
117; SSE-NEXT:    movq %xmm0, %rax
118; SSE-NEXT:    retq
119;
120; AVX1-LABEL: test_v4i64:
121; AVX1:       # %bb.0:
122; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
123; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
124; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
125; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
126; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
127; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
128; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
129; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
130; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
131; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
132; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
133; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
134; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
135; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
136; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
137; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
138; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
139; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
140; AVX1-NEXT:    vmovq %xmm0, %rax
141; AVX1-NEXT:    vzeroupper
142; AVX1-NEXT:    retq
143;
144; AVX2-LABEL: test_v4i64:
145; AVX2:       # %bb.0:
146; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
147; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
148; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
149; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3
150; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
151; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
152; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
153; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
154; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
155; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
156; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
157; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
158; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
159; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
160; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
161; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
162; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
163; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
164; AVX2-NEXT:    vmovq %xmm0, %rax
165; AVX2-NEXT:    vzeroupper
166; AVX2-NEXT:    retq
167;
168; AVX512BW-LABEL: test_v4i64:
169; AVX512BW:       # %bb.0:
170; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
171; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
172; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
173; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3
174; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
175; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
176; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
177; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
178; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
179; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
180; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
181; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
182; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
183; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
184; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
185; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
186; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
187; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
188; AVX512BW-NEXT:    vmovq %xmm0, %rax
189; AVX512BW-NEXT:    vzeroupper
190; AVX512BW-NEXT:    retq
191;
192; AVX512BWVL-LABEL: test_v4i64:
193; AVX512BWVL:       # %bb.0:
194; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
195; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
196; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
197; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
198; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
199; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
200; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
201; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
202; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
203; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
204; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
205; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
206; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
207; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
208; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
209; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
210; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
211; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
212; AVX512BWVL-NEXT:    vmovq %xmm0, %rax
213; AVX512BWVL-NEXT:    vzeroupper
214; AVX512BWVL-NEXT:    retq
215;
216; AVX512DQ-LABEL: test_v4i64:
217; AVX512DQ:       # %bb.0:
218; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
219; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
220; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
221; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
222; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
223; AVX512DQ-NEXT:    vmovq %xmm0, %rax
224; AVX512DQ-NEXT:    vzeroupper
225; AVX512DQ-NEXT:    retq
226;
227; AVX512DQVL-LABEL: test_v4i64:
228; AVX512DQVL:       # %bb.0:
229; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
230; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
231; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
232; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
233; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
234; AVX512DQVL-NEXT:    vzeroupper
235; AVX512DQVL-NEXT:    retq
236  %1 = call i64 @llvm.experimental.vector.reduce.mul.v4i64(<4 x i64> %a0)
237  ret i64 %1
238}
239
240define i64 @test_v8i64(<8 x i64> %a0) {
241; SSE-LABEL: test_v8i64:
242; SSE:       # %bb.0:
243; SSE-NEXT:    movdqa %xmm1, %xmm4
244; SSE-NEXT:    psrlq $32, %xmm4
245; SSE-NEXT:    pmuludq %xmm3, %xmm4
246; SSE-NEXT:    movdqa %xmm3, %xmm5
247; SSE-NEXT:    psrlq $32, %xmm5
248; SSE-NEXT:    pmuludq %xmm1, %xmm5
249; SSE-NEXT:    paddq %xmm4, %xmm5
250; SSE-NEXT:    psllq $32, %xmm5
251; SSE-NEXT:    pmuludq %xmm3, %xmm1
252; SSE-NEXT:    paddq %xmm5, %xmm1
253; SSE-NEXT:    movdqa %xmm0, %xmm3
254; SSE-NEXT:    psrlq $32, %xmm3
255; SSE-NEXT:    pmuludq %xmm2, %xmm3
256; SSE-NEXT:    movdqa %xmm2, %xmm4
257; SSE-NEXT:    psrlq $32, %xmm4
258; SSE-NEXT:    pmuludq %xmm0, %xmm4
259; SSE-NEXT:    paddq %xmm3, %xmm4
260; SSE-NEXT:    psllq $32, %xmm4
261; SSE-NEXT:    pmuludq %xmm2, %xmm0
262; SSE-NEXT:    paddq %xmm4, %xmm0
263; SSE-NEXT:    movdqa %xmm0, %xmm2
264; SSE-NEXT:    psrlq $32, %xmm2
265; SSE-NEXT:    pmuludq %xmm1, %xmm2
266; SSE-NEXT:    movdqa %xmm1, %xmm3
267; SSE-NEXT:    psrlq $32, %xmm3
268; SSE-NEXT:    pmuludq %xmm0, %xmm3
269; SSE-NEXT:    paddq %xmm2, %xmm3
270; SSE-NEXT:    psllq $32, %xmm3
271; SSE-NEXT:    pmuludq %xmm1, %xmm0
272; SSE-NEXT:    paddq %xmm3, %xmm0
273; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
274; SSE-NEXT:    movdqa %xmm0, %xmm2
275; SSE-NEXT:    psrlq $32, %xmm2
276; SSE-NEXT:    pmuludq %xmm1, %xmm2
277; SSE-NEXT:    movdqa %xmm0, %xmm3
278; SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
279; SSE-NEXT:    pmuludq %xmm0, %xmm3
280; SSE-NEXT:    paddq %xmm2, %xmm3
281; SSE-NEXT:    psllq $32, %xmm3
282; SSE-NEXT:    pmuludq %xmm1, %xmm0
283; SSE-NEXT:    paddq %xmm3, %xmm0
284; SSE-NEXT:    movq %xmm0, %rax
285; SSE-NEXT:    retq
286;
287; AVX1-LABEL: test_v8i64:
288; AVX1:       # %bb.0:
289; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
290; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
291; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm4
292; AVX1-NEXT:    vpmuludq %xmm2, %xmm4, %xmm4
293; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm5
294; AVX1-NEXT:    vpmuludq %xmm5, %xmm3, %xmm5
295; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4
296; AVX1-NEXT:    vpsllq $32, %xmm4, %xmm4
297; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
298; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
299; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3
300; AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm3
301; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
302; AVX1-NEXT:    vpmuludq %xmm4, %xmm0, %xmm4
303; AVX1-NEXT:    vpaddq %xmm3, %xmm4, %xmm3
304; AVX1-NEXT:    vpsllq $32, %xmm3, %xmm3
305; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
306; AVX1-NEXT:    vpaddq %xmm3, %xmm0, %xmm0
307; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm1
308; AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
309; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm3
310; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
311; AVX1-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
312; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
313; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
314; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
315; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
316; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
317; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
318; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
319; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
320; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
321; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
322; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
323; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
324; AVX1-NEXT:    vmovq %xmm0, %rax
325; AVX1-NEXT:    vzeroupper
326; AVX1-NEXT:    retq
327;
328; AVX2-LABEL: test_v8i64:
329; AVX2:       # %bb.0:
330; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm2
331; AVX2-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2
332; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm3
333; AVX2-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3
334; AVX2-NEXT:    vpaddq %ymm2, %ymm3, %ymm2
335; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
336; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
337; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
338; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
339; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
340; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
341; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3
342; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
343; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
344; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
345; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
346; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
347; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
348; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
349; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
350; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
351; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
352; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
353; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
354; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
355; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
356; AVX2-NEXT:    vmovq %xmm0, %rax
357; AVX2-NEXT:    vzeroupper
358; AVX2-NEXT:    retq
359;
360; AVX512BW-LABEL: test_v8i64:
361; AVX512BW:       # %bb.0:
362; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
363; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm2
364; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
365; AVX512BW-NEXT:    vpsrlq $32, %zmm1, %zmm3
366; AVX512BW-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
367; AVX512BW-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
368; AVX512BW-NEXT:    vpsllq $32, %zmm2, %zmm2
369; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
370; AVX512BW-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
371; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
372; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
373; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
374; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3
375; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
376; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
377; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
378; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
379; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
380; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
381; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
382; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
383; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
384; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
385; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
386; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
387; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
388; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
389; AVX512BW-NEXT:    vmovq %xmm0, %rax
390; AVX512BW-NEXT:    vzeroupper
391; AVX512BW-NEXT:    retq
392;
393; AVX512BWVL-LABEL: test_v8i64:
394; AVX512BWVL:       # %bb.0:
395; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
396; AVX512BWVL-NEXT:    vpsrlq $32, %zmm0, %zmm2
397; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
398; AVX512BWVL-NEXT:    vpsrlq $32, %zmm1, %zmm3
399; AVX512BWVL-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
400; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
401; AVX512BWVL-NEXT:    vpsllq $32, %zmm2, %zmm2
402; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
403; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
404; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
405; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
406; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
407; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
408; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
409; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
410; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
411; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
412; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
413; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
414; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
415; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
416; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
417; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
418; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
419; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
420; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
421; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
422; AVX512BWVL-NEXT:    vmovq %xmm0, %rax
423; AVX512BWVL-NEXT:    vzeroupper
424; AVX512BWVL-NEXT:    retq
425;
426; AVX512DQ-LABEL: test_v8i64:
427; AVX512DQ:       # %bb.0:
428; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
429; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
430; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
431; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
432; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
433; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
434; AVX512DQ-NEXT:    vmovq %xmm0, %rax
435; AVX512DQ-NEXT:    vzeroupper
436; AVX512DQ-NEXT:    retq
437;
438; AVX512DQVL-LABEL: test_v8i64:
439; AVX512DQVL:       # %bb.0:
440; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
441; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
442; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
443; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
444; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
445; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
446; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
447; AVX512DQVL-NEXT:    vzeroupper
448; AVX512DQVL-NEXT:    retq
449  %1 = call i64 @llvm.experimental.vector.reduce.mul.v8i64(<8 x i64> %a0)
450  ret i64 %1
451}
452
453define i64 @test_v16i64(<16 x i64> %a0) {
454; SSE-LABEL: test_v16i64:
455; SSE:       # %bb.0:
456; SSE-NEXT:    movdqa %xmm2, %xmm8
457; SSE-NEXT:    psrlq $32, %xmm8
458; SSE-NEXT:    pmuludq %xmm6, %xmm8
459; SSE-NEXT:    movdqa %xmm6, %xmm9
460; SSE-NEXT:    psrlq $32, %xmm9
461; SSE-NEXT:    pmuludq %xmm2, %xmm9
462; SSE-NEXT:    paddq %xmm8, %xmm9
463; SSE-NEXT:    psllq $32, %xmm9
464; SSE-NEXT:    pmuludq %xmm6, %xmm2
465; SSE-NEXT:    paddq %xmm9, %xmm2
466; SSE-NEXT:    movdqa %xmm0, %xmm8
467; SSE-NEXT:    psrlq $32, %xmm8
468; SSE-NEXT:    pmuludq %xmm4, %xmm8
469; SSE-NEXT:    movdqa %xmm4, %xmm6
470; SSE-NEXT:    psrlq $32, %xmm6
471; SSE-NEXT:    pmuludq %xmm0, %xmm6
472; SSE-NEXT:    paddq %xmm8, %xmm6
473; SSE-NEXT:    psllq $32, %xmm6
474; SSE-NEXT:    pmuludq %xmm4, %xmm0
475; SSE-NEXT:    paddq %xmm6, %xmm0
476; SSE-NEXT:    movdqa %xmm3, %xmm4
477; SSE-NEXT:    psrlq $32, %xmm4
478; SSE-NEXT:    pmuludq %xmm7, %xmm4
479; SSE-NEXT:    movdqa %xmm7, %xmm6
480; SSE-NEXT:    psrlq $32, %xmm6
481; SSE-NEXT:    pmuludq %xmm3, %xmm6
482; SSE-NEXT:    paddq %xmm4, %xmm6
483; SSE-NEXT:    psllq $32, %xmm6
484; SSE-NEXT:    pmuludq %xmm7, %xmm3
485; SSE-NEXT:    paddq %xmm6, %xmm3
486; SSE-NEXT:    movdqa %xmm1, %xmm4
487; SSE-NEXT:    psrlq $32, %xmm4
488; SSE-NEXT:    pmuludq %xmm5, %xmm4
489; SSE-NEXT:    movdqa %xmm5, %xmm6
490; SSE-NEXT:    psrlq $32, %xmm6
491; SSE-NEXT:    pmuludq %xmm1, %xmm6
492; SSE-NEXT:    paddq %xmm4, %xmm6
493; SSE-NEXT:    psllq $32, %xmm6
494; SSE-NEXT:    pmuludq %xmm5, %xmm1
495; SSE-NEXT:    paddq %xmm6, %xmm1
496; SSE-NEXT:    movdqa %xmm1, %xmm4
497; SSE-NEXT:    psrlq $32, %xmm4
498; SSE-NEXT:    pmuludq %xmm3, %xmm4
499; SSE-NEXT:    movdqa %xmm3, %xmm5
500; SSE-NEXT:    psrlq $32, %xmm5
501; SSE-NEXT:    pmuludq %xmm1, %xmm5
502; SSE-NEXT:    paddq %xmm4, %xmm5
503; SSE-NEXT:    psllq $32, %xmm5
504; SSE-NEXT:    pmuludq %xmm3, %xmm1
505; SSE-NEXT:    paddq %xmm5, %xmm1
506; SSE-NEXT:    movdqa %xmm0, %xmm3
507; SSE-NEXT:    psrlq $32, %xmm3
508; SSE-NEXT:    pmuludq %xmm2, %xmm3
509; SSE-NEXT:    movdqa %xmm2, %xmm4
510; SSE-NEXT:    psrlq $32, %xmm4
511; SSE-NEXT:    pmuludq %xmm0, %xmm4
512; SSE-NEXT:    paddq %xmm3, %xmm4
513; SSE-NEXT:    psllq $32, %xmm4
514; SSE-NEXT:    pmuludq %xmm2, %xmm0
515; SSE-NEXT:    paddq %xmm4, %xmm0
516; SSE-NEXT:    movdqa %xmm0, %xmm2
517; SSE-NEXT:    psrlq $32, %xmm2
518; SSE-NEXT:    pmuludq %xmm1, %xmm2
519; SSE-NEXT:    movdqa %xmm1, %xmm3
520; SSE-NEXT:    psrlq $32, %xmm3
521; SSE-NEXT:    pmuludq %xmm0, %xmm3
522; SSE-NEXT:    paddq %xmm2, %xmm3
523; SSE-NEXT:    psllq $32, %xmm3
524; SSE-NEXT:    pmuludq %xmm1, %xmm0
525; SSE-NEXT:    paddq %xmm3, %xmm0
526; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
527; SSE-NEXT:    movdqa %xmm0, %xmm2
528; SSE-NEXT:    psrlq $32, %xmm2
529; SSE-NEXT:    pmuludq %xmm1, %xmm2
530; SSE-NEXT:    movdqa %xmm0, %xmm3
531; SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
532; SSE-NEXT:    pmuludq %xmm0, %xmm3
533; SSE-NEXT:    paddq %xmm2, %xmm3
534; SSE-NEXT:    psllq $32, %xmm3
535; SSE-NEXT:    pmuludq %xmm1, %xmm0
536; SSE-NEXT:    paddq %xmm3, %xmm0
537; SSE-NEXT:    movq %xmm0, %rax
538; SSE-NEXT:    retq
539;
540; AVX1-LABEL: test_v16i64:
541; AVX1:       # %bb.0:
542; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
543; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
544; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
545; AVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm5
546; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4
547; AVX1-NEXT:    vpsllq $32, %xmm4, %xmm4
548; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm5
549; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4
550; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm5
551; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm5
552; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm6
553; AVX1-NEXT:    vpmuludq %xmm6, %xmm0, %xmm6
554; AVX1-NEXT:    vpaddq %xmm5, %xmm6, %xmm5
555; AVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
556; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm6
557; AVX1-NEXT:    vpaddq %xmm5, %xmm6, %xmm5
558; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
559; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
560; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
561; AVX1-NEXT:    vpmuludq %xmm3, %xmm6, %xmm6
562; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm7
563; AVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
564; AVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
565; AVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
566; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
567; AVX1-NEXT:    vpaddq %xmm6, %xmm1, %xmm1
568; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
569; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
570; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3
571; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm3
572; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm6
573; AVX1-NEXT:    vpmuludq %xmm6, %xmm0, %xmm6
574; AVX1-NEXT:    vpaddq %xmm3, %xmm6, %xmm3
575; AVX1-NEXT:    vpsllq $32, %xmm3, %xmm3
576; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
577; AVX1-NEXT:    vpaddq %xmm3, %xmm0, %xmm0
578; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
579; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
580; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
581; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
582; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
583; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
584; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
585; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
586; AVX1-NEXT:    vpsrlq $32, %xmm5, %xmm1
587; AVX1-NEXT:    vpmuludq %xmm4, %xmm1, %xmm1
588; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm2
589; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
590; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
591; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
592; AVX1-NEXT:    vpmuludq %xmm4, %xmm5, %xmm2
593; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
594; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm2
595; AVX1-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2
596; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3
597; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm3
598; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
599; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
600; AVX1-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
601; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
602; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
603; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
604; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
605; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
606; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
607; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
608; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
609; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
610; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
611; AVX1-NEXT:    vmovq %xmm0, %rax
612; AVX1-NEXT:    vzeroupper
613; AVX1-NEXT:    retq
614;
615; AVX2-LABEL: test_v16i64:
616; AVX2:       # %bb.0:
617; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm4
618; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
619; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm5
620; AVX2-NEXT:    vpmuludq %ymm5, %ymm1, %ymm5
621; AVX2-NEXT:    vpaddq %ymm4, %ymm5, %ymm4
622; AVX2-NEXT:    vpsllq $32, %ymm4, %ymm4
623; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
624; AVX2-NEXT:    vpaddq %ymm4, %ymm1, %ymm1
625; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm3
626; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm3
627; AVX2-NEXT:    vpsrlq $32, %ymm2, %ymm4
628; AVX2-NEXT:    vpmuludq %ymm4, %ymm0, %ymm4
629; AVX2-NEXT:    vpaddq %ymm3, %ymm4, %ymm3
630; AVX2-NEXT:    vpsllq $32, %ymm3, %ymm3
631; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm0
632; AVX2-NEXT:    vpaddq %ymm3, %ymm0, %ymm0
633; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm2
634; AVX2-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2
635; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm3
636; AVX2-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3
637; AVX2-NEXT:    vpaddq %ymm2, %ymm3, %ymm2
638; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
639; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
640; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
641; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
642; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
643; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
644; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3
645; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
646; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
647; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
648; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
649; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
650; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
651; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
652; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
653; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
654; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
655; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
656; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
657; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
658; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
659; AVX2-NEXT:    vmovq %xmm0, %rax
660; AVX2-NEXT:    vzeroupper
661; AVX2-NEXT:    retq
662;
663; AVX512BW-LABEL: test_v16i64:
664; AVX512BW:       # %bb.0:
665; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm2
666; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
667; AVX512BW-NEXT:    vpsrlq $32, %zmm1, %zmm3
668; AVX512BW-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
669; AVX512BW-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
670; AVX512BW-NEXT:    vpsllq $32, %zmm2, %zmm2
671; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
672; AVX512BW-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
673; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
674; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm2
675; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
676; AVX512BW-NEXT:    vpsrlq $32, %zmm1, %zmm3
677; AVX512BW-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
678; AVX512BW-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
679; AVX512BW-NEXT:    vpsllq $32, %zmm2, %zmm2
680; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
681; AVX512BW-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
682; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
683; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
684; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
685; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3
686; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
687; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
688; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
689; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
690; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
691; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
692; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
693; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
694; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
695; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
696; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
697; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
698; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
699; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
700; AVX512BW-NEXT:    vmovq %xmm0, %rax
701; AVX512BW-NEXT:    vzeroupper
702; AVX512BW-NEXT:    retq
703;
704; AVX512BWVL-LABEL: test_v16i64:
705; AVX512BWVL:       # %bb.0:
706; AVX512BWVL-NEXT:    vpsrlq $32, %zmm0, %zmm2
707; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
708; AVX512BWVL-NEXT:    vpsrlq $32, %zmm1, %zmm3
709; AVX512BWVL-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
710; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
711; AVX512BWVL-NEXT:    vpsllq $32, %zmm2, %zmm2
712; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
713; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
714; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
715; AVX512BWVL-NEXT:    vpsrlq $32, %zmm0, %zmm2
716; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2
717; AVX512BWVL-NEXT:    vpsrlq $32, %zmm1, %zmm3
718; AVX512BWVL-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3
719; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
720; AVX512BWVL-NEXT:    vpsllq $32, %zmm2, %zmm2
721; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
722; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
723; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
724; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
725; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
726; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
727; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
728; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
729; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
730; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
731; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
732; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
733; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
734; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
735; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3
736; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
737; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
738; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
739; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
740; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
741; AVX512BWVL-NEXT:    vmovq %xmm0, %rax
742; AVX512BWVL-NEXT:    vzeroupper
743; AVX512BWVL-NEXT:    retq
744;
745; AVX512DQ-LABEL: test_v16i64:
746; AVX512DQ:       # %bb.0:
747; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
748; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
749; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
750; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
751; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
752; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
753; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
754; AVX512DQ-NEXT:    vmovq %xmm0, %rax
755; AVX512DQ-NEXT:    vzeroupper
756; AVX512DQ-NEXT:    retq
757;
758; AVX512DQVL-LABEL: test_v16i64:
759; AVX512DQVL:       # %bb.0:
760; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
761; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
762; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
763; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
764; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
765; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
766; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
767; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
768; AVX512DQVL-NEXT:    vzeroupper
769; AVX512DQVL-NEXT:    retq
770  %1 = call i64 @llvm.experimental.vector.reduce.mul.v16i64(<16 x i64> %a0)
771  ret i64 %1
772}
773
774;
775; vXi32
776;
777
778define i32 @test_v2i32(<2 x i32> %a0) {
779; SSE2-LABEL: test_v2i32:
780; SSE2:       # %bb.0:
781; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
782; SSE2-NEXT:    pmuludq %xmm0, %xmm1
783; SSE2-NEXT:    movd %xmm1, %eax
784; SSE2-NEXT:    retq
785;
786; SSE41-LABEL: test_v2i32:
787; SSE41:       # %bb.0:
788; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
789; SSE41-NEXT:    pmulld %xmm0, %xmm1
790; SSE41-NEXT:    movd %xmm1, %eax
791; SSE41-NEXT:    retq
792;
793; AVX-LABEL: test_v2i32:
794; AVX:       # %bb.0:
795; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
796; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
797; AVX-NEXT:    vmovd %xmm0, %eax
798; AVX-NEXT:    retq
799;
800; AVX512-LABEL: test_v2i32:
801; AVX512:       # %bb.0:
802; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
803; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
804; AVX512-NEXT:    vmovd %xmm0, %eax
805; AVX512-NEXT:    retq
806  %1 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> %a0)
807  ret i32 %1
808}
809
810define i32 @test_v4i32(<4 x i32> %a0) {
811; SSE2-LABEL: test_v4i32:
812; SSE2:       # %bb.0:
813; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
814; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
815; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
816; SSE2-NEXT:    pmuludq %xmm2, %xmm3
817; SSE2-NEXT:    pmuludq %xmm0, %xmm1
818; SSE2-NEXT:    pmuludq %xmm3, %xmm1
819; SSE2-NEXT:    movd %xmm1, %eax
820; SSE2-NEXT:    retq
821;
822; SSE41-LABEL: test_v4i32:
823; SSE41:       # %bb.0:
824; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
825; SSE41-NEXT:    pmulld %xmm0, %xmm1
826; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
827; SSE41-NEXT:    pmulld %xmm1, %xmm0
828; SSE41-NEXT:    movd %xmm0, %eax
829; SSE41-NEXT:    retq
830;
831; AVX-LABEL: test_v4i32:
832; AVX:       # %bb.0:
833; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
834; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
835; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
836; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
837; AVX-NEXT:    vmovd %xmm0, %eax
838; AVX-NEXT:    retq
839;
840; AVX512-LABEL: test_v4i32:
841; AVX512:       # %bb.0:
842; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
843; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
844; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
845; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
846; AVX512-NEXT:    vmovd %xmm0, %eax
847; AVX512-NEXT:    retq
848  %1 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> %a0)
849  ret i32 %1
850}
851
852define i32 @test_v8i32(<8 x i32> %a0) {
853; SSE2-LABEL: test_v8i32:
854; SSE2:       # %bb.0:
855; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
856; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
857; SSE2-NEXT:    pmuludq %xmm2, %xmm3
858; SSE2-NEXT:    pmuludq %xmm1, %xmm0
859; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
860; SSE2-NEXT:    pmuludq %xmm0, %xmm1
861; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,0,2,2]
862; SSE2-NEXT:    pmuludq %xmm3, %xmm0
863; SSE2-NEXT:    pmuludq %xmm1, %xmm0
864; SSE2-NEXT:    movd %xmm0, %eax
865; SSE2-NEXT:    retq
866;
867; SSE41-LABEL: test_v8i32:
868; SSE41:       # %bb.0:
869; SSE41-NEXT:    pmulld %xmm1, %xmm0
870; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
871; SSE41-NEXT:    pmulld %xmm0, %xmm1
872; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
873; SSE41-NEXT:    pmulld %xmm1, %xmm0
874; SSE41-NEXT:    movd %xmm0, %eax
875; SSE41-NEXT:    retq
876;
877; AVX1-LABEL: test_v8i32:
878; AVX1:       # %bb.0:
879; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
880; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
881; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
882; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
883; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
884; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
885; AVX1-NEXT:    vmovd %xmm0, %eax
886; AVX1-NEXT:    vzeroupper
887; AVX1-NEXT:    retq
888;
889; AVX2-LABEL: test_v8i32:
890; AVX2:       # %bb.0:
891; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
892; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
893; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
894; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
895; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
896; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
897; AVX2-NEXT:    vmovd %xmm0, %eax
898; AVX2-NEXT:    vzeroupper
899; AVX2-NEXT:    retq
900;
901; AVX512-LABEL: test_v8i32:
902; AVX512:       # %bb.0:
903; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
904; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
905; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
906; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
907; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
908; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
909; AVX512-NEXT:    vmovd %xmm0, %eax
910; AVX512-NEXT:    vzeroupper
911; AVX512-NEXT:    retq
912  %1 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> %a0)
913  ret i32 %1
914}
915
916define i32 @test_v16i32(<16 x i32> %a0) {
917; SSE2-LABEL: test_v16i32:
918; SSE2:       # %bb.0:
919; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
920; SSE2-NEXT:    pmuludq %xmm3, %xmm1
921; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
922; SSE2-NEXT:    pmuludq %xmm2, %xmm0
923; SSE2-NEXT:    pmuludq %xmm1, %xmm0
924; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
925; SSE2-NEXT:    pmuludq %xmm4, %xmm1
926; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
927; SSE2-NEXT:    pmuludq %xmm5, %xmm2
928; SSE2-NEXT:    pmuludq %xmm1, %xmm2
929; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
930; SSE2-NEXT:    pmuludq %xmm0, %xmm1
931; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,0,2,2]
932; SSE2-NEXT:    pmuludq %xmm2, %xmm0
933; SSE2-NEXT:    pmuludq %xmm1, %xmm0
934; SSE2-NEXT:    movd %xmm0, %eax
935; SSE2-NEXT:    retq
936;
937; SSE41-LABEL: test_v16i32:
938; SSE41:       # %bb.0:
939; SSE41-NEXT:    pmulld %xmm3, %xmm1
940; SSE41-NEXT:    pmulld %xmm2, %xmm1
941; SSE41-NEXT:    pmulld %xmm0, %xmm1
942; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
943; SSE41-NEXT:    pmulld %xmm1, %xmm0
944; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
945; SSE41-NEXT:    pmulld %xmm0, %xmm1
946; SSE41-NEXT:    movd %xmm1, %eax
947; SSE41-NEXT:    retq
948;
949; AVX1-LABEL: test_v16i32:
950; AVX1:       # %bb.0:
951; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
952; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
953; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
954; AVX1-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
955; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
956; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
957; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
958; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
959; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
960; AVX1-NEXT:    vmovd %xmm0, %eax
961; AVX1-NEXT:    vzeroupper
962; AVX1-NEXT:    retq
963;
964; AVX2-LABEL: test_v16i32:
965; AVX2:       # %bb.0:
966; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
967; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
968; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
969; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
970; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
971; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
972; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
973; AVX2-NEXT:    vmovd %xmm0, %eax
974; AVX2-NEXT:    vzeroupper
975; AVX2-NEXT:    retq
976;
977; AVX512-LABEL: test_v16i32:
978; AVX512:       # %bb.0:
979; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
980; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
981; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
982; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
983; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
984; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
985; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
986; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
987; AVX512-NEXT:    vmovd %xmm0, %eax
988; AVX512-NEXT:    vzeroupper
989; AVX512-NEXT:    retq
990  %1 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> %a0)
991  ret i32 %1
992}
993
994define i32 @test_v32i32(<32 x i32> %a0) {
995; SSE2-LABEL: test_v32i32:
996; SSE2:       # %bb.0:
997; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
998; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[1,1,3,3]
999; SSE2-NEXT:    pmuludq %xmm8, %xmm9
1000; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
1001; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
1002; SSE2-NEXT:    pmuludq %xmm8, %xmm10
1003; SSE2-NEXT:    pmuludq %xmm9, %xmm10
1004; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
1005; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
1006; SSE2-NEXT:    pmuludq %xmm8, %xmm9
1007; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,1,3,3]
1008; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,1,3,3]
1009; SSE2-NEXT:    pmuludq %xmm8, %xmm11
1010; SSE2-NEXT:    pmuludq %xmm9, %xmm11
1011; SSE2-NEXT:    pmuludq %xmm10, %xmm11
1012; SSE2-NEXT:    pmuludq %xmm6, %xmm2
1013; SSE2-NEXT:    pmuludq %xmm4, %xmm0
1014; SSE2-NEXT:    pmuludq %xmm2, %xmm0
1015; SSE2-NEXT:    pmuludq %xmm7, %xmm3
1016; SSE2-NEXT:    pmuludq %xmm5, %xmm1
1017; SSE2-NEXT:    pmuludq %xmm3, %xmm1
1018; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1019; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1020; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1021; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,0,2,2]
1022; SSE2-NEXT:    pmuludq %xmm11, %xmm1
1023; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1024; SSE2-NEXT:    movd %xmm1, %eax
1025; SSE2-NEXT:    retq
1026;
1027; SSE41-LABEL: test_v32i32:
1028; SSE41:       # %bb.0:
1029; SSE41-NEXT:    pmulld %xmm6, %xmm2
1030; SSE41-NEXT:    pmulld %xmm7, %xmm3
1031; SSE41-NEXT:    pmulld %xmm5, %xmm3
1032; SSE41-NEXT:    pmulld %xmm1, %xmm3
1033; SSE41-NEXT:    pmulld %xmm4, %xmm2
1034; SSE41-NEXT:    pmulld %xmm3, %xmm2
1035; SSE41-NEXT:    pmulld %xmm0, %xmm2
1036; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1037; SSE41-NEXT:    pmulld %xmm2, %xmm0
1038; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1039; SSE41-NEXT:    pmulld %xmm0, %xmm1
1040; SSE41-NEXT:    movd %xmm1, %eax
1041; SSE41-NEXT:    retq
1042;
1043; AVX1-LABEL: test_v32i32:
1044; AVX1:       # %bb.0:
1045; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm4
1046; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
1047; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
1048; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm1
1049; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
1050; AVX1-NEXT:    vpmulld %xmm1, %xmm3, %xmm1
1051; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
1052; AVX1-NEXT:    vpmulld %xmm1, %xmm3, %xmm1
1053; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
1054; AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
1055; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1056; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1057; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1058; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1059; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1060; AVX1-NEXT:    vmovd %xmm0, %eax
1061; AVX1-NEXT:    vzeroupper
1062; AVX1-NEXT:    retq
1063;
1064; AVX2-LABEL: test_v32i32:
1065; AVX2:       # %bb.0:
1066; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm1
1067; AVX2-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
1068; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
1069; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1070; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1071; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1072; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1073; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1074; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1075; AVX2-NEXT:    vmovd %xmm0, %eax
1076; AVX2-NEXT:    vzeroupper
1077; AVX2-NEXT:    retq
1078;
1079; AVX512-LABEL: test_v32i32:
1080; AVX512:       # %bb.0:
1081; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
1082; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1083; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
1084; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
1085; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1086; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1087; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1088; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1089; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
1090; AVX512-NEXT:    vmovd %xmm0, %eax
1091; AVX512-NEXT:    vzeroupper
1092; AVX512-NEXT:    retq
1093  %1 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> %a0)
1094  ret i32 %1
1095}
1096
1097;
1098; vXi16
1099;
1100
1101define i16 @test_v2i16(<2 x i16> %a0) {
1102; SSE-LABEL: test_v2i16:
1103; SSE:       # %bb.0:
1104; SSE-NEXT:    movdqa %xmm0, %xmm1
1105; SSE-NEXT:    psrld $16, %xmm1
1106; SSE-NEXT:    pmullw %xmm0, %xmm1
1107; SSE-NEXT:    movd %xmm1, %eax
1108; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1109; SSE-NEXT:    retq
1110;
1111; AVX-LABEL: test_v2i16:
1112; AVX:       # %bb.0:
1113; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
1114; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1115; AVX-NEXT:    vmovd %xmm0, %eax
1116; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
1117; AVX-NEXT:    retq
1118;
1119; AVX512-LABEL: test_v2i16:
1120; AVX512:       # %bb.0:
1121; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
1122; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1123; AVX512-NEXT:    vmovd %xmm0, %eax
1124; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
1125; AVX512-NEXT:    retq
1126  %1 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> %a0)
1127  ret i16 %1
1128}
1129
1130define i16 @test_v4i16(<4 x i16> %a0) {
1131; SSE-LABEL: test_v4i16:
1132; SSE:       # %bb.0:
1133; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1134; SSE-NEXT:    pmullw %xmm0, %xmm1
1135; SSE-NEXT:    movdqa %xmm1, %xmm0
1136; SSE-NEXT:    psrld $16, %xmm0
1137; SSE-NEXT:    pmullw %xmm1, %xmm0
1138; SSE-NEXT:    movd %xmm0, %eax
1139; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1140; SSE-NEXT:    retq
1141;
1142; AVX-LABEL: test_v4i16:
1143; AVX:       # %bb.0:
1144; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1145; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1146; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
1147; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1148; AVX-NEXT:    vmovd %xmm0, %eax
1149; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
1150; AVX-NEXT:    retq
1151;
1152; AVX512-LABEL: test_v4i16:
1153; AVX512:       # %bb.0:
1154; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1155; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1156; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
1157; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1158; AVX512-NEXT:    vmovd %xmm0, %eax
1159; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
1160; AVX512-NEXT:    retq
1161  %1 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> %a0)
1162  ret i16 %1
1163}
1164
1165define i16 @test_v8i16(<8 x i16> %a0) {
1166; SSE-LABEL: test_v8i16:
1167; SSE:       # %bb.0:
1168; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1169; SSE-NEXT:    pmullw %xmm0, %xmm1
1170; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1171; SSE-NEXT:    pmullw %xmm1, %xmm0
1172; SSE-NEXT:    movdqa %xmm0, %xmm1
1173; SSE-NEXT:    psrld $16, %xmm1
1174; SSE-NEXT:    pmullw %xmm0, %xmm1
1175; SSE-NEXT:    movd %xmm1, %eax
1176; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1177; SSE-NEXT:    retq
1178;
1179; AVX-LABEL: test_v8i16:
1180; AVX:       # %bb.0:
1181; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1182; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1183; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1184; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1185; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
1186; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1187; AVX-NEXT:    vmovd %xmm0, %eax
1188; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
1189; AVX-NEXT:    retq
1190;
1191; AVX512-LABEL: test_v8i16:
1192; AVX512:       # %bb.0:
1193; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1194; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1195; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1196; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1197; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
1198; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1199; AVX512-NEXT:    vmovd %xmm0, %eax
1200; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
1201; AVX512-NEXT:    retq
1202  %1 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> %a0)
1203  ret i16 %1
1204}
1205
1206define i16 @test_v16i16(<16 x i16> %a0) {
1207; SSE-LABEL: test_v16i16:
1208; SSE:       # %bb.0:
1209; SSE-NEXT:    pmullw %xmm1, %xmm0
1210; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1211; SSE-NEXT:    pmullw %xmm0, %xmm1
1212; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1213; SSE-NEXT:    pmullw %xmm1, %xmm0
1214; SSE-NEXT:    movdqa %xmm0, %xmm1
1215; SSE-NEXT:    psrld $16, %xmm1
1216; SSE-NEXT:    pmullw %xmm0, %xmm1
1217; SSE-NEXT:    movd %xmm1, %eax
1218; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1219; SSE-NEXT:    retq
1220;
1221; AVX1-LABEL: test_v16i16:
1222; AVX1:       # %bb.0:
1223; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1224; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1225; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1226; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1227; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1228; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1229; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
1230; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1231; AVX1-NEXT:    vmovd %xmm0, %eax
1232; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
1233; AVX1-NEXT:    vzeroupper
1234; AVX1-NEXT:    retq
1235;
1236; AVX2-LABEL: test_v16i16:
1237; AVX2:       # %bb.0:
1238; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1239; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1240; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1241; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1242; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1243; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1244; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
1245; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1246; AVX2-NEXT:    vmovd %xmm0, %eax
1247; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
1248; AVX2-NEXT:    vzeroupper
1249; AVX2-NEXT:    retq
1250;
1251; AVX512-LABEL: test_v16i16:
1252; AVX512:       # %bb.0:
1253; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
1254; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1255; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1256; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1257; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1258; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1259; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
1260; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1261; AVX512-NEXT:    vmovd %xmm0, %eax
1262; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
1263; AVX512-NEXT:    vzeroupper
1264; AVX512-NEXT:    retq
1265  %1 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> %a0)
1266  ret i16 %1
1267}
1268
1269define i16 @test_v32i16(<32 x i16> %a0) {
1270; SSE-LABEL: test_v32i16:
1271; SSE:       # %bb.0:
1272; SSE-NEXT:    pmullw %xmm3, %xmm1
1273; SSE-NEXT:    pmullw %xmm2, %xmm1
1274; SSE-NEXT:    pmullw %xmm0, %xmm1
1275; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1276; SSE-NEXT:    pmullw %xmm1, %xmm0
1277; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1278; SSE-NEXT:    pmullw %xmm0, %xmm1
1279; SSE-NEXT:    movdqa %xmm1, %xmm0
1280; SSE-NEXT:    psrld $16, %xmm0
1281; SSE-NEXT:    pmullw %xmm1, %xmm0
1282; SSE-NEXT:    movd %xmm0, %eax
1283; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1284; SSE-NEXT:    retq
1285;
1286; AVX1-LABEL: test_v32i16:
1287; AVX1:       # %bb.0:
1288; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1289; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
1290; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1291; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
1292; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1293; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1294; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1295; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1296; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1297; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
1298; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1299; AVX1-NEXT:    vmovd %xmm0, %eax
1300; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
1301; AVX1-NEXT:    vzeroupper
1302; AVX1-NEXT:    retq
1303;
1304; AVX2-LABEL: test_v32i16:
1305; AVX2:       # %bb.0:
1306; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1307; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1308; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1309; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1310; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1311; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1312; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1313; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
1314; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1315; AVX2-NEXT:    vmovd %xmm0, %eax
1316; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
1317; AVX2-NEXT:    vzeroupper
1318; AVX2-NEXT:    retq
1319;
1320; AVX512BW-LABEL: test_v32i16:
1321; AVX512BW:       # %bb.0:
1322; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1323; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1324; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
1325; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1326; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1327; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1328; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1329; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1330; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
1331; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1332; AVX512BW-NEXT:    vmovd %xmm0, %eax
1333; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
1334; AVX512BW-NEXT:    vzeroupper
1335; AVX512BW-NEXT:    retq
1336;
1337; AVX512BWVL-LABEL: test_v32i16:
1338; AVX512BWVL:       # %bb.0:
1339; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1340; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1341; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
1342; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1343; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1344; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1345; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1346; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1347; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
1348; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1349; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
1350; AVX512BWVL-NEXT:    # kill: def $ax killed $ax killed $eax
1351; AVX512BWVL-NEXT:    vzeroupper
1352; AVX512BWVL-NEXT:    retq
1353;
1354; AVX512DQ-LABEL: test_v32i16:
1355; AVX512DQ:       # %bb.0:
1356; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1357; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1358; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
1359; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1360; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1361; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1362; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1363; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1364; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
1365; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1366; AVX512DQ-NEXT:    vmovd %xmm0, %eax
1367; AVX512DQ-NEXT:    # kill: def $ax killed $ax killed $eax
1368; AVX512DQ-NEXT:    vzeroupper
1369; AVX512DQ-NEXT:    retq
1370;
1371; AVX512DQVL-LABEL: test_v32i16:
1372; AVX512DQVL:       # %bb.0:
1373; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1374; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1375; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
1376; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1377; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1378; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1379; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1380; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1381; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
1382; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1383; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
1384; AVX512DQVL-NEXT:    # kill: def $ax killed $ax killed $eax
1385; AVX512DQVL-NEXT:    vzeroupper
1386; AVX512DQVL-NEXT:    retq
1387  %1 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> %a0)
1388  ret i16 %1
1389}
1390
1391define i16 @test_v64i16(<64 x i16> %a0) {
1392; SSE-LABEL: test_v64i16:
1393; SSE:       # %bb.0:
1394; SSE-NEXT:    pmullw %xmm6, %xmm2
1395; SSE-NEXT:    pmullw %xmm7, %xmm3
1396; SSE-NEXT:    pmullw %xmm5, %xmm3
1397; SSE-NEXT:    pmullw %xmm1, %xmm3
1398; SSE-NEXT:    pmullw %xmm4, %xmm2
1399; SSE-NEXT:    pmullw %xmm3, %xmm2
1400; SSE-NEXT:    pmullw %xmm0, %xmm2
1401; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1402; SSE-NEXT:    pmullw %xmm2, %xmm0
1403; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1404; SSE-NEXT:    pmullw %xmm0, %xmm1
1405; SSE-NEXT:    movdqa %xmm1, %xmm0
1406; SSE-NEXT:    psrld $16, %xmm0
1407; SSE-NEXT:    pmullw %xmm1, %xmm0
1408; SSE-NEXT:    movd %xmm0, %eax
1409; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
1410; SSE-NEXT:    retq
1411;
1412; AVX1-LABEL: test_v64i16:
1413; AVX1:       # %bb.0:
1414; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm4
1415; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
1416; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
1417; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
1418; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
1419; AVX1-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
1420; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
1421; AVX1-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
1422; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
1423; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
1424; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1425; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1426; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1427; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1428; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1429; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
1430; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1431; AVX1-NEXT:    vmovd %xmm0, %eax
1432; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
1433; AVX1-NEXT:    vzeroupper
1434; AVX1-NEXT:    retq
1435;
1436; AVX2-LABEL: test_v64i16:
1437; AVX2:       # %bb.0:
1438; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1439; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
1440; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1441; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1442; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1443; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1444; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1445; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1446; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1447; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
1448; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1449; AVX2-NEXT:    vmovd %xmm0, %eax
1450; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
1451; AVX2-NEXT:    vzeroupper
1452; AVX2-NEXT:    retq
1453;
1454; AVX512BW-LABEL: test_v64i16:
1455; AVX512BW:       # %bb.0:
1456; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1457; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1458; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1459; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
1460; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1461; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1462; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1463; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1464; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1465; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
1466; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1467; AVX512BW-NEXT:    vmovd %xmm0, %eax
1468; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
1469; AVX512BW-NEXT:    vzeroupper
1470; AVX512BW-NEXT:    retq
1471;
1472; AVX512BWVL-LABEL: test_v64i16:
1473; AVX512BWVL:       # %bb.0:
1474; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1475; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1476; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1477; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
1478; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1479; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1480; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1481; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1482; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1483; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
1484; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1485; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
1486; AVX512BWVL-NEXT:    # kill: def $ax killed $ax killed $eax
1487; AVX512BWVL-NEXT:    vzeroupper
1488; AVX512BWVL-NEXT:    retq
1489;
1490; AVX512DQ-LABEL: test_v64i16:
1491; AVX512DQ:       # %bb.0:
1492; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1493; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
1494; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1495; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
1496; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1497; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1498; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1499; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1500; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1501; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
1502; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1503; AVX512DQ-NEXT:    vmovd %xmm0, %eax
1504; AVX512DQ-NEXT:    # kill: def $ax killed $ax killed $eax
1505; AVX512DQ-NEXT:    vzeroupper
1506; AVX512DQ-NEXT:    retq
1507;
1508; AVX512DQVL-LABEL: test_v64i16:
1509; AVX512DQVL:       # %bb.0:
1510; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1511; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
1512; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1513; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
1514; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1515; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1516; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1517; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1518; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1519; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
1520; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1521; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
1522; AVX512DQVL-NEXT:    # kill: def $ax killed $ax killed $eax
1523; AVX512DQVL-NEXT:    vzeroupper
1524; AVX512DQVL-NEXT:    retq
1525  %1 = call i16 @llvm.experimental.vector.reduce.mul.v64i16(<64 x i16> %a0)
1526  ret i16 %1
1527}
1528
1529;
1530; vXi8
1531;
1532
1533define i8 @test_v2i8(<2 x i8> %a0) {
1534; SSE-LABEL: test_v2i8:
1535; SSE:       # %bb.0:
1536; SSE-NEXT:    movdqa %xmm0, %xmm1
1537; SSE-NEXT:    psrlw $8, %xmm1
1538; SSE-NEXT:    pmullw %xmm0, %xmm1
1539; SSE-NEXT:    movd %xmm1, %eax
1540; SSE-NEXT:    # kill: def $al killed $al killed $eax
1541; SSE-NEXT:    retq
1542;
1543; AVX-LABEL: test_v2i8:
1544; AVX:       # %bb.0:
1545; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
1546; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1547; AVX-NEXT:    vmovd %xmm0, %eax
1548; AVX-NEXT:    # kill: def $al killed $al killed $eax
1549; AVX-NEXT:    retq
1550;
1551; AVX512-LABEL: test_v2i8:
1552; AVX512:       # %bb.0:
1553; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
1554; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1555; AVX512-NEXT:    vmovd %xmm0, %eax
1556; AVX512-NEXT:    # kill: def $al killed $al killed $eax
1557; AVX512-NEXT:    retq
1558  %1 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> %a0)
1559  ret i8 %1
1560}
1561
1562define i8 @test_v4i8(<4 x i8> %a0) {
1563; SSE2-LABEL: test_v4i8:
1564; SSE2:       # %bb.0:
1565; SSE2-NEXT:    movdqa %xmm0, %xmm1
1566; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1567; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1568; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1569; SSE2-NEXT:    pmullw %xmm1, %xmm0
1570; SSE2-NEXT:    movdqa %xmm0, %xmm1
1571; SSE2-NEXT:    psrld $16, %xmm1
1572; SSE2-NEXT:    pmullw %xmm0, %xmm1
1573; SSE2-NEXT:    movd %xmm1, %eax
1574; SSE2-NEXT:    # kill: def $al killed $al killed $eax
1575; SSE2-NEXT:    retq
1576;
1577; SSE41-LABEL: test_v4i8:
1578; SSE41:       # %bb.0:
1579; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1580; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1581; SSE41-NEXT:    pmullw %xmm0, %xmm1
1582; SSE41-NEXT:    movdqa %xmm1, %xmm0
1583; SSE41-NEXT:    psrld $16, %xmm0
1584; SSE41-NEXT:    pmullw %xmm1, %xmm0
1585; SSE41-NEXT:    movd %xmm0, %eax
1586; SSE41-NEXT:    # kill: def $al killed $al killed $eax
1587; SSE41-NEXT:    retq
1588;
1589; AVX-LABEL: test_v4i8:
1590; AVX:       # %bb.0:
1591; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1592; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1593; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1594; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
1595; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1596; AVX-NEXT:    vmovd %xmm0, %eax
1597; AVX-NEXT:    # kill: def $al killed $al killed $eax
1598; AVX-NEXT:    retq
1599;
1600; AVX512BW-LABEL: test_v4i8:
1601; AVX512BW:       # %bb.0:
1602; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1603; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1604; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1605; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
1606; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1607; AVX512BW-NEXT:    vmovd %xmm0, %eax
1608; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
1609; AVX512BW-NEXT:    retq
1610;
1611; AVX512VL-LABEL: test_v4i8:
1612; AVX512VL:       # %bb.0:
1613; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1614; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1615; AVX512VL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1616; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1617; AVX512VL-NEXT:    vpsrld $16, %xmm0, %xmm1
1618; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1619; AVX512VL-NEXT:    vmovd %xmm0, %eax
1620; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
1621; AVX512VL-NEXT:    retq
1622;
1623; AVX512DQ-LABEL: test_v4i8:
1624; AVX512DQ:       # %bb.0:
1625; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1626; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1627; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1628; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
1629; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1630; AVX512DQ-NEXT:    vmovd %xmm0, %eax
1631; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
1632; AVX512DQ-NEXT:    retq
1633  %1 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> %a0)
1634  ret i8 %1
1635}
1636
1637define i8 @test_v8i8(<8 x i8> %a0) {
1638; SSE2-LABEL: test_v8i8:
1639; SSE2:       # %bb.0:
1640; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,2,3,3]
1641; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1642; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1643; SSE2-NEXT:    pmullw %xmm0, %xmm1
1644; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,2,3,0]
1645; SSE2-NEXT:    pmullw %xmm1, %xmm0
1646; SSE2-NEXT:    movdqa %xmm0, %xmm1
1647; SSE2-NEXT:    psrld $16, %xmm1
1648; SSE2-NEXT:    pmullw %xmm0, %xmm1
1649; SSE2-NEXT:    movd %xmm1, %eax
1650; SSE2-NEXT:    # kill: def $al killed $al killed $eax
1651; SSE2-NEXT:    retq
1652;
1653; SSE41-LABEL: test_v8i8:
1654; SSE41:       # %bb.0:
1655; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1656; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1657; SSE41-NEXT:    pmullw %xmm0, %xmm1
1658; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1659; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1660; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1661; SSE41-NEXT:    pmullw %xmm0, %xmm1
1662; SSE41-NEXT:    movdqa %xmm1, %xmm0
1663; SSE41-NEXT:    psrld $16, %xmm0
1664; SSE41-NEXT:    pmullw %xmm1, %xmm0
1665; SSE41-NEXT:    movd %xmm0, %eax
1666; SSE41-NEXT:    # kill: def $al killed $al killed $eax
1667; SSE41-NEXT:    retq
1668;
1669; AVX-LABEL: test_v8i8:
1670; AVX:       # %bb.0:
1671; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1672; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1673; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1674; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1675; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1676; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
1677; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1678; AVX-NEXT:    vmovd %xmm0, %eax
1679; AVX-NEXT:    # kill: def $al killed $al killed $eax
1680; AVX-NEXT:    retq
1681;
1682; AVX512BW-LABEL: test_v8i8:
1683; AVX512BW:       # %bb.0:
1684; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1685; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1686; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1687; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1688; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1689; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
1690; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1691; AVX512BW-NEXT:    vmovd %xmm0, %eax
1692; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
1693; AVX512BW-NEXT:    retq
1694;
1695; AVX512VL-LABEL: test_v8i8:
1696; AVX512VL:       # %bb.0:
1697; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1698; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1699; AVX512VL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1700; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1701; AVX512VL-NEXT:    vpalignr {{.*#+}} xmm1 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3]
1702; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1703; AVX512VL-NEXT:    vpsrld $16, %xmm0, %xmm1
1704; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1705; AVX512VL-NEXT:    vmovd %xmm0, %eax
1706; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
1707; AVX512VL-NEXT:    retq
1708;
1709; AVX512DQ-LABEL: test_v8i8:
1710; AVX512DQ:       # %bb.0:
1711; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1712; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1713; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1714; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1715; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1716; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
1717; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1718; AVX512DQ-NEXT:    vmovd %xmm0, %eax
1719; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
1720; AVX512DQ-NEXT:    retq
1721  %1 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> %a0)
1722  ret i8 %1
1723}
1724
1725define i8 @test_v16i8(<16 x i8> %a0) {
1726; SSE2-LABEL: test_v16i8:
1727; SSE2:       # %bb.0:
1728; SSE2-NEXT:    movdqa %xmm0, %xmm1
1729; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1730; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1731; SSE2-NEXT:    pmullw %xmm1, %xmm0
1732; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1733; SSE2-NEXT:    movdqa %xmm0, %xmm2
1734; SSE2-NEXT:    pand %xmm1, %xmm2
1735; SSE2-NEXT:    pxor %xmm3, %xmm3
1736; SSE2-NEXT:    packuswb %xmm3, %xmm2
1737; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
1738; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1739; SSE2-NEXT:    pmullw %xmm0, %xmm2
1740; SSE2-NEXT:    movdqa %xmm2, %xmm0
1741; SSE2-NEXT:    pand %xmm1, %xmm0
1742; SSE2-NEXT:    packuswb %xmm3, %xmm0
1743; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1744; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1745; SSE2-NEXT:    pmullw %xmm2, %xmm0
1746; SSE2-NEXT:    pand %xmm1, %xmm0
1747; SSE2-NEXT:    packuswb %xmm3, %xmm0
1748; SSE2-NEXT:    movdqa %xmm0, %xmm1
1749; SSE2-NEXT:    psrlw $8, %xmm1
1750; SSE2-NEXT:    pmullw %xmm0, %xmm1
1751; SSE2-NEXT:    movd %xmm1, %eax
1752; SSE2-NEXT:    # kill: def $al killed $al killed $eax
1753; SSE2-NEXT:    retq
1754;
1755; SSE41-LABEL: test_v16i8:
1756; SSE41:       # %bb.0:
1757; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1758; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1759; SSE41-NEXT:    pmullw %xmm1, %xmm0
1760; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1761; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1762; SSE41-NEXT:    pand %xmm1, %xmm0
1763; SSE41-NEXT:    pxor %xmm3, %xmm3
1764; SSE41-NEXT:    packuswb %xmm3, %xmm0
1765; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1766; SSE41-NEXT:    pmullw %xmm2, %xmm0
1767; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
1768; SSE41-NEXT:    pand %xmm1, %xmm0
1769; SSE41-NEXT:    packuswb %xmm3, %xmm0
1770; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1771; SSE41-NEXT:    pmullw %xmm2, %xmm0
1772; SSE41-NEXT:    pand %xmm1, %xmm0
1773; SSE41-NEXT:    packuswb %xmm3, %xmm0
1774; SSE41-NEXT:    movdqa %xmm0, %xmm1
1775; SSE41-NEXT:    psrlw $8, %xmm1
1776; SSE41-NEXT:    pmullw %xmm0, %xmm1
1777; SSE41-NEXT:    movd %xmm1, %eax
1778; SSE41-NEXT:    # kill: def $al killed $al killed $eax
1779; SSE41-NEXT:    retq
1780;
1781; AVX1-LABEL: test_v16i8:
1782; AVX1:       # %bb.0:
1783; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1784; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1785; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1786; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1787; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1788; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1789; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1790; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1791; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1792; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1793; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
1794; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1795; AVX1-NEXT:    vmovd %xmm0, %eax
1796; AVX1-NEXT:    # kill: def $al killed $al killed $eax
1797; AVX1-NEXT:    retq
1798;
1799; AVX2-LABEL: test_v16i8:
1800; AVX2:       # %bb.0:
1801; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1802; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1803; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1804; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
1805; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1806; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1807; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1808; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1809; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[2],zero,xmm0[4],zero,xmm0[6],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
1810; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1811; AVX2-NEXT:    vmovd %xmm0, %eax
1812; AVX2-NEXT:    # kill: def $al killed $al killed $eax
1813; AVX2-NEXT:    retq
1814;
1815; AVX512BW-LABEL: test_v16i8:
1816; AVX512BW:       # %bb.0:
1817; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1818; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1819; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1820; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1821; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
1822; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1823; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1824; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1825; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
1826; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
1827; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1828; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1829; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
1830; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
1831; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1832; AVX512BW-NEXT:    vmovd %xmm0, %eax
1833; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
1834; AVX512BW-NEXT:    vzeroupper
1835; AVX512BW-NEXT:    retq
1836;
1837; AVX512BWVL-LABEL: test_v16i8:
1838; AVX512BWVL:       # %bb.0:
1839; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1840; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1841; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1842; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1843; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
1844; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1845; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1846; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1847; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
1848; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
1849; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1850; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1851; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
1852; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
1853; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1854; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
1855; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
1856; AVX512BWVL-NEXT:    vzeroupper
1857; AVX512BWVL-NEXT:    retq
1858;
1859; AVX512DQ-LABEL: test_v16i8:
1860; AVX512DQ:       # %bb.0:
1861; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1862; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1863; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1864; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1865; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1866; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
1867; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1868; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1869; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1870; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1871; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
1872; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
1873; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1874; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1875; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1876; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
1877; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
1878; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1879; AVX512DQ-NEXT:    vmovd %xmm0, %eax
1880; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
1881; AVX512DQ-NEXT:    vzeroupper
1882; AVX512DQ-NEXT:    retq
1883;
1884; AVX512DQVL-LABEL: test_v16i8:
1885; AVX512DQVL:       # %bb.0:
1886; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1887; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1888; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1889; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1890; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1891; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
1892; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1893; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1894; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1895; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1896; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
1897; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
1898; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1899; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1900; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1901; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
1902; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
1903; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1904; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
1905; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
1906; AVX512DQVL-NEXT:    vzeroupper
1907; AVX512DQVL-NEXT:    retq
1908  %1 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> %a0)
1909  ret i8 %1
1910}
1911
1912define i8 @test_v32i8(<32 x i8> %a0) {
1913; SSE2-LABEL: test_v32i8:
1914; SSE2:       # %bb.0:
1915; SSE2-NEXT:    movdqa %xmm1, %xmm2
1916; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1917; SSE2-NEXT:    movdqa %xmm0, %xmm3
1918; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1919; SSE2-NEXT:    pmullw %xmm2, %xmm3
1920; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1921; SSE2-NEXT:    pmullw %xmm3, %xmm1
1922; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1923; SSE2-NEXT:    pmullw %xmm1, %xmm0
1924; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1925; SSE2-NEXT:    movdqa %xmm0, %xmm2
1926; SSE2-NEXT:    pand %xmm1, %xmm2
1927; SSE2-NEXT:    pxor %xmm3, %xmm3
1928; SSE2-NEXT:    packuswb %xmm3, %xmm2
1929; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
1930; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1931; SSE2-NEXT:    pmullw %xmm0, %xmm2
1932; SSE2-NEXT:    movdqa %xmm2, %xmm0
1933; SSE2-NEXT:    pand %xmm1, %xmm0
1934; SSE2-NEXT:    packuswb %xmm3, %xmm0
1935; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1936; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1937; SSE2-NEXT:    pmullw %xmm2, %xmm0
1938; SSE2-NEXT:    pand %xmm1, %xmm0
1939; SSE2-NEXT:    packuswb %xmm3, %xmm0
1940; SSE2-NEXT:    movdqa %xmm0, %xmm1
1941; SSE2-NEXT:    psrlw $8, %xmm1
1942; SSE2-NEXT:    pmullw %xmm0, %xmm1
1943; SSE2-NEXT:    movd %xmm1, %eax
1944; SSE2-NEXT:    # kill: def $al killed $al killed $eax
1945; SSE2-NEXT:    retq
1946;
1947; SSE41-LABEL: test_v32i8:
1948; SSE41:       # %bb.0:
1949; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1950; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1951; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1952; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1953; SSE41-NEXT:    pmullw %xmm1, %xmm0
1954; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1955; SSE41-NEXT:    movdqa %xmm0, %xmm4
1956; SSE41-NEXT:    pand %xmm1, %xmm4
1957; SSE41-NEXT:    pmullw %xmm2, %xmm3
1958; SSE41-NEXT:    pand %xmm1, %xmm3
1959; SSE41-NEXT:    packuswb %xmm4, %xmm3
1960; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1961; SSE41-NEXT:    pmullw %xmm0, %xmm2
1962; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1963; SSE41-NEXT:    pand %xmm1, %xmm2
1964; SSE41-NEXT:    pxor %xmm3, %xmm3
1965; SSE41-NEXT:    packuswb %xmm3, %xmm2
1966; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1967; SSE41-NEXT:    pmullw %xmm0, %xmm2
1968; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
1969; SSE41-NEXT:    pand %xmm1, %xmm2
1970; SSE41-NEXT:    packuswb %xmm3, %xmm2
1971; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1972; SSE41-NEXT:    pmullw %xmm0, %xmm2
1973; SSE41-NEXT:    pand %xmm1, %xmm2
1974; SSE41-NEXT:    packuswb %xmm3, %xmm2
1975; SSE41-NEXT:    movdqa %xmm2, %xmm0
1976; SSE41-NEXT:    psrlw $8, %xmm0
1977; SSE41-NEXT:    pmullw %xmm2, %xmm0
1978; SSE41-NEXT:    movd %xmm0, %eax
1979; SSE41-NEXT:    # kill: def $al killed $al killed $eax
1980; SSE41-NEXT:    retq
1981;
1982; AVX1-LABEL: test_v32i8:
1983; AVX1:       # %bb.0:
1984; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1985; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1986; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1987; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1988; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1989; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
1990; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1991; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1992; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1993; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1994; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1995; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1996; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1997; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1998; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1999; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
2000; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2001; AVX1-NEXT:    vmovd %xmm0, %eax
2002; AVX1-NEXT:    # kill: def $al killed $al killed $eax
2003; AVX1-NEXT:    vzeroupper
2004; AVX1-NEXT:    retq
2005;
2006; AVX2-LABEL: test_v32i8:
2007; AVX2:       # %bb.0:
2008; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
2009; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2010; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2011; AVX2-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2012; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2013; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
2014; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2015; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2016; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2017; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2018; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm1
2019; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2020; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2021; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2022; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2023; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2024; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm1
2025; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2026; AVX2-NEXT:    vpsrld $16, %xmm1, %xmm1
2027; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2028; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2029; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
2030; AVX2-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
2031; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
2032; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2033; AVX2-NEXT:    vmovd %xmm0, %eax
2034; AVX2-NEXT:    # kill: def $al killed $al killed $eax
2035; AVX2-NEXT:    vzeroupper
2036; AVX2-NEXT:    retq
2037;
2038; AVX512BW-LABEL: test_v32i8:
2039; AVX512BW:       # %bb.0:
2040; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
2041; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2042; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2043; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2044; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
2045; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
2046; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2047; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2048; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
2049; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2050; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2051; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2052; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
2053; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
2054; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2055; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2056; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
2057; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
2058; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2059; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2060; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
2061; AVX512BW-NEXT:    vmovd %xmm0, %eax
2062; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
2063; AVX512BW-NEXT:    vzeroupper
2064; AVX512BW-NEXT:    retq
2065;
2066; AVX512BWVL-LABEL: test_v32i8:
2067; AVX512BWVL:       # %bb.0:
2068; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
2069; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2070; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2071; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2072; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
2073; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
2074; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2075; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2076; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
2077; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2078; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2079; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2080; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
2081; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
2082; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2083; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2084; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
2085; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
2086; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2087; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2088; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
2089; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
2090; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
2091; AVX512BWVL-NEXT:    vzeroupper
2092; AVX512BWVL-NEXT:    retq
2093;
2094; AVX512DQ-LABEL: test_v32i8:
2095; AVX512DQ:       # %bb.0:
2096; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
2097; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2098; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2099; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2100; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2101; AVX512DQ-NEXT:    vpand %xmm3, %xmm2, %xmm2
2102; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2103; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2104; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2105; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2106; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm1
2107; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2108; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2109; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2110; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2111; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2112; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm1
2113; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2114; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
2115; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2116; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2117; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm0
2118; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
2119; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
2120; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2121; AVX512DQ-NEXT:    vmovd %xmm0, %eax
2122; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
2123; AVX512DQ-NEXT:    vzeroupper
2124; AVX512DQ-NEXT:    retq
2125;
2126; AVX512DQVL-LABEL: test_v32i8:
2127; AVX512DQVL:       # %bb.0:
2128; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
2129; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2130; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2131; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2132; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2133; AVX512DQVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
2134; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2135; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2136; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2137; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2138; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
2139; AVX512DQVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2140; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2141; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2142; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2143; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2144; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
2145; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2146; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
2147; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2148; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2149; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm0
2150; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
2151; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
2152; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2153; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
2154; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
2155; AVX512DQVL-NEXT:    vzeroupper
2156; AVX512DQVL-NEXT:    retq
2157  %1 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> %a0)
2158  ret i8 %1
2159}
2160
2161define i8 @test_v64i8(<64 x i8> %a0) {
2162; SSE2-LABEL: test_v64i8:
2163; SSE2:       # %bb.0:
2164; SSE2-NEXT:    movdqa %xmm3, %xmm4
2165; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2166; SSE2-NEXT:    movdqa %xmm1, %xmm5
2167; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2168; SSE2-NEXT:    pmullw %xmm4, %xmm5
2169; SSE2-NEXT:    movdqa %xmm2, %xmm4
2170; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2171; SSE2-NEXT:    pmullw %xmm5, %xmm4
2172; SSE2-NEXT:    movdqa %xmm0, %xmm5
2173; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2174; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2175; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2176; SSE2-NEXT:    pmullw %xmm3, %xmm1
2177; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2178; SSE2-NEXT:    pmullw %xmm1, %xmm2
2179; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2180; SSE2-NEXT:    pmullw %xmm2, %xmm0
2181; SSE2-NEXT:    pmullw %xmm4, %xmm0
2182; SSE2-NEXT:    pmullw %xmm5, %xmm0
2183; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2184; SSE2-NEXT:    movdqa %xmm0, %xmm2
2185; SSE2-NEXT:    pand %xmm1, %xmm2
2186; SSE2-NEXT:    pxor %xmm3, %xmm3
2187; SSE2-NEXT:    packuswb %xmm3, %xmm2
2188; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
2189; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2190; SSE2-NEXT:    pmullw %xmm0, %xmm2
2191; SSE2-NEXT:    movdqa %xmm2, %xmm0
2192; SSE2-NEXT:    pand %xmm1, %xmm0
2193; SSE2-NEXT:    packuswb %xmm3, %xmm0
2194; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2195; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2196; SSE2-NEXT:    pmullw %xmm2, %xmm0
2197; SSE2-NEXT:    pand %xmm1, %xmm0
2198; SSE2-NEXT:    packuswb %xmm3, %xmm0
2199; SSE2-NEXT:    movdqa %xmm0, %xmm1
2200; SSE2-NEXT:    psrlw $8, %xmm1
2201; SSE2-NEXT:    pmullw %xmm0, %xmm1
2202; SSE2-NEXT:    movd %xmm1, %eax
2203; SSE2-NEXT:    # kill: def $al killed $al killed $eax
2204; SSE2-NEXT:    retq
2205;
2206; SSE41-LABEL: test_v64i8:
2207; SSE41:       # %bb.0:
2208; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2209; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2210; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2211; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2212; SSE41-NEXT:    pmullw %xmm3, %xmm1
2213; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2214; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2215; SSE41-NEXT:    pmullw %xmm1, %xmm2
2216; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2217; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2218; SSE41-NEXT:    pmullw %xmm2, %xmm0
2219; SSE41-NEXT:    pmullw %xmm3, %xmm1
2220; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2221; SSE41-NEXT:    pshufb %xmm2, %xmm1
2222; SSE41-NEXT:    pmullw %xmm4, %xmm5
2223; SSE41-NEXT:    pshufb %xmm2, %xmm5
2224; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
2225; SSE41-NEXT:    movdqa %xmm0, %xmm3
2226; SSE41-NEXT:    pand %xmm2, %xmm3
2227; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2228; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2229; SSE41-NEXT:    pmullw %xmm4, %xmm1
2230; SSE41-NEXT:    pand %xmm2, %xmm1
2231; SSE41-NEXT:    packuswb %xmm3, %xmm1
2232; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2233; SSE41-NEXT:    pmullw %xmm0, %xmm1
2234; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
2235; SSE41-NEXT:    pand %xmm2, %xmm1
2236; SSE41-NEXT:    pxor %xmm3, %xmm3
2237; SSE41-NEXT:    packuswb %xmm3, %xmm1
2238; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2239; SSE41-NEXT:    pmullw %xmm0, %xmm1
2240; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
2241; SSE41-NEXT:    pand %xmm2, %xmm1
2242; SSE41-NEXT:    packuswb %xmm3, %xmm1
2243; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2244; SSE41-NEXT:    pmullw %xmm0, %xmm1
2245; SSE41-NEXT:    pand %xmm2, %xmm1
2246; SSE41-NEXT:    packuswb %xmm3, %xmm1
2247; SSE41-NEXT:    movdqa %xmm1, %xmm0
2248; SSE41-NEXT:    psrlw $8, %xmm0
2249; SSE41-NEXT:    pmullw %xmm1, %xmm0
2250; SSE41-NEXT:    movd %xmm0, %eax
2251; SSE41-NEXT:    # kill: def $al killed $al killed $eax
2252; SSE41-NEXT:    retq
2253;
2254; AVX1-LABEL: test_v64i8:
2255; AVX1:       # %bb.0:
2256; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
2257; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2258; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
2259; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2260; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
2261; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2262; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
2263; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2264; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
2265; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2266; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2267; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
2268; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2269; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2270; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2271; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2272; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2273; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2274; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2275; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2276; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2277; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
2278; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2279; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
2280; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
2281; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2282; AVX1-NEXT:    vmovd %xmm0, %eax
2283; AVX1-NEXT:    # kill: def $al killed $al killed $eax
2284; AVX1-NEXT:    vzeroupper
2285; AVX1-NEXT:    retq
2286;
2287; AVX2-LABEL: test_v64i8:
2288; AVX2:       # %bb.0:
2289; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2290; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2291; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
2292; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2293; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2294; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2295; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
2296; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2297; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2298; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
2299; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
2300; AVX2-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
2301; AVX2-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
2302; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
2303; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2304; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2305; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2306; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2307; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2308; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
2309; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2310; AVX2-NEXT:    vpsrld $16, %xmm2, %xmm2
2311; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2312; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2313; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
2314; AVX2-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2315; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
2316; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2317; AVX2-NEXT:    vmovd %xmm0, %eax
2318; AVX2-NEXT:    # kill: def $al killed $al killed $eax
2319; AVX2-NEXT:    vzeroupper
2320; AVX2-NEXT:    retq
2321;
2322; AVX512BW-LABEL: test_v64i8:
2323; AVX512BW:       # %bb.0:
2324; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2325; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2326; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2327; AVX512BW-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
2328; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2329; AVX512BW-NEXT:    vpand %ymm3, %ymm2, %ymm4
2330; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2331; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2332; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2333; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
2334; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
2335; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
2336; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2337; AVX512BW-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
2338; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2339; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
2340; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2341; AVX512BW-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2342; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2343; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
2344; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2345; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2346; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2347; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2348; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2349; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
2350; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2351; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
2352; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2353; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2354; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
2355; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
2356; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
2357; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2358; AVX512BW-NEXT:    vmovd %xmm0, %eax
2359; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
2360; AVX512BW-NEXT:    vzeroupper
2361; AVX512BW-NEXT:    retq
2362;
2363; AVX512BWVL-LABEL: test_v64i8:
2364; AVX512BWVL:       # %bb.0:
2365; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2366; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2367; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2368; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
2369; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2370; AVX512BWVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
2371; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2372; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2373; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2374; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
2375; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
2376; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
2377; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2378; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
2379; AVX512BWVL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
2380; AVX512BWVL-NEXT:    vpmovwb %ymm2, %xmm2
2381; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2382; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2383; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2384; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2385; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
2386; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2387; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2388; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2389; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
2390; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
2391; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2392; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2393; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
2394; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
2395; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2396; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
2397; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
2398; AVX512BWVL-NEXT:    vzeroupper
2399; AVX512BWVL-NEXT:    retq
2400;
2401; AVX512DQ-LABEL: test_v64i8:
2402; AVX512DQ:       # %bb.0:
2403; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2404; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2405; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2406; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
2407; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2408; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2409; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2410; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
2411; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2412; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2413; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm0
2414; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
2415; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
2416; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
2417; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
2418; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2419; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2420; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2421; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2422; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2423; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
2424; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2425; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
2426; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2427; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2428; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm0
2429; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2430; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
2431; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2432; AVX512DQ-NEXT:    vmovd %xmm0, %eax
2433; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
2434; AVX512DQ-NEXT:    vzeroupper
2435; AVX512DQ-NEXT:    retq
2436;
2437; AVX512DQVL-LABEL: test_v64i8:
2438; AVX512DQVL:       # %bb.0:
2439; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2440; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2441; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2442; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
2443; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2444; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2445; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2446; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
2447; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2448; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2449; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
2450; AVX512DQVL-NEXT:    vextracti128 $1, %ymm2, %xmm3
2451; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
2452; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
2453; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
2454; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2455; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2456; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2457; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2458; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2459; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
2460; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
2461; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
2462; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2463; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2464; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
2465; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2466; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
2467; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2468; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
2469; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
2470; AVX512DQVL-NEXT:    vzeroupper
2471; AVX512DQVL-NEXT:    retq
2472  %1 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> %a0)
2473  ret i8 %1
2474}
2475
2476define i8 @test_v128i8(<128 x i8> %a0) {
2477; SSE2-LABEL: test_v128i8:
2478; SSE2:       # %bb.0:
2479; SSE2-NEXT:    movdqa %xmm6, %xmm8
2480; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2481; SSE2-NEXT:    movdqa %xmm2, %xmm9
2482; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
2483; SSE2-NEXT:    pmullw %xmm8, %xmm9
2484; SSE2-NEXT:    movdqa %xmm4, %xmm10
2485; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]
2486; SSE2-NEXT:    pmullw %xmm9, %xmm10
2487; SSE2-NEXT:    movdqa %xmm0, %xmm9
2488; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
2489; SSE2-NEXT:    movdqa %xmm7, %xmm8
2490; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2491; SSE2-NEXT:    movdqa %xmm3, %xmm11
2492; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2493; SSE2-NEXT:    pmullw %xmm8, %xmm11
2494; SSE2-NEXT:    movdqa %xmm5, %xmm12
2495; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm0[8],xmm12[9],xmm0[9],xmm12[10],xmm0[10],xmm12[11],xmm0[11],xmm12[12],xmm0[12],xmm12[13],xmm0[13],xmm12[14],xmm0[14],xmm12[15],xmm0[15]
2496; SSE2-NEXT:    pmullw %xmm11, %xmm12
2497; SSE2-NEXT:    movdqa %xmm1, %xmm8
2498; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2499; SSE2-NEXT:    pmullw %xmm12, %xmm8
2500; SSE2-NEXT:    pmullw %xmm10, %xmm8
2501; SSE2-NEXT:    pmullw %xmm9, %xmm8
2502; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
2503; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2504; SSE2-NEXT:    pmullw %xmm6, %xmm2
2505; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2506; SSE2-NEXT:    pmullw %xmm2, %xmm4
2507; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2508; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
2509; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
2510; SSE2-NEXT:    pmullw %xmm7, %xmm3
2511; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2512; SSE2-NEXT:    pmullw %xmm3, %xmm5
2513; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2514; SSE2-NEXT:    pmullw %xmm5, %xmm1
2515; SSE2-NEXT:    pmullw %xmm4, %xmm1
2516; SSE2-NEXT:    pmullw %xmm8, %xmm1
2517; SSE2-NEXT:    pmullw %xmm0, %xmm1
2518; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
2519; SSE2-NEXT:    movdqa %xmm1, %xmm2
2520; SSE2-NEXT:    pand %xmm0, %xmm2
2521; SSE2-NEXT:    pxor %xmm3, %xmm3
2522; SSE2-NEXT:    packuswb %xmm3, %xmm2
2523; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
2524; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2525; SSE2-NEXT:    pmullw %xmm1, %xmm2
2526; SSE2-NEXT:    movdqa %xmm2, %xmm1
2527; SSE2-NEXT:    pand %xmm0, %xmm1
2528; SSE2-NEXT:    packuswb %xmm3, %xmm1
2529; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2530; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2531; SSE2-NEXT:    pmullw %xmm2, %xmm1
2532; SSE2-NEXT:    pand %xmm0, %xmm1
2533; SSE2-NEXT:    packuswb %xmm3, %xmm1
2534; SSE2-NEXT:    movdqa %xmm1, %xmm0
2535; SSE2-NEXT:    psrlw $8, %xmm0
2536; SSE2-NEXT:    pmullw %xmm1, %xmm0
2537; SSE2-NEXT:    movd %xmm0, %eax
2538; SSE2-NEXT:    # kill: def $al killed $al killed $eax
2539; SSE2-NEXT:    retq
2540;
2541; SSE41-LABEL: test_v128i8:
2542; SSE41:       # %bb.0:
2543; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2544; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
2545; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2546; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2547; SSE41-NEXT:    pmullw %xmm5, %xmm1
2548; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2549; SSE41-NEXT:    pmullw %xmm10, %xmm8
2550; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2551; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2552; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2553; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2554; SSE41-NEXT:    pmullw %xmm7, %xmm3
2555; SSE41-NEXT:    movdqa %xmm3, %xmm7
2556; SSE41-NEXT:    pmullw %xmm1, %xmm3
2557; SSE41-NEXT:    pand %xmm9, %xmm1
2558; SSE41-NEXT:    pand %xmm9, %xmm8
2559; SSE41-NEXT:    packuswb %xmm1, %xmm8
2560; SSE41-NEXT:    pand %xmm9, %xmm7
2561; SSE41-NEXT:    pmullw %xmm10, %xmm5
2562; SSE41-NEXT:    pand %xmm9, %xmm5
2563; SSE41-NEXT:    packuswb %xmm7, %xmm5
2564; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2565; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2566; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2567; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2568; SSE41-NEXT:    pmullw %xmm4, %xmm0
2569; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2570; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
2571; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2572; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2573; SSE41-NEXT:    pmullw %xmm6, %xmm2
2574; SSE41-NEXT:    pmullw %xmm2, %xmm3
2575; SSE41-NEXT:    pmullw %xmm0, %xmm3
2576; SSE41-NEXT:    pand %xmm9, %xmm0
2577; SSE41-NEXT:    pmullw %xmm10, %xmm7
2578; SSE41-NEXT:    pand %xmm9, %xmm7
2579; SSE41-NEXT:    packuswb %xmm0, %xmm7
2580; SSE41-NEXT:    pand %xmm9, %xmm2
2581; SSE41-NEXT:    pmullw %xmm4, %xmm1
2582; SSE41-NEXT:    pand %xmm9, %xmm1
2583; SSE41-NEXT:    packuswb %xmm2, %xmm1
2584; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2585; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2586; SSE41-NEXT:    pmullw %xmm0, %xmm1
2587; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2588; SSE41-NEXT:    pshufb %xmm0, %xmm1
2589; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2590; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
2591; SSE41-NEXT:    pmullw %xmm2, %xmm4
2592; SSE41-NEXT:    pshufb %xmm0, %xmm4
2593; SSE41-NEXT:    movdqa %xmm3, %xmm0
2594; SSE41-NEXT:    pand %xmm9, %xmm0
2595; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2596; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2597; SSE41-NEXT:    pmullw %xmm2, %xmm1
2598; SSE41-NEXT:    pand %xmm9, %xmm1
2599; SSE41-NEXT:    packuswb %xmm0, %xmm1
2600; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2601; SSE41-NEXT:    pmullw %xmm3, %xmm0
2602; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2603; SSE41-NEXT:    pand %xmm9, %xmm0
2604; SSE41-NEXT:    pxor %xmm2, %xmm2
2605; SSE41-NEXT:    packuswb %xmm2, %xmm0
2606; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2607; SSE41-NEXT:    pmullw %xmm1, %xmm0
2608; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2609; SSE41-NEXT:    pand %xmm9, %xmm0
2610; SSE41-NEXT:    packuswb %xmm2, %xmm0
2611; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2612; SSE41-NEXT:    pmullw %xmm1, %xmm0
2613; SSE41-NEXT:    pand %xmm9, %xmm0
2614; SSE41-NEXT:    packuswb %xmm2, %xmm0
2615; SSE41-NEXT:    movdqa %xmm0, %xmm1
2616; SSE41-NEXT:    psrlw $8, %xmm1
2617; SSE41-NEXT:    pmullw %xmm0, %xmm1
2618; SSE41-NEXT:    movd %xmm1, %eax
2619; SSE41-NEXT:    # kill: def $al killed $al killed $eax
2620; SSE41-NEXT:    retq
2621;
2622; AVX1-LABEL: test_v128i8:
2623; AVX1:       # %bb.0:
2624; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2625; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2626; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2627; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2628; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm8
2629; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2630; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
2631; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]
2632; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
2633; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2634; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
2635; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
2636; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2637; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2638; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
2639; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
2640; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
2641; AVX1-NEXT:    vpmullw %xmm4, %xmm8, %xmm4
2642; AVX1-NEXT:    vpmullw %xmm4, %xmm9, %xmm4
2643; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2644; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2645; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2646; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2647; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
2648; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2649; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
2650; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2651; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2652; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2653; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2654; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2655; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
2656; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
2657; AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
2658; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2659; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2660; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2661; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2662; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2663; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
2664; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2665; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
2666; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
2667; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2668; AVX1-NEXT:    vmovd %xmm0, %eax
2669; AVX1-NEXT:    # kill: def $al killed $al killed $eax
2670; AVX1-NEXT:    vzeroupper
2671; AVX1-NEXT:    retq
2672;
2673; AVX2-LABEL: test_v128i8:
2674; AVX2:       # %bb.0:
2675; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
2676; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2677; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2678; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
2679; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2680; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2681; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2682; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
2683; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2684; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2685; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
2686; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
2687; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2688; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2689; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm1
2690; AVX2-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
2691; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
2692; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
2693; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
2694; AVX2-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
2695; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2696; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm1
2697; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2698; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2699; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2700; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2701; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2702; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm1
2703; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2704; AVX2-NEXT:    vpsrld $16, %xmm1, %xmm1
2705; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2706; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2707; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
2708; AVX2-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2709; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
2710; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2711; AVX2-NEXT:    vmovd %xmm0, %eax
2712; AVX2-NEXT:    # kill: def $al killed $al killed $eax
2713; AVX2-NEXT:    vzeroupper
2714; AVX2-NEXT:    retq
2715;
2716; AVX512BW-LABEL: test_v128i8:
2717; AVX512BW:       # %bb.0:
2718; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2719; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2720; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2721; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2722; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2723; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2724; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2725; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2726; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2727; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm3
2728; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
2729; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
2730; AVX512BW-NEXT:    vpand %ymm1, %ymm2, %ymm1
2731; AVX512BW-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
2732; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
2733; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2734; AVX512BW-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
2735; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2736; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
2737; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2738; AVX512BW-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
2739; AVX512BW-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
2740; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
2741; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2742; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2743; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2744; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2745; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2746; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
2747; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
2748; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
2749; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2750; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2751; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
2752; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
2753; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
2754; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2755; AVX512BW-NEXT:    vmovd %xmm0, %eax
2756; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
2757; AVX512BW-NEXT:    vzeroupper
2758; AVX512BW-NEXT:    retq
2759;
2760; AVX512BWVL-LABEL: test_v128i8:
2761; AVX512BWVL:       # %bb.0:
2762; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2763; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2764; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2765; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2766; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2767; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2768; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
2769; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2770; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2771; AVX512BWVL-NEXT:    vpand %ymm1, %ymm0, %ymm3
2772; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
2773; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
2774; AVX512BWVL-NEXT:    vpand %ymm1, %ymm2, %ymm1
2775; AVX512BWVL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
2776; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
2777; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2778; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
2779; AVX512BWVL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
2780; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
2781; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2782; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2783; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
2784; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
2785; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
2786; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2787; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2788; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2789; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
2790; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
2791; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2792; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2793; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
2794; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
2795; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2796; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
2797; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
2798; AVX512BWVL-NEXT:    vzeroupper
2799; AVX512BWVL-NEXT:    retq
2800;
2801; AVX512DQ-LABEL: test_v128i8:
2802; AVX512DQ:       # %bb.0:
2803; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
2804; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2805; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2806; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
2807; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2808; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2809; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2810; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
2811; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2812; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2813; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
2814; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
2815; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2816; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2817; AVX512DQ-NEXT:    vextracti128 $1, %ymm4, %xmm1
2818; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
2819; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
2820; AVX512DQ-NEXT:    vpand %xmm2, %xmm1, %xmm1
2821; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
2822; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
2823; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2824; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm1
2825; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2826; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2827; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2828; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2829; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2830; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm1
2831; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2832; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
2833; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2834; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2835; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm0
2836; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2837; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
2838; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2839; AVX512DQ-NEXT:    vmovd %xmm0, %eax
2840; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
2841; AVX512DQ-NEXT:    vzeroupper
2842; AVX512DQ-NEXT:    retq
2843;
2844; AVX512DQVL-LABEL: test_v128i8:
2845; AVX512DQVL:       # %bb.0:
2846; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
2847; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2848; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2849; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
2850; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2851; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2852; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2853; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
2854; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2855; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2856; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
2857; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
2858; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2859; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2860; AVX512DQVL-NEXT:    vextracti128 $1, %ymm4, %xmm1
2861; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
2862; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
2863; AVX512DQVL-NEXT:    vpand %xmm2, %xmm1, %xmm1
2864; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm3
2865; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
2866; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2867; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
2868; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2869; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2870; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2871; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2872; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2873; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
2874; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
2875; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
2876; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2877; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2878; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
2879; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
2880; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
2881; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
2882; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
2883; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
2884; AVX512DQVL-NEXT:    vzeroupper
2885; AVX512DQVL-NEXT:    retq
2886  %1 = call i8 @llvm.experimental.vector.reduce.mul.v128i8(<128 x i8> %a0)
2887  ret i8 %1
2888}
2889
2890declare i64 @llvm.experimental.vector.reduce.mul.v2i64(<2 x i64>)
2891declare i64 @llvm.experimental.vector.reduce.mul.v4i64(<4 x i64>)
2892declare i64 @llvm.experimental.vector.reduce.mul.v8i64(<8 x i64>)
2893declare i64 @llvm.experimental.vector.reduce.mul.v16i64(<16 x i64>)
2894
2895declare i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32>)
2896declare i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32>)
2897declare i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32>)
2898declare i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32>)
2899declare i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32>)
2900
2901declare i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16>)
2902declare i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16>)
2903declare i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16>)
2904declare i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16>)
2905declare i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16>)
2906declare i16 @llvm.experimental.vector.reduce.mul.v64i16(<64 x i16>)
2907
2908declare i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8>)
2909declare i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8>)
2910declare i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8>)
2911declare i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8>)
2912declare i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8>)
2913declare i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8>)
2914declare i8 @llvm.experimental.vector.reduce.mul.v128i8(<128 x i8>)
2915