1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
9
10declare {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
11declare {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
12declare {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
13declare {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
14declare {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
15declare {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
16declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
17
18declare {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
19declare {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
20declare {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
21declare {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
22declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
23
24declare {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
25declare {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
26declare {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
27
28define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
29; CHECK-LABEL: smulo_v1i32:
30; CHECK:       # %bb.0:
31; CHECK-NEXT:    xorl %eax, %eax
32; CHECK-NEXT:    imull %esi, %edi
33; CHECK-NEXT:    seto %al
34; CHECK-NEXT:    negl %eax
35; CHECK-NEXT:    movl %edi, (%rdx)
36; CHECK-NEXT:    retq
37  %t = call {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
38  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
39  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
40  %res = sext <1 x i1> %obit to <1 x i32>
41  store <1 x i32> %val, <1 x i32>* %p2
42  ret <1 x i32> %res
43}
44
45define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
46; SSE2-LABEL: smulo_v2i32:
47; SSE2:       # %bb.0:
48; SSE2-NEXT:    pxor %xmm2, %xmm2
49; SSE2-NEXT:    pxor %xmm3, %xmm3
50; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
51; SSE2-NEXT:    pand %xmm0, %xmm3
52; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
53; SSE2-NEXT:    pand %xmm1, %xmm2
54; SSE2-NEXT:    paddd %xmm3, %xmm2
55; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
56; SSE2-NEXT:    pmuludq %xmm1, %xmm0
57; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
58; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
59; SSE2-NEXT:    pmuludq %xmm3, %xmm1
60; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
61; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
62; SSE2-NEXT:    psubd %xmm2, %xmm4
63; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
64; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
65; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
66; SSE2-NEXT:    movq %xmm0, (%rdi)
67; SSE2-NEXT:    movdqa %xmm0, %xmm1
68; SSE2-NEXT:    psrad $31, %xmm1
69; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
70; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
71; SSE2-NEXT:    pxor %xmm1, %xmm0
72; SSE2-NEXT:    retq
73;
74; SSSE3-LABEL: smulo_v2i32:
75; SSSE3:       # %bb.0:
76; SSSE3-NEXT:    pxor %xmm2, %xmm2
77; SSSE3-NEXT:    pxor %xmm3, %xmm3
78; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
79; SSSE3-NEXT:    pand %xmm0, %xmm3
80; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
81; SSSE3-NEXT:    pand %xmm1, %xmm2
82; SSSE3-NEXT:    paddd %xmm3, %xmm2
83; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
84; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
85; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
86; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
87; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
88; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
89; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
90; SSSE3-NEXT:    psubd %xmm2, %xmm4
91; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
92; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
93; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
94; SSSE3-NEXT:    movq %xmm0, (%rdi)
95; SSSE3-NEXT:    movdqa %xmm0, %xmm1
96; SSSE3-NEXT:    psrad $31, %xmm1
97; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
98; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
99; SSSE3-NEXT:    pxor %xmm1, %xmm0
100; SSSE3-NEXT:    retq
101;
102; SSE41-LABEL: smulo_v2i32:
103; SSE41:       # %bb.0:
104; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
105; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
106; SSE41-NEXT:    pmuldq %xmm2, %xmm3
107; SSE41-NEXT:    movdqa %xmm0, %xmm2
108; SSE41-NEXT:    pmuldq %xmm1, %xmm2
109; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
110; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
111; SSE41-NEXT:    pmulld %xmm1, %xmm0
112; SSE41-NEXT:    movq %xmm0, (%rdi)
113; SSE41-NEXT:    psrad $31, %xmm0
114; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
115; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
116; SSE41-NEXT:    pxor %xmm1, %xmm0
117; SSE41-NEXT:    retq
118;
119; AVX1-LABEL: smulo_v2i32:
120; AVX1:       # %bb.0:
121; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
122; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
123; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
124; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
125; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
126; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
127; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
128; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
129; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
130; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
131; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
132; AVX1-NEXT:    vmovq %xmm1, (%rdi)
133; AVX1-NEXT:    retq
134;
135; AVX2-LABEL: smulo_v2i32:
136; AVX2:       # %bb.0:
137; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
138; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
139; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
140; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
141; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
142; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
143; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
144; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
145; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
146; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
147; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
148; AVX2-NEXT:    vmovq %xmm1, (%rdi)
149; AVX2-NEXT:    retq
150;
151; AVX512-LABEL: smulo_v2i32:
152; AVX512:       # %bb.0:
153; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
154; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
155; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
156; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
157; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
158; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
159; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
160; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
161; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
162; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
163; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
164; AVX512-NEXT:    vmovq %xmm1, (%rdi)
165; AVX512-NEXT:    retq
166  %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
167  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
168  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
169  %res = sext <2 x i1> %obit to <2 x i32>
170  store <2 x i32> %val, <2 x i32>* %p2
171  ret <2 x i32> %res
172}
173
174define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
175; SSE2-LABEL: smulo_v3i32:
176; SSE2:       # %bb.0:
177; SSE2-NEXT:    pxor %xmm2, %xmm2
178; SSE2-NEXT:    pxor %xmm3, %xmm3
179; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
180; SSE2-NEXT:    pand %xmm0, %xmm3
181; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
182; SSE2-NEXT:    pand %xmm1, %xmm2
183; SSE2-NEXT:    paddd %xmm3, %xmm2
184; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
185; SSE2-NEXT:    pmuludq %xmm1, %xmm0
186; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
187; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
188; SSE2-NEXT:    pmuludq %xmm3, %xmm1
189; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
190; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
191; SSE2-NEXT:    psubd %xmm2, %xmm4
192; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
193; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
194; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
195; SSE2-NEXT:    movq %xmm2, (%rdi)
196; SSE2-NEXT:    psrad $31, %xmm2
197; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
198; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
199; SSE2-NEXT:    pxor %xmm2, %xmm1
200; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
201; SSE2-NEXT:    movd %xmm0, 8(%rdi)
202; SSE2-NEXT:    movdqa %xmm1, %xmm0
203; SSE2-NEXT:    retq
204;
205; SSSE3-LABEL: smulo_v3i32:
206; SSSE3:       # %bb.0:
207; SSSE3-NEXT:    pxor %xmm2, %xmm2
208; SSSE3-NEXT:    pxor %xmm3, %xmm3
209; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
210; SSSE3-NEXT:    pand %xmm0, %xmm3
211; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
212; SSSE3-NEXT:    pand %xmm1, %xmm2
213; SSSE3-NEXT:    paddd %xmm3, %xmm2
214; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
215; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
216; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
217; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
218; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
219; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
220; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
221; SSSE3-NEXT:    psubd %xmm2, %xmm4
222; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
223; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
224; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
225; SSSE3-NEXT:    movq %xmm2, (%rdi)
226; SSSE3-NEXT:    psrad $31, %xmm2
227; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
228; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
229; SSSE3-NEXT:    pxor %xmm2, %xmm1
230; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
231; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
232; SSSE3-NEXT:    movdqa %xmm1, %xmm0
233; SSSE3-NEXT:    retq
234;
235; SSE41-LABEL: smulo_v3i32:
236; SSE41:       # %bb.0:
237; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
238; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
239; SSE41-NEXT:    pmuldq %xmm2, %xmm3
240; SSE41-NEXT:    movdqa %xmm0, %xmm2
241; SSE41-NEXT:    pmuldq %xmm1, %xmm2
242; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
243; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
244; SSE41-NEXT:    pmulld %xmm1, %xmm0
245; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
246; SSE41-NEXT:    movq %xmm0, (%rdi)
247; SSE41-NEXT:    psrad $31, %xmm0
248; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
249; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
250; SSE41-NEXT:    pxor %xmm1, %xmm0
251; SSE41-NEXT:    retq
252;
253; AVX1-LABEL: smulo_v3i32:
254; AVX1:       # %bb.0:
255; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
256; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
257; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
258; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
259; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
260; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
261; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
262; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
263; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
264; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
265; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
266; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
267; AVX1-NEXT:    vmovq %xmm1, (%rdi)
268; AVX1-NEXT:    retq
269;
270; AVX2-LABEL: smulo_v3i32:
271; AVX2:       # %bb.0:
272; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
273; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
274; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
275; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
276; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
277; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
278; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
279; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
280; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
281; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
282; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
283; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
284; AVX2-NEXT:    vmovq %xmm1, (%rdi)
285; AVX2-NEXT:    retq
286;
287; AVX512-LABEL: smulo_v3i32:
288; AVX512:       # %bb.0:
289; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
290; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
291; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
292; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
293; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
294; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
295; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
296; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
297; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
298; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
299; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
300; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
301; AVX512-NEXT:    vmovq %xmm1, (%rdi)
302; AVX512-NEXT:    retq
303  %t = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
304  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
305  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
306  %res = sext <3 x i1> %obit to <3 x i32>
307  store <3 x i32> %val, <3 x i32>* %p2
308  ret <3 x i32> %res
309}
310
311define <4 x i32> @smulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
312; SSE2-LABEL: smulo_v4i32:
313; SSE2:       # %bb.0:
314; SSE2-NEXT:    pxor %xmm2, %xmm2
315; SSE2-NEXT:    pxor %xmm3, %xmm3
316; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
317; SSE2-NEXT:    pand %xmm0, %xmm3
318; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
319; SSE2-NEXT:    pand %xmm1, %xmm2
320; SSE2-NEXT:    paddd %xmm3, %xmm2
321; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
322; SSE2-NEXT:    pmuludq %xmm1, %xmm0
323; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
324; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
325; SSE2-NEXT:    pmuludq %xmm3, %xmm1
326; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
327; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
328; SSE2-NEXT:    psubd %xmm2, %xmm4
329; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
330; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
331; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
332; SSE2-NEXT:    movdqa %xmm0, (%rdi)
333; SSE2-NEXT:    movdqa %xmm0, %xmm1
334; SSE2-NEXT:    psrad $31, %xmm1
335; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
336; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
337; SSE2-NEXT:    pxor %xmm1, %xmm0
338; SSE2-NEXT:    retq
339;
340; SSSE3-LABEL: smulo_v4i32:
341; SSSE3:       # %bb.0:
342; SSSE3-NEXT:    pxor %xmm2, %xmm2
343; SSSE3-NEXT:    pxor %xmm3, %xmm3
344; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
345; SSSE3-NEXT:    pand %xmm0, %xmm3
346; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
347; SSSE3-NEXT:    pand %xmm1, %xmm2
348; SSSE3-NEXT:    paddd %xmm3, %xmm2
349; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
350; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
351; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
352; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
353; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
354; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
355; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
356; SSSE3-NEXT:    psubd %xmm2, %xmm4
357; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
358; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
359; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
360; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
361; SSSE3-NEXT:    movdqa %xmm0, %xmm1
362; SSSE3-NEXT:    psrad $31, %xmm1
363; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
364; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
365; SSSE3-NEXT:    pxor %xmm1, %xmm0
366; SSSE3-NEXT:    retq
367;
368; SSE41-LABEL: smulo_v4i32:
369; SSE41:       # %bb.0:
370; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
371; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
372; SSE41-NEXT:    pmuldq %xmm2, %xmm3
373; SSE41-NEXT:    movdqa %xmm0, %xmm2
374; SSE41-NEXT:    pmuldq %xmm1, %xmm2
375; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
376; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
377; SSE41-NEXT:    pmulld %xmm1, %xmm0
378; SSE41-NEXT:    movdqa %xmm0, (%rdi)
379; SSE41-NEXT:    psrad $31, %xmm0
380; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
381; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
382; SSE41-NEXT:    pxor %xmm1, %xmm0
383; SSE41-NEXT:    retq
384;
385; AVX1-LABEL: smulo_v4i32:
386; AVX1:       # %bb.0:
387; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
388; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
389; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
390; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
391; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
392; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
393; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
394; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
395; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
396; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
397; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
398; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
399; AVX1-NEXT:    retq
400;
401; AVX2-LABEL: smulo_v4i32:
402; AVX2:       # %bb.0:
403; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
404; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
405; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
406; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
407; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
408; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
409; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
410; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
411; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
412; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
413; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
414; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
415; AVX2-NEXT:    retq
416;
417; AVX512-LABEL: smulo_v4i32:
418; AVX512:       # %bb.0:
419; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
420; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
421; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
422; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
423; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
424; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
425; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
426; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
427; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
428; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
429; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
430; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
431; AVX512-NEXT:    retq
432  %t = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
433  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
434  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
435  %res = sext <4 x i1> %obit to <4 x i32>
436  store <4 x i32> %val, <4 x i32>* %p2
437  ret <4 x i32> %res
438}
439
440define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
441; SSE2-LABEL: smulo_v6i32:
442; SSE2:       # %bb.0:
443; SSE2-NEXT:    movq %rdi, %rax
444; SSE2-NEXT:    movd %r8d, %xmm0
445; SSE2-NEXT:    movd %ecx, %xmm1
446; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
447; SSE2-NEXT:    movd %edx, %xmm0
448; SSE2-NEXT:    movd %esi, %xmm2
449; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
450; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
451; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
452; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
453; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
454; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
455; SSE2-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
456; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
457; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
458; SSE2-NEXT:    movd %r9d, %xmm10
459; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
460; SSE2-NEXT:    movdqa %xmm10, %xmm9
461; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
462; SSE2-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
463; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
464; SSE2-NEXT:    pmuludq %xmm6, %xmm10
465; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
466; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
467; SSE2-NEXT:    pxor %xmm7, %xmm7
468; SSE2-NEXT:    pxor %xmm4, %xmm4
469; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
470; SSE2-NEXT:    pand %xmm2, %xmm4
471; SSE2-NEXT:    pxor %xmm3, %xmm3
472; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
473; SSE2-NEXT:    pand %xmm5, %xmm3
474; SSE2-NEXT:    paddd %xmm4, %xmm3
475; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
476; SSE2-NEXT:    pmuludq %xmm5, %xmm2
477; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
478; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
479; SSE2-NEXT:    pmuludq %xmm4, %xmm5
480; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
481; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
482; SSE2-NEXT:    psubd %xmm3, %xmm0
483; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
484; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
485; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
486; SSE2-NEXT:    movdqa %xmm2, (%rcx)
487; SSE2-NEXT:    psrad $31, %xmm2
488; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
489; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
490; SSE2-NEXT:    pxor %xmm0, %xmm2
491; SSE2-NEXT:    pxor %xmm3, %xmm3
492; SSE2-NEXT:    pcmpgtd %xmm6, %xmm3
493; SSE2-NEXT:    pand %xmm9, %xmm3
494; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
495; SSE2-NEXT:    pand %xmm6, %xmm7
496; SSE2-NEXT:    paddd %xmm3, %xmm7
497; SSE2-NEXT:    pmuludq %xmm8, %xmm1
498; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
499; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
500; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
501; SSE2-NEXT:    psubd %xmm7, %xmm4
502; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
503; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
504; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
505; SSE2-NEXT:    movq %xmm3, 16(%rcx)
506; SSE2-NEXT:    psrad $31, %xmm3
507; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
508; SSE2-NEXT:    pxor %xmm0, %xmm3
509; SSE2-NEXT:    movq %xmm3, 16(%rdi)
510; SSE2-NEXT:    movdqa %xmm2, (%rdi)
511; SSE2-NEXT:    retq
512;
513; SSSE3-LABEL: smulo_v6i32:
514; SSSE3:       # %bb.0:
515; SSSE3-NEXT:    movq %rdi, %rax
516; SSSE3-NEXT:    movd %r8d, %xmm0
517; SSSE3-NEXT:    movd %ecx, %xmm1
518; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
519; SSSE3-NEXT:    movd %edx, %xmm0
520; SSSE3-NEXT:    movd %esi, %xmm2
521; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
522; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
523; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
524; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
525; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
526; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
527; SSSE3-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
528; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
529; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
530; SSSE3-NEXT:    movd %r9d, %xmm10
531; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
532; SSSE3-NEXT:    movdqa %xmm10, %xmm9
533; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
534; SSSE3-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
535; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
536; SSSE3-NEXT:    pmuludq %xmm6, %xmm10
537; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
538; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
539; SSSE3-NEXT:    pxor %xmm7, %xmm7
540; SSSE3-NEXT:    pxor %xmm4, %xmm4
541; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
542; SSSE3-NEXT:    pand %xmm2, %xmm4
543; SSSE3-NEXT:    pxor %xmm3, %xmm3
544; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
545; SSSE3-NEXT:    pand %xmm5, %xmm3
546; SSSE3-NEXT:    paddd %xmm4, %xmm3
547; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
548; SSSE3-NEXT:    pmuludq %xmm5, %xmm2
549; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
550; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
551; SSSE3-NEXT:    pmuludq %xmm4, %xmm5
552; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
553; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
554; SSSE3-NEXT:    psubd %xmm3, %xmm0
555; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
556; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
557; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
558; SSSE3-NEXT:    movdqa %xmm2, (%rcx)
559; SSSE3-NEXT:    psrad $31, %xmm2
560; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
561; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
562; SSSE3-NEXT:    pxor %xmm0, %xmm2
563; SSSE3-NEXT:    pxor %xmm3, %xmm3
564; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm3
565; SSSE3-NEXT:    pand %xmm9, %xmm3
566; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm7
567; SSSE3-NEXT:    pand %xmm6, %xmm7
568; SSSE3-NEXT:    paddd %xmm3, %xmm7
569; SSSE3-NEXT:    pmuludq %xmm8, %xmm1
570; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
571; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
572; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
573; SSSE3-NEXT:    psubd %xmm7, %xmm4
574; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
575; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
576; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
577; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
578; SSSE3-NEXT:    psrad $31, %xmm3
579; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
580; SSSE3-NEXT:    pxor %xmm0, %xmm3
581; SSSE3-NEXT:    movq %xmm3, 16(%rdi)
582; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
583; SSSE3-NEXT:    retq
584;
585; SSE41-LABEL: smulo_v6i32:
586; SSE41:       # %bb.0:
587; SSE41-NEXT:    movq %rdi, %rax
588; SSE41-NEXT:    movd %esi, %xmm2
589; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
590; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
591; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
592; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
593; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
594; SSE41-NEXT:    movdqa %xmm1, %xmm0
595; SSE41-NEXT:    pmuldq %xmm2, %xmm1
596; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
597; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
598; SSE41-NEXT:    movd %r9d, %xmm4
599; SSE41-NEXT:    movdqa %xmm4, %xmm5
600; SSE41-NEXT:    pmuldq %xmm3, %xmm4
601; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
602; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
603; SSE41-NEXT:    pmulld %xmm3, %xmm5
604; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
605; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
606; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
607; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
608; SSE41-NEXT:    pmuldq %xmm3, %xmm6
609; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
610; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
611; SSE41-NEXT:    pmulld %xmm2, %xmm0
612; SSE41-NEXT:    movdqa %xmm0, (%rcx)
613; SSE41-NEXT:    psrad $31, %xmm0
614; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
615; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
616; SSE41-NEXT:    pxor %xmm1, %xmm0
617; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
618; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
619; SSE41-NEXT:    pmuldq %xmm2, %xmm3
620; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
621; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
622; SSE41-NEXT:    movq %xmm5, 16(%rcx)
623; SSE41-NEXT:    psrad $31, %xmm5
624; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
625; SSE41-NEXT:    pxor %xmm1, %xmm5
626; SSE41-NEXT:    movq %xmm5, 16(%rdi)
627; SSE41-NEXT:    movdqa %xmm0, (%rdi)
628; SSE41-NEXT:    retq
629;
630; AVX1-LABEL: smulo_v6i32:
631; AVX1:       # %bb.0:
632; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
633; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
634; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
635; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
636; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
637; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
638; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
639; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
640; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
641; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
642; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
643; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
644; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
645; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
646; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
647; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
648; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
649; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
650; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
651; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
652; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
653; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
654; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
655; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
656; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
657; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
658; AVX1-NEXT:    retq
659;
660; AVX2-LABEL: smulo_v6i32:
661; AVX2:       # %bb.0:
662; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
663; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
664; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
665; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
666; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
667; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
668; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
669; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
670; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
671; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
672; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
673; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
674; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
675; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
676; AVX2-NEXT:    retq
677;
678; AVX512-LABEL: smulo_v6i32:
679; AVX512:       # %bb.0:
680; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
681; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
682; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
683; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
684; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
685; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
686; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
687; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
688; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
689; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
690; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
691; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
692; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
693; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
694; AVX512-NEXT:    retq
695  %t = call {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
696  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
697  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
698  %res = sext <6 x i1> %obit to <6 x i32>
699  store <6 x i32> %val, <6 x i32>* %p2
700  ret <6 x i32> %res
701}
702
703define <8 x i32> @smulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
704; SSE2-LABEL: smulo_v8i32:
705; SSE2:       # %bb.0:
706; SSE2-NEXT:    pxor %xmm4, %xmm4
707; SSE2-NEXT:    pxor %xmm5, %xmm5
708; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
709; SSE2-NEXT:    pand %xmm0, %xmm5
710; SSE2-NEXT:    pxor %xmm6, %xmm6
711; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
712; SSE2-NEXT:    pand %xmm2, %xmm6
713; SSE2-NEXT:    paddd %xmm5, %xmm6
714; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
715; SSE2-NEXT:    pmuludq %xmm2, %xmm0
716; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
717; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
718; SSE2-NEXT:    pmuludq %xmm7, %xmm2
719; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
720; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
721; SSE2-NEXT:    psubd %xmm6, %xmm5
722; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
723; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
724; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
725; SSE2-NEXT:    movdqa %xmm0, (%rdi)
726; SSE2-NEXT:    psrad $31, %xmm0
727; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
728; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
729; SSE2-NEXT:    pxor %xmm2, %xmm0
730; SSE2-NEXT:    pxor %xmm5, %xmm5
731; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
732; SSE2-NEXT:    pand %xmm1, %xmm5
733; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
734; SSE2-NEXT:    pand %xmm3, %xmm4
735; SSE2-NEXT:    paddd %xmm5, %xmm4
736; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
737; SSE2-NEXT:    pmuludq %xmm3, %xmm1
738; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
739; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
740; SSE2-NEXT:    pmuludq %xmm5, %xmm3
741; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
742; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
743; SSE2-NEXT:    psubd %xmm4, %xmm6
744; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
745; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
746; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
747; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
748; SSE2-NEXT:    psrad $31, %xmm1
749; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
750; SSE2-NEXT:    pxor %xmm2, %xmm1
751; SSE2-NEXT:    retq
752;
753; SSSE3-LABEL: smulo_v8i32:
754; SSSE3:       # %bb.0:
755; SSSE3-NEXT:    pxor %xmm4, %xmm4
756; SSSE3-NEXT:    pxor %xmm5, %xmm5
757; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
758; SSSE3-NEXT:    pand %xmm0, %xmm5
759; SSSE3-NEXT:    pxor %xmm6, %xmm6
760; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
761; SSSE3-NEXT:    pand %xmm2, %xmm6
762; SSSE3-NEXT:    paddd %xmm5, %xmm6
763; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
764; SSSE3-NEXT:    pmuludq %xmm2, %xmm0
765; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
766; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
767; SSSE3-NEXT:    pmuludq %xmm7, %xmm2
768; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
769; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
770; SSSE3-NEXT:    psubd %xmm6, %xmm5
771; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
772; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
773; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
774; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
775; SSSE3-NEXT:    psrad $31, %xmm0
776; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
777; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
778; SSSE3-NEXT:    pxor %xmm2, %xmm0
779; SSSE3-NEXT:    pxor %xmm5, %xmm5
780; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
781; SSSE3-NEXT:    pand %xmm1, %xmm5
782; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
783; SSSE3-NEXT:    pand %xmm3, %xmm4
784; SSSE3-NEXT:    paddd %xmm5, %xmm4
785; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
786; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
787; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
788; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
789; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
790; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
791; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
792; SSSE3-NEXT:    psubd %xmm4, %xmm6
793; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
794; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
795; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
796; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
797; SSSE3-NEXT:    psrad $31, %xmm1
798; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm1
799; SSSE3-NEXT:    pxor %xmm2, %xmm1
800; SSSE3-NEXT:    retq
801;
802; SSE41-LABEL: smulo_v8i32:
803; SSE41:       # %bb.0:
804; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
805; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
806; SSE41-NEXT:    pmuldq %xmm4, %xmm5
807; SSE41-NEXT:    movdqa %xmm0, %xmm4
808; SSE41-NEXT:    pmuldq %xmm2, %xmm4
809; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
810; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
811; SSE41-NEXT:    pmulld %xmm2, %xmm0
812; SSE41-NEXT:    movdqa %xmm0, (%rdi)
813; SSE41-NEXT:    psrad $31, %xmm0
814; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
815; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
816; SSE41-NEXT:    pxor %xmm2, %xmm0
817; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
818; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
819; SSE41-NEXT:    pmuldq %xmm4, %xmm5
820; SSE41-NEXT:    movdqa %xmm1, %xmm4
821; SSE41-NEXT:    pmuldq %xmm3, %xmm4
822; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
823; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
824; SSE41-NEXT:    pmulld %xmm3, %xmm1
825; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
826; SSE41-NEXT:    psrad $31, %xmm1
827; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
828; SSE41-NEXT:    pxor %xmm2, %xmm1
829; SSE41-NEXT:    retq
830;
831; AVX1-LABEL: smulo_v8i32:
832; AVX1:       # %bb.0:
833; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
834; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
835; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
836; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
837; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
838; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
839; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
840; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
841; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
842; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
843; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
844; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
845; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
846; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
847; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
848; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
849; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
850; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
851; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
852; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
853; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
854; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
855; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
856; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
857; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
858; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
859; AVX1-NEXT:    retq
860;
861; AVX2-LABEL: smulo_v8i32:
862; AVX2:       # %bb.0:
863; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
864; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
865; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
866; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
867; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
868; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
869; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
870; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
871; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
872; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
873; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
874; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
875; AVX2-NEXT:    retq
876;
877; AVX512-LABEL: smulo_v8i32:
878; AVX512:       # %bb.0:
879; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
880; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
881; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
882; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
883; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
884; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
885; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
886; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
887; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
888; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
889; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
890; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
891; AVX512-NEXT:    retq
892  %t = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
893  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
894  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
895  %res = sext <8 x i1> %obit to <8 x i32>
896  store <8 x i32> %val, <8 x i32>* %p2
897  ret <8 x i32> %res
898}
899
900define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
901; SSE2-LABEL: smulo_v16i32:
902; SSE2:       # %bb.0:
903; SSE2-NEXT:    pxor %xmm8, %xmm8
904; SSE2-NEXT:    pxor %xmm9, %xmm9
905; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
906; SSE2-NEXT:    pand %xmm0, %xmm9
907; SSE2-NEXT:    pxor %xmm10, %xmm10
908; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
909; SSE2-NEXT:    pand %xmm4, %xmm10
910; SSE2-NEXT:    paddd %xmm9, %xmm10
911; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
912; SSE2-NEXT:    pmuludq %xmm4, %xmm0
913; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
914; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
915; SSE2-NEXT:    pmuludq %xmm11, %xmm12
916; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
917; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
918; SSE2-NEXT:    psubd %xmm10, %xmm9
919; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
920; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
921; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
922; SSE2-NEXT:    movdqa %xmm0, (%rdi)
923; SSE2-NEXT:    psrad $31, %xmm0
924; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
925; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
926; SSE2-NEXT:    pxor %xmm9, %xmm0
927; SSE2-NEXT:    pxor %xmm10, %xmm10
928; SSE2-NEXT:    pcmpgtd %xmm5, %xmm10
929; SSE2-NEXT:    pand %xmm1, %xmm10
930; SSE2-NEXT:    pxor %xmm4, %xmm4
931; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
932; SSE2-NEXT:    pand %xmm5, %xmm4
933; SSE2-NEXT:    paddd %xmm10, %xmm4
934; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
935; SSE2-NEXT:    pmuludq %xmm5, %xmm1
936; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
937; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
938; SSE2-NEXT:    pmuludq %xmm10, %xmm12
939; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
940; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
941; SSE2-NEXT:    psubd %xmm4, %xmm11
942; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
943; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
944; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
945; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
946; SSE2-NEXT:    psrad $31, %xmm1
947; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
948; SSE2-NEXT:    pxor %xmm9, %xmm1
949; SSE2-NEXT:    pxor %xmm4, %xmm4
950; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
951; SSE2-NEXT:    pand %xmm2, %xmm4
952; SSE2-NEXT:    pxor %xmm5, %xmm5
953; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
954; SSE2-NEXT:    pand %xmm6, %xmm5
955; SSE2-NEXT:    paddd %xmm4, %xmm5
956; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
957; SSE2-NEXT:    pmuludq %xmm6, %xmm2
958; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
959; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
960; SSE2-NEXT:    pmuludq %xmm10, %xmm11
961; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
962; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
963; SSE2-NEXT:    psubd %xmm5, %xmm4
964; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
965; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
966; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
967; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
968; SSE2-NEXT:    psrad $31, %xmm2
969; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
970; SSE2-NEXT:    pxor %xmm9, %xmm2
971; SSE2-NEXT:    pxor %xmm4, %xmm4
972; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
973; SSE2-NEXT:    pand %xmm3, %xmm4
974; SSE2-NEXT:    pcmpgtd %xmm3, %xmm8
975; SSE2-NEXT:    pand %xmm7, %xmm8
976; SSE2-NEXT:    paddd %xmm4, %xmm8
977; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
978; SSE2-NEXT:    pmuludq %xmm7, %xmm3
979; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
980; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
981; SSE2-NEXT:    pmuludq %xmm4, %xmm6
982; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
983; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
984; SSE2-NEXT:    psubd %xmm8, %xmm5
985; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
986; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
987; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
988; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
989; SSE2-NEXT:    psrad $31, %xmm3
990; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
991; SSE2-NEXT:    pxor %xmm9, %xmm3
992; SSE2-NEXT:    retq
993;
994; SSSE3-LABEL: smulo_v16i32:
995; SSSE3:       # %bb.0:
996; SSSE3-NEXT:    pxor %xmm8, %xmm8
997; SSSE3-NEXT:    pxor %xmm9, %xmm9
998; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
999; SSSE3-NEXT:    pand %xmm0, %xmm9
1000; SSSE3-NEXT:    pxor %xmm10, %xmm10
1001; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm10
1002; SSSE3-NEXT:    pand %xmm4, %xmm10
1003; SSSE3-NEXT:    paddd %xmm9, %xmm10
1004; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
1005; SSSE3-NEXT:    pmuludq %xmm4, %xmm0
1006; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
1007; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
1008; SSSE3-NEXT:    pmuludq %xmm11, %xmm12
1009; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
1010; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
1011; SSSE3-NEXT:    psubd %xmm10, %xmm9
1012; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1013; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1014; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1015; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
1016; SSSE3-NEXT:    psrad $31, %xmm0
1017; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
1018; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
1019; SSSE3-NEXT:    pxor %xmm9, %xmm0
1020; SSSE3-NEXT:    pxor %xmm10, %xmm10
1021; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm10
1022; SSSE3-NEXT:    pand %xmm1, %xmm10
1023; SSSE3-NEXT:    pxor %xmm4, %xmm4
1024; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
1025; SSSE3-NEXT:    pand %xmm5, %xmm4
1026; SSSE3-NEXT:    paddd %xmm10, %xmm4
1027; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1028; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
1029; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
1030; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
1031; SSSE3-NEXT:    pmuludq %xmm10, %xmm12
1032; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
1033; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
1034; SSSE3-NEXT:    psubd %xmm4, %xmm11
1035; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1036; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1037; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1038; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
1039; SSSE3-NEXT:    psrad $31, %xmm1
1040; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
1041; SSSE3-NEXT:    pxor %xmm9, %xmm1
1042; SSSE3-NEXT:    pxor %xmm4, %xmm4
1043; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
1044; SSSE3-NEXT:    pand %xmm2, %xmm4
1045; SSSE3-NEXT:    pxor %xmm5, %xmm5
1046; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
1047; SSSE3-NEXT:    pand %xmm6, %xmm5
1048; SSSE3-NEXT:    paddd %xmm4, %xmm5
1049; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
1050; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
1051; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
1052; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
1053; SSSE3-NEXT:    pmuludq %xmm10, %xmm11
1054; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
1055; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
1056; SSSE3-NEXT:    psubd %xmm5, %xmm4
1057; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
1058; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
1059; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1060; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
1061; SSSE3-NEXT:    psrad $31, %xmm2
1062; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
1063; SSSE3-NEXT:    pxor %xmm9, %xmm2
1064; SSSE3-NEXT:    pxor %xmm4, %xmm4
1065; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
1066; SSSE3-NEXT:    pand %xmm3, %xmm4
1067; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm8
1068; SSSE3-NEXT:    pand %xmm7, %xmm8
1069; SSSE3-NEXT:    paddd %xmm4, %xmm8
1070; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
1071; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
1072; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
1073; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
1074; SSSE3-NEXT:    pmuludq %xmm4, %xmm6
1075; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
1076; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
1077; SSSE3-NEXT:    psubd %xmm8, %xmm5
1078; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
1079; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
1080; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1081; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
1082; SSSE3-NEXT:    psrad $31, %xmm3
1083; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
1084; SSSE3-NEXT:    pxor %xmm9, %xmm3
1085; SSSE3-NEXT:    retq
1086;
1087; SSE41-LABEL: smulo_v16i32:
1088; SSE41:       # %bb.0:
1089; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
1090; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
1091; SSE41-NEXT:    pmuldq %xmm8, %xmm9
1092; SSE41-NEXT:    movdqa %xmm0, %xmm8
1093; SSE41-NEXT:    pmuldq %xmm4, %xmm8
1094; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
1095; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
1096; SSE41-NEXT:    pmulld %xmm4, %xmm0
1097; SSE41-NEXT:    movdqa %xmm0, (%rdi)
1098; SSE41-NEXT:    psrad $31, %xmm0
1099; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
1100; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1101; SSE41-NEXT:    pxor %xmm8, %xmm0
1102; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
1103; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1104; SSE41-NEXT:    pmuldq %xmm9, %xmm10
1105; SSE41-NEXT:    movdqa %xmm1, %xmm4
1106; SSE41-NEXT:    pmuldq %xmm5, %xmm4
1107; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1108; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm10[2,3],xmm4[4,5],xmm10[6,7]
1109; SSE41-NEXT:    pmulld %xmm5, %xmm1
1110; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
1111; SSE41-NEXT:    psrad $31, %xmm1
1112; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
1113; SSE41-NEXT:    pxor %xmm8, %xmm1
1114; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
1115; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
1116; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1117; SSE41-NEXT:    movdqa %xmm2, %xmm4
1118; SSE41-NEXT:    pmuldq %xmm6, %xmm4
1119; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1120; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1121; SSE41-NEXT:    pmulld %xmm6, %xmm2
1122; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
1123; SSE41-NEXT:    psrad $31, %xmm2
1124; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
1125; SSE41-NEXT:    pxor %xmm8, %xmm2
1126; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
1127; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
1128; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1129; SSE41-NEXT:    movdqa %xmm3, %xmm4
1130; SSE41-NEXT:    pmuldq %xmm7, %xmm4
1131; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1132; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1133; SSE41-NEXT:    pmulld %xmm7, %xmm3
1134; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
1135; SSE41-NEXT:    psrad $31, %xmm3
1136; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
1137; SSE41-NEXT:    pxor %xmm8, %xmm3
1138; SSE41-NEXT:    retq
1139;
1140; AVX1-LABEL: smulo_v16i32:
1141; AVX1:       # %bb.0:
1142; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
1143; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
1144; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
1145; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
1146; AVX1-NEXT:    vpmuldq %xmm5, %xmm7, %xmm5
1147; AVX1-NEXT:    vpmuldq %xmm4, %xmm6, %xmm7
1148; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1149; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1150; AVX1-NEXT:    vpmulld %xmm4, %xmm6, %xmm8
1151; AVX1-NEXT:    vpsrad $31, %xmm8, %xmm6
1152; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm6
1153; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
1154; AVX1-NEXT:    vpxor %xmm6, %xmm9, %xmm6
1155; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
1156; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
1157; AVX1-NEXT:    vpmuldq %xmm7, %xmm4, %xmm4
1158; AVX1-NEXT:    vpmuldq %xmm3, %xmm1, %xmm7
1159; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1160; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3],xmm7[4,5],xmm4[6,7]
1161; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1162; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm1
1163; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
1164; AVX1-NEXT:    vpxor %xmm1, %xmm9, %xmm1
1165; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
1166; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1167; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
1168; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1169; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
1170; AVX1-NEXT:    vpmuldq %xmm6, %xmm5, %xmm5
1171; AVX1-NEXT:    vpmuldq %xmm4, %xmm7, %xmm6
1172; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
1173; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
1174; AVX1-NEXT:    vpmulld %xmm4, %xmm7, %xmm4
1175; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm6
1176; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm5
1177; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
1178; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
1179; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
1180; AVX1-NEXT:    vpmuldq %xmm6, %xmm7, %xmm6
1181; AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm7
1182; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1183; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
1184; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1185; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm0
1186; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
1187; AVX1-NEXT:    vpxor %xmm0, %xmm9, %xmm0
1188; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
1189; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
1190; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
1191; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1192; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
1193; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
1194; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1195; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1196; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1197; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1198; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1199; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
1200; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1201; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1202; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1203; AVX1-NEXT:    retq
1204;
1205; AVX2-LABEL: smulo_v16i32:
1206; AVX2:       # %bb.0:
1207; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1208; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1209; AVX2-NEXT:    vpmuldq %ymm4, %ymm5, %ymm4
1210; AVX2-NEXT:    vpmuldq %ymm3, %ymm1, %ymm5
1211; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1212; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1213; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1214; AVX2-NEXT:    vpsrad $31, %ymm3, %ymm1
1215; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm4, %ymm1
1216; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
1217; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
1218; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1219; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
1220; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[1,1,3,3,5,5,7,7]
1221; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm0[1,1,3,3,5,5,7,7]
1222; AVX2-NEXT:    vpmuldq %ymm5, %ymm6, %ymm5
1223; AVX2-NEXT:    vpmuldq %ymm2, %ymm0, %ymm6
1224; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,1,3,3,5,5,7,7]
1225; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
1226; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1227; AVX2-NEXT:    vpsrad $31, %ymm2, %ymm0
1228; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
1229; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
1230; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
1231; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
1232; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
1233; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
1234; AVX2-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1235; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1236; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1237; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1238; AVX2-NEXT:    retq
1239;
1240; AVX512-LABEL: smulo_v16i32:
1241; AVX512:       # %bb.0:
1242; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm2
1243; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1244; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1245; AVX512-NEXT:    vpmuldq %zmm3, %zmm4, %zmm3
1246; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1247; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1248; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1249; AVX512-NEXT:    vpsrad $31, %zmm1, %zmm0
1250; AVX512-NEXT:    vpcmpneqd %zmm0, %zmm4, %k1
1251; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1252; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1253; AVX512-NEXT:    retq
1254  %t = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1255  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1256  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1257  %res = sext <16 x i1> %obit to <16 x i32>
1258  store <16 x i32> %val, <16 x i32>* %p2
1259  ret <16 x i32> %res
1260}
1261
1262define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1263; SSE2-LABEL: smulo_v16i8:
1264; SSE2:       # %bb.0:
1265; SSE2-NEXT:    movdqa %xmm1, %xmm2
1266; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1267; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1268; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1269; SSE2-NEXT:    movdqa %xmm0, %xmm4
1270; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1271; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1272; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1273; SSE2-NEXT:    pmullw %xmm1, %xmm0
1274; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1275; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1276; SSE2-NEXT:    pmullw %xmm2, %xmm4
1277; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1278; SSE2-NEXT:    pand %xmm1, %xmm0
1279; SSE2-NEXT:    pand %xmm1, %xmm4
1280; SSE2-NEXT:    packuswb %xmm0, %xmm4
1281; SSE2-NEXT:    pxor %xmm0, %xmm0
1282; SSE2-NEXT:    pcmpgtb %xmm4, %xmm0
1283; SSE2-NEXT:    psraw $8, %xmm3
1284; SSE2-NEXT:    psraw $8, %xmm6
1285; SSE2-NEXT:    pmullw %xmm3, %xmm6
1286; SSE2-NEXT:    psrlw $8, %xmm6
1287; SSE2-NEXT:    psraw $8, %xmm5
1288; SSE2-NEXT:    psraw $8, %xmm7
1289; SSE2-NEXT:    pmullw %xmm5, %xmm7
1290; SSE2-NEXT:    psrlw $8, %xmm7
1291; SSE2-NEXT:    packuswb %xmm6, %xmm7
1292; SSE2-NEXT:    pcmpeqb %xmm0, %xmm7
1293; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1294; SSE2-NEXT:    pxor %xmm7, %xmm3
1295; SSE2-NEXT:    movdqa %xmm3, %xmm1
1296; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1297; SSE2-NEXT:    movdqa %xmm1, %xmm0
1298; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1299; SSE2-NEXT:    pslld $31, %xmm0
1300; SSE2-NEXT:    psrad $31, %xmm0
1301; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1302; SSE2-NEXT:    pslld $31, %xmm1
1303; SSE2-NEXT:    psrad $31, %xmm1
1304; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1305; SSE2-NEXT:    movdqa %xmm3, %xmm2
1306; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1307; SSE2-NEXT:    pslld $31, %xmm2
1308; SSE2-NEXT:    psrad $31, %xmm2
1309; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1310; SSE2-NEXT:    pslld $31, %xmm3
1311; SSE2-NEXT:    psrad $31, %xmm3
1312; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1313; SSE2-NEXT:    retq
1314;
1315; SSSE3-LABEL: smulo_v16i8:
1316; SSSE3:       # %bb.0:
1317; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1318; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1319; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1320; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1321; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1322; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1323; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1324; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1325; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1326; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1327; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1328; SSSE3-NEXT:    pmullw %xmm2, %xmm4
1329; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1330; SSSE3-NEXT:    pand %xmm1, %xmm0
1331; SSSE3-NEXT:    pand %xmm1, %xmm4
1332; SSSE3-NEXT:    packuswb %xmm0, %xmm4
1333; SSSE3-NEXT:    pxor %xmm0, %xmm0
1334; SSSE3-NEXT:    pcmpgtb %xmm4, %xmm0
1335; SSSE3-NEXT:    psraw $8, %xmm3
1336; SSSE3-NEXT:    psraw $8, %xmm6
1337; SSSE3-NEXT:    pmullw %xmm3, %xmm6
1338; SSSE3-NEXT:    psrlw $8, %xmm6
1339; SSSE3-NEXT:    psraw $8, %xmm5
1340; SSSE3-NEXT:    psraw $8, %xmm7
1341; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1342; SSSE3-NEXT:    psrlw $8, %xmm7
1343; SSSE3-NEXT:    packuswb %xmm6, %xmm7
1344; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm7
1345; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1346; SSSE3-NEXT:    pxor %xmm7, %xmm3
1347; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1348; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1349; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1350; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1351; SSSE3-NEXT:    pslld $31, %xmm0
1352; SSSE3-NEXT:    psrad $31, %xmm0
1353; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1354; SSSE3-NEXT:    pslld $31, %xmm1
1355; SSSE3-NEXT:    psrad $31, %xmm1
1356; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1357; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1358; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1359; SSSE3-NEXT:    pslld $31, %xmm2
1360; SSSE3-NEXT:    psrad $31, %xmm2
1361; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1362; SSSE3-NEXT:    pslld $31, %xmm3
1363; SSSE3-NEXT:    psrad $31, %xmm3
1364; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1365; SSSE3-NEXT:    retq
1366;
1367; SSE41-LABEL: smulo_v16i8:
1368; SSE41:       # %bb.0:
1369; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1370; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
1371; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
1372; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1373; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1374; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
1375; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
1376; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1377; SSE41-NEXT:    pmullw %xmm1, %xmm0
1378; SSE41-NEXT:    pmullw %xmm2, %xmm4
1379; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1380; SSE41-NEXT:    pand %xmm1, %xmm0
1381; SSE41-NEXT:    pand %xmm1, %xmm4
1382; SSE41-NEXT:    packuswb %xmm0, %xmm4
1383; SSE41-NEXT:    pxor %xmm0, %xmm0
1384; SSE41-NEXT:    pcmpgtb %xmm4, %xmm0
1385; SSE41-NEXT:    pmullw %xmm3, %xmm6
1386; SSE41-NEXT:    psrlw $8, %xmm6
1387; SSE41-NEXT:    pmovsxbw %xmm5, %xmm1
1388; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1389; SSE41-NEXT:    pmullw %xmm1, %xmm2
1390; SSE41-NEXT:    psrlw $8, %xmm2
1391; SSE41-NEXT:    packuswb %xmm2, %xmm6
1392; SSE41-NEXT:    pcmpeqb %xmm0, %xmm6
1393; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1394; SSE41-NEXT:    pxor %xmm6, %xmm3
1395; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1396; SSE41-NEXT:    pslld $31, %xmm0
1397; SSE41-NEXT:    psrad $31, %xmm0
1398; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
1399; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1400; SSE41-NEXT:    pslld $31, %xmm1
1401; SSE41-NEXT:    psrad $31, %xmm1
1402; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1403; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1404; SSE41-NEXT:    pslld $31, %xmm2
1405; SSE41-NEXT:    psrad $31, %xmm2
1406; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1407; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1408; SSE41-NEXT:    pslld $31, %xmm3
1409; SSE41-NEXT:    psrad $31, %xmm3
1410; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1411; SSE41-NEXT:    retq
1412;
1413; AVX1-LABEL: smulo_v16i8:
1414; AVX1:       # %bb.0:
1415; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1416; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1417; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1418; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1419; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1420; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1421; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1422; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1423; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1424; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1425; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1426; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1427; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm4
1428; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
1429; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1430; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1431; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1432; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1433; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1434; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1435; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1436; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1437; AVX1-NEXT:    vpackuswb %xmm0, %xmm4, %xmm0
1438; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1439; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1440; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1441; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1442; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
1443; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1444; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1445; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1446; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1447; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1448; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1449; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1450; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1451; AVX1-NEXT:    retq
1452;
1453; AVX2-LABEL: smulo_v16i8:
1454; AVX2:       # %bb.0:
1455; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm2
1456; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm3
1457; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1458; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1459; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
1460; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
1461; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1462; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1463; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1464; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1465; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1466; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm3
1467; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
1468; AVX2-NEXT:    vpcmpgtb %xmm3, %xmm0, %xmm0
1469; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
1470; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1471; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1472; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1473; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1474; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1475; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
1476; AVX2-NEXT:    retq
1477;
1478; AVX512F-LABEL: smulo_v16i8:
1479; AVX512F:       # %bb.0:
1480; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1481; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1482; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1483; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1484; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1485; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1486; AVX512F-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1487; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1488; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1489; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1490; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
1491; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1492; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1493; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1494; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1495; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1496; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
1497; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1498; AVX512F-NEXT:    vmovdqa %xmm2, (%rdi)
1499; AVX512F-NEXT:    retq
1500;
1501; AVX512BW-LABEL: smulo_v16i8:
1502; AVX512BW:       # %bb.0:
1503; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1504; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1505; AVX512BW-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1506; AVX512BW-NEXT:    vpmovwb %ymm2, %xmm2
1507; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1508; AVX512BW-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1509; AVX512BW-NEXT:    vpmovsxbw %xmm1, %ymm1
1510; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
1511; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1512; AVX512BW-NEXT:    vpsrlw $8, %ymm0, %ymm0
1513; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
1514; AVX512BW-NEXT:    vpcmpneqb %xmm3, %xmm0, %k1
1515; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1516; AVX512BW-NEXT:    vmovdqa %xmm2, (%rdi)
1517; AVX512BW-NEXT:    retq
1518  %t = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1519  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1520  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1521  %res = sext <16 x i1> %obit to <16 x i32>
1522  store <16 x i8> %val, <16 x i8>* %p2
1523  ret <16 x i32> %res
1524}
1525
1526define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nounwind {
1527; SSE2-LABEL: smulo_v32i8:
1528; SSE2:       # %bb.0:
1529; SSE2-NEXT:    movq %rdi, %rax
1530; SSE2-NEXT:    movdqa %xmm3, %xmm10
1531; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1532; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
1533; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1534; SSE2-NEXT:    movdqa %xmm1, %xmm9
1535; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1536; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1537; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1538; SSE2-NEXT:    pmullw %xmm3, %xmm1
1539; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1540; SSE2-NEXT:    pand %xmm8, %xmm1
1541; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1542; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1543; SSE2-NEXT:    pmullw %xmm10, %xmm9
1544; SSE2-NEXT:    pand %xmm8, %xmm9
1545; SSE2-NEXT:    packuswb %xmm1, %xmm9
1546; SSE2-NEXT:    pxor %xmm1, %xmm1
1547; SSE2-NEXT:    pcmpgtb %xmm9, %xmm1
1548; SSE2-NEXT:    psraw $8, %xmm7
1549; SSE2-NEXT:    psraw $8, %xmm6
1550; SSE2-NEXT:    pmullw %xmm7, %xmm6
1551; SSE2-NEXT:    psrlw $8, %xmm6
1552; SSE2-NEXT:    psraw $8, %xmm4
1553; SSE2-NEXT:    psraw $8, %xmm5
1554; SSE2-NEXT:    pmullw %xmm4, %xmm5
1555; SSE2-NEXT:    psrlw $8, %xmm5
1556; SSE2-NEXT:    packuswb %xmm6, %xmm5
1557; SSE2-NEXT:    pcmpeqb %xmm1, %xmm5
1558; SSE2-NEXT:    movdqa %xmm2, %xmm10
1559; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
1560; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1561; SSE2-NEXT:    movdqa %xmm2, %xmm7
1562; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1563; SSE2-NEXT:    movdqa %xmm0, %xmm2
1564; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1565; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1566; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1567; SSE2-NEXT:    pmullw %xmm7, %xmm0
1568; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1569; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1570; SSE2-NEXT:    pmullw %xmm10, %xmm2
1571; SSE2-NEXT:    pxor %xmm7, %xmm7
1572; SSE2-NEXT:    pand %xmm8, %xmm0
1573; SSE2-NEXT:    pand %xmm8, %xmm2
1574; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
1575; SSE2-NEXT:    pxor %xmm8, %xmm5
1576; SSE2-NEXT:    packuswb %xmm0, %xmm2
1577; SSE2-NEXT:    pcmpgtb %xmm2, %xmm7
1578; SSE2-NEXT:    psraw $8, %xmm4
1579; SSE2-NEXT:    psraw $8, %xmm3
1580; SSE2-NEXT:    pmullw %xmm4, %xmm3
1581; SSE2-NEXT:    psrlw $8, %xmm3
1582; SSE2-NEXT:    psraw $8, %xmm6
1583; SSE2-NEXT:    psraw $8, %xmm1
1584; SSE2-NEXT:    pmullw %xmm6, %xmm1
1585; SSE2-NEXT:    psrlw $8, %xmm1
1586; SSE2-NEXT:    packuswb %xmm3, %xmm1
1587; SSE2-NEXT:    pcmpeqb %xmm7, %xmm1
1588; SSE2-NEXT:    pxor %xmm8, %xmm1
1589; SSE2-NEXT:    movdqa %xmm1, %xmm0
1590; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1591; SSE2-NEXT:    movdqa %xmm0, %xmm8
1592; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
1593; SSE2-NEXT:    pslld $31, %xmm8
1594; SSE2-NEXT:    psrad $31, %xmm8
1595; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1596; SSE2-NEXT:    pslld $31, %xmm0
1597; SSE2-NEXT:    psrad $31, %xmm0
1598; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1599; SSE2-NEXT:    movdqa %xmm1, %xmm6
1600; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1601; SSE2-NEXT:    pslld $31, %xmm6
1602; SSE2-NEXT:    psrad $31, %xmm6
1603; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1604; SSE2-NEXT:    pslld $31, %xmm1
1605; SSE2-NEXT:    psrad $31, %xmm1
1606; SSE2-NEXT:    movdqa %xmm5, %xmm3
1607; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1608; SSE2-NEXT:    movdqa %xmm3, %xmm7
1609; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1610; SSE2-NEXT:    pslld $31, %xmm7
1611; SSE2-NEXT:    psrad $31, %xmm7
1612; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1613; SSE2-NEXT:    pslld $31, %xmm3
1614; SSE2-NEXT:    psrad $31, %xmm3
1615; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1616; SSE2-NEXT:    movdqa %xmm5, %xmm4
1617; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1618; SSE2-NEXT:    pslld $31, %xmm4
1619; SSE2-NEXT:    psrad $31, %xmm4
1620; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1621; SSE2-NEXT:    pslld $31, %xmm5
1622; SSE2-NEXT:    psrad $31, %xmm5
1623; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
1624; SSE2-NEXT:    movdqa %xmm2, (%rsi)
1625; SSE2-NEXT:    movdqa %xmm5, 112(%rdi)
1626; SSE2-NEXT:    movdqa %xmm4, 96(%rdi)
1627; SSE2-NEXT:    movdqa %xmm3, 80(%rdi)
1628; SSE2-NEXT:    movdqa %xmm7, 64(%rdi)
1629; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
1630; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
1631; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
1632; SSE2-NEXT:    movdqa %xmm8, (%rdi)
1633; SSE2-NEXT:    retq
1634;
1635; SSSE3-LABEL: smulo_v32i8:
1636; SSSE3:       # %bb.0:
1637; SSSE3-NEXT:    movq %rdi, %rax
1638; SSSE3-NEXT:    movdqa %xmm3, %xmm10
1639; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1640; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
1641; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1642; SSSE3-NEXT:    movdqa %xmm1, %xmm9
1643; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1644; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1645; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1646; SSSE3-NEXT:    pmullw %xmm3, %xmm1
1647; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1648; SSSE3-NEXT:    pand %xmm8, %xmm1
1649; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1650; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1651; SSSE3-NEXT:    pmullw %xmm10, %xmm9
1652; SSSE3-NEXT:    pand %xmm8, %xmm9
1653; SSSE3-NEXT:    packuswb %xmm1, %xmm9
1654; SSSE3-NEXT:    pxor %xmm1, %xmm1
1655; SSSE3-NEXT:    pcmpgtb %xmm9, %xmm1
1656; SSSE3-NEXT:    psraw $8, %xmm7
1657; SSSE3-NEXT:    psraw $8, %xmm6
1658; SSSE3-NEXT:    pmullw %xmm7, %xmm6
1659; SSSE3-NEXT:    psrlw $8, %xmm6
1660; SSSE3-NEXT:    psraw $8, %xmm4
1661; SSSE3-NEXT:    psraw $8, %xmm5
1662; SSSE3-NEXT:    pmullw %xmm4, %xmm5
1663; SSSE3-NEXT:    psrlw $8, %xmm5
1664; SSSE3-NEXT:    packuswb %xmm6, %xmm5
1665; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm5
1666; SSSE3-NEXT:    movdqa %xmm2, %xmm10
1667; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
1668; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1669; SSSE3-NEXT:    movdqa %xmm2, %xmm7
1670; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1671; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1672; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1673; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1674; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1675; SSSE3-NEXT:    pmullw %xmm7, %xmm0
1676; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1677; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1678; SSSE3-NEXT:    pmullw %xmm10, %xmm2
1679; SSSE3-NEXT:    pxor %xmm7, %xmm7
1680; SSSE3-NEXT:    pand %xmm8, %xmm0
1681; SSSE3-NEXT:    pand %xmm8, %xmm2
1682; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
1683; SSSE3-NEXT:    pxor %xmm8, %xmm5
1684; SSSE3-NEXT:    packuswb %xmm0, %xmm2
1685; SSSE3-NEXT:    pcmpgtb %xmm2, %xmm7
1686; SSSE3-NEXT:    psraw $8, %xmm4
1687; SSSE3-NEXT:    psraw $8, %xmm3
1688; SSSE3-NEXT:    pmullw %xmm4, %xmm3
1689; SSSE3-NEXT:    psrlw $8, %xmm3
1690; SSSE3-NEXT:    psraw $8, %xmm6
1691; SSSE3-NEXT:    psraw $8, %xmm1
1692; SSSE3-NEXT:    pmullw %xmm6, %xmm1
1693; SSSE3-NEXT:    psrlw $8, %xmm1
1694; SSSE3-NEXT:    packuswb %xmm3, %xmm1
1695; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm1
1696; SSSE3-NEXT:    pxor %xmm8, %xmm1
1697; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1698; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1699; SSSE3-NEXT:    movdqa %xmm0, %xmm8
1700; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
1701; SSSE3-NEXT:    pslld $31, %xmm8
1702; SSSE3-NEXT:    psrad $31, %xmm8
1703; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1704; SSSE3-NEXT:    pslld $31, %xmm0
1705; SSSE3-NEXT:    psrad $31, %xmm0
1706; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1707; SSSE3-NEXT:    movdqa %xmm1, %xmm6
1708; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1709; SSSE3-NEXT:    pslld $31, %xmm6
1710; SSSE3-NEXT:    psrad $31, %xmm6
1711; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1712; SSSE3-NEXT:    pslld $31, %xmm1
1713; SSSE3-NEXT:    psrad $31, %xmm1
1714; SSSE3-NEXT:    movdqa %xmm5, %xmm3
1715; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1716; SSSE3-NEXT:    movdqa %xmm3, %xmm7
1717; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1718; SSSE3-NEXT:    pslld $31, %xmm7
1719; SSSE3-NEXT:    psrad $31, %xmm7
1720; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1721; SSSE3-NEXT:    pslld $31, %xmm3
1722; SSSE3-NEXT:    psrad $31, %xmm3
1723; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1724; SSSE3-NEXT:    movdqa %xmm5, %xmm4
1725; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1726; SSSE3-NEXT:    pslld $31, %xmm4
1727; SSSE3-NEXT:    psrad $31, %xmm4
1728; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1729; SSSE3-NEXT:    pslld $31, %xmm5
1730; SSSE3-NEXT:    psrad $31, %xmm5
1731; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
1732; SSSE3-NEXT:    movdqa %xmm2, (%rsi)
1733; SSSE3-NEXT:    movdqa %xmm5, 112(%rdi)
1734; SSSE3-NEXT:    movdqa %xmm4, 96(%rdi)
1735; SSSE3-NEXT:    movdqa %xmm3, 80(%rdi)
1736; SSSE3-NEXT:    movdqa %xmm7, 64(%rdi)
1737; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
1738; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
1739; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
1740; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
1741; SSSE3-NEXT:    retq
1742;
1743; SSE41-LABEL: smulo_v32i8:
1744; SSE41:       # %bb.0:
1745; SSE41-NEXT:    movq %rdi, %rax
1746; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1747; SSE41-NEXT:    pmovsxbw %xmm3, %xmm7
1748; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
1749; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1750; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1751; SSE41-NEXT:    pmovsxbw %xmm1, %xmm4
1752; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
1753; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1754; SSE41-NEXT:    pmullw %xmm3, %xmm1
1755; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1756; SSE41-NEXT:    pand %xmm8, %xmm1
1757; SSE41-NEXT:    pmullw %xmm10, %xmm9
1758; SSE41-NEXT:    pand %xmm8, %xmm9
1759; SSE41-NEXT:    packuswb %xmm1, %xmm9
1760; SSE41-NEXT:    pxor %xmm1, %xmm1
1761; SSE41-NEXT:    pcmpgtb %xmm9, %xmm1
1762; SSE41-NEXT:    pmullw %xmm7, %xmm4
1763; SSE41-NEXT:    psrlw $8, %xmm4
1764; SSE41-NEXT:    pmovsxbw %xmm5, %xmm3
1765; SSE41-NEXT:    pmovsxbw %xmm6, %xmm5
1766; SSE41-NEXT:    pmullw %xmm3, %xmm5
1767; SSE41-NEXT:    psrlw $8, %xmm5
1768; SSE41-NEXT:    packuswb %xmm5, %xmm4
1769; SSE41-NEXT:    pcmpeqb %xmm1, %xmm4
1770; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1771; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
1772; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
1773; SSE41-NEXT:    movdqa %xmm2, %xmm7
1774; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1775; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1776; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
1777; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
1778; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1779; SSE41-NEXT:    pmullw %xmm7, %xmm0
1780; SSE41-NEXT:    pmullw %xmm10, %xmm1
1781; SSE41-NEXT:    pxor %xmm7, %xmm7
1782; SSE41-NEXT:    pand %xmm8, %xmm0
1783; SSE41-NEXT:    pand %xmm8, %xmm1
1784; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1785; SSE41-NEXT:    pxor %xmm8, %xmm4
1786; SSE41-NEXT:    packuswb %xmm0, %xmm1
1787; SSE41-NEXT:    pcmpgtb %xmm1, %xmm7
1788; SSE41-NEXT:    pmullw %xmm5, %xmm2
1789; SSE41-NEXT:    psrlw $8, %xmm2
1790; SSE41-NEXT:    pmovsxbw %xmm6, %xmm0
1791; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
1792; SSE41-NEXT:    pmullw %xmm0, %xmm3
1793; SSE41-NEXT:    psrlw $8, %xmm3
1794; SSE41-NEXT:    packuswb %xmm3, %xmm2
1795; SSE41-NEXT:    pcmpeqb %xmm7, %xmm2
1796; SSE41-NEXT:    pxor %xmm8, %xmm2
1797; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
1798; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1799; SSE41-NEXT:    pslld $31, %xmm8
1800; SSE41-NEXT:    psrad $31, %xmm8
1801; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
1802; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1803; SSE41-NEXT:    pslld $31, %xmm5
1804; SSE41-NEXT:    psrad $31, %xmm5
1805; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
1806; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1807; SSE41-NEXT:    pslld $31, %xmm6
1808; SSE41-NEXT:    psrad $31, %xmm6
1809; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
1810; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1811; SSE41-NEXT:    pslld $31, %xmm7
1812; SSE41-NEXT:    psrad $31, %xmm7
1813; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
1814; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1815; SSE41-NEXT:    pslld $31, %xmm3
1816; SSE41-NEXT:    psrad $31, %xmm3
1817; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
1818; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1819; SSE41-NEXT:    pslld $31, %xmm0
1820; SSE41-NEXT:    psrad $31, %xmm0
1821; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1822; SSE41-NEXT:    pslld $31, %xmm2
1823; SSE41-NEXT:    psrad $31, %xmm2
1824; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
1825; SSE41-NEXT:    pslld $31, %xmm4
1826; SSE41-NEXT:    psrad $31, %xmm4
1827; SSE41-NEXT:    movdqa %xmm9, 16(%rsi)
1828; SSE41-NEXT:    movdqa %xmm1, (%rsi)
1829; SSE41-NEXT:    movdqa %xmm4, 64(%rdi)
1830; SSE41-NEXT:    movdqa %xmm2, (%rdi)
1831; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
1832; SSE41-NEXT:    movdqa %xmm3, 96(%rdi)
1833; SSE41-NEXT:    movdqa %xmm7, 80(%rdi)
1834; SSE41-NEXT:    movdqa %xmm6, 48(%rdi)
1835; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)
1836; SSE41-NEXT:    movdqa %xmm8, 16(%rdi)
1837; SSE41-NEXT:    retq
1838;
1839; AVX1-LABEL: smulo_v32i8:
1840; AVX1:       # %bb.0:
1841; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1842; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1843; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
1844; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1845; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
1846; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
1847; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
1848; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1849; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
1850; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
1851; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
1852; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm9
1853; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
1854; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm8, %xmm3
1855; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm7
1856; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
1857; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
1858; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1859; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
1860; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
1861; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
1862; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
1863; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1864; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1865; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
1866; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
1867; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1868; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm3
1869; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1870; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1871; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1872; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
1873; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1874; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1875; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
1876; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
1877; AVX1-NEXT:    vpackuswb %xmm2, %xmm5, %xmm5
1878; AVX1-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
1879; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm6
1880; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm7
1881; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
1882; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
1883; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1884; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1885; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1886; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1887; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1888; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1889; AVX1-NEXT:    vpackuswb %xmm0, %xmm6, %xmm0
1890; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
1891; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
1892; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1893; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1894; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
1895; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1896; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm2
1897; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
1898; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1899; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
1900; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
1901; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1902; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1903; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1904; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
1905; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
1906; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1907; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1908; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1909; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1910; AVX1-NEXT:    vmovdqa %xmm9, 16(%rdi)
1911; AVX1-NEXT:    vmovdqa %xmm5, (%rdi)
1912; AVX1-NEXT:    retq
1913;
1914; AVX2-LABEL: smulo_v32i8:
1915; AVX2:       # %bb.0:
1916; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1917; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1918; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1919; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1920; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
1921; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1922; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1923; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1924; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
1925; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
1926; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1927; AVX2-NEXT:    vpcmpgtb %ymm4, %ymm2, %ymm2
1928; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1929; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
1930; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1931; AVX2-NEXT:    vpsraw $8, %ymm5, %ymm5
1932; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
1933; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
1934; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1935; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
1936; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1937; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
1938; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1939; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1940; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
1941; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
1942; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
1943; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
1944; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1945; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1946; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
1947; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1948; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1949; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1950; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
1951; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
1952; AVX2-NEXT:    retq
1953;
1954; AVX512F-LABEL: smulo_v32i8:
1955; AVX512F:       # %bb.0:
1956; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
1957; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
1958; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm4
1959; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1960; AVX512F-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
1961; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1962; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1963; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1964; AVX512F-NEXT:    vpcmpgtb %xmm2, %xmm5, %xmm6
1965; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
1966; AVX512F-NEXT:    vpmovsxbw %xmm4, %ymm4
1967; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
1968; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
1969; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1970; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1971; AVX512F-NEXT:    vpcmpeqb %xmm6, %xmm3, %xmm3
1972; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
1973; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
1974; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k1
1975; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1976; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1977; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
1978; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1979; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1980; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm5, %xmm4
1981; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1982; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1983; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1984; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
1985; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1986; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1987; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
1988; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1989; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1990; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
1991; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
1992; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1993; AVX512F-NEXT:    vmovdqa %xmm2, 16(%rdi)
1994; AVX512F-NEXT:    vmovdqa %xmm3, (%rdi)
1995; AVX512F-NEXT:    retq
1996;
1997; AVX512BW-LABEL: smulo_v32i8:
1998; AVX512BW:       # %bb.0:
1999; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2000; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2001; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2002; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
2003; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
2004; AVX512BW-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm3
2005; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
2006; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
2007; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2008; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
2009; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
2010; AVX512BW-NEXT:    vpcmpneqb %ymm3, %ymm0, %k1
2011; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2012; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2013; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
2014; AVX512BW-NEXT:    vmovdqa %ymm2, (%rdi)
2015; AVX512BW-NEXT:    retq
2016  %t = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)
2017  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 0
2018  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 1
2019  %res = sext <32 x i1> %obit to <32 x i32>
2020  store <32 x i8> %val, <32 x i8>* %p2
2021  ret <32 x i32> %res
2022}
2023
2024define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nounwind {
2025; SSE2-LABEL: smulo_v64i8:
2026; SSE2:       # %bb.0:
2027; SSE2-NEXT:    movq %rdi, %rax
2028; SSE2-NEXT:    movdqa %xmm7, %xmm9
2029; SSE2-NEXT:    movdqa %xmm7, %xmm10
2030; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2031; SSE2-NEXT:    movdqa %xmm3, %xmm8
2032; SSE2-NEXT:    movdqa %xmm3, %xmm11
2033; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2034; SSE2-NEXT:    pmullw %xmm10, %xmm11
2035; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2036; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2037; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2038; SSE2-NEXT:    pmullw %xmm9, %xmm8
2039; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2040; SSE2-NEXT:    pand %xmm9, %xmm11
2041; SSE2-NEXT:    pand %xmm9, %xmm8
2042; SSE2-NEXT:    packuswb %xmm11, %xmm8
2043; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
2044; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2045; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
2046; SSE2-NEXT:    psraw $8, %xmm10
2047; SSE2-NEXT:    psraw $8, %xmm11
2048; SSE2-NEXT:    pmullw %xmm10, %xmm11
2049; SSE2-NEXT:    pxor %xmm3, %xmm3
2050; SSE2-NEXT:    pcmpgtb %xmm8, %xmm3
2051; SSE2-NEXT:    psrlw $8, %xmm11
2052; SSE2-NEXT:    psraw $8, %xmm7
2053; SSE2-NEXT:    psraw $8, %xmm12
2054; SSE2-NEXT:    pmullw %xmm7, %xmm12
2055; SSE2-NEXT:    psrlw $8, %xmm12
2056; SSE2-NEXT:    packuswb %xmm11, %xmm12
2057; SSE2-NEXT:    pcmpeqb %xmm3, %xmm12
2058; SSE2-NEXT:    movdqa %xmm6, %xmm10
2059; SSE2-NEXT:    movdqa %xmm6, %xmm7
2060; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2061; SSE2-NEXT:    movdqa %xmm2, %xmm3
2062; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2063; SSE2-NEXT:    pmullw %xmm7, %xmm3
2064; SSE2-NEXT:    movdqa %xmm2, %xmm11
2065; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2066; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2067; SSE2-NEXT:    pmullw %xmm10, %xmm11
2068; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2069; SSE2-NEXT:    pand %xmm9, %xmm3
2070; SSE2-NEXT:    pand %xmm9, %xmm11
2071; SSE2-NEXT:    packuswb %xmm3, %xmm11
2072; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2073; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2074; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2075; SSE2-NEXT:    psraw $8, %xmm7
2076; SSE2-NEXT:    psraw $8, %xmm3
2077; SSE2-NEXT:    pmullw %xmm7, %xmm3
2078; SSE2-NEXT:    pxor %xmm7, %xmm7
2079; SSE2-NEXT:    pcmpgtb %xmm11, %xmm7
2080; SSE2-NEXT:    psrlw $8, %xmm3
2081; SSE2-NEXT:    psraw $8, %xmm6
2082; SSE2-NEXT:    psraw $8, %xmm2
2083; SSE2-NEXT:    pmullw %xmm6, %xmm2
2084; SSE2-NEXT:    psrlw $8, %xmm2
2085; SSE2-NEXT:    packuswb %xmm3, %xmm2
2086; SSE2-NEXT:    pcmpeqb %xmm7, %xmm2
2087; SSE2-NEXT:    movdqa %xmm5, %xmm3
2088; SSE2-NEXT:    movdqa %xmm5, %xmm6
2089; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2090; SSE2-NEXT:    movdqa %xmm1, %xmm7
2091; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2092; SSE2-NEXT:    pmullw %xmm6, %xmm7
2093; SSE2-NEXT:    movdqa %xmm1, %xmm6
2094; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2095; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2096; SSE2-NEXT:    pmullw %xmm3, %xmm6
2097; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2098; SSE2-NEXT:    pand %xmm9, %xmm7
2099; SSE2-NEXT:    pand %xmm9, %xmm6
2100; SSE2-NEXT:    packuswb %xmm7, %xmm6
2101; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2102; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2103; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2104; SSE2-NEXT:    psraw $8, %xmm3
2105; SSE2-NEXT:    psraw $8, %xmm7
2106; SSE2-NEXT:    pmullw %xmm3, %xmm7
2107; SSE2-NEXT:    pxor %xmm3, %xmm3
2108; SSE2-NEXT:    pcmpgtb %xmm6, %xmm3
2109; SSE2-NEXT:    psrlw $8, %xmm7
2110; SSE2-NEXT:    psraw $8, %xmm5
2111; SSE2-NEXT:    psraw $8, %xmm1
2112; SSE2-NEXT:    pmullw %xmm5, %xmm1
2113; SSE2-NEXT:    psrlw $8, %xmm1
2114; SSE2-NEXT:    packuswb %xmm7, %xmm1
2115; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
2116; SSE2-NEXT:    movdqa %xmm4, %xmm3
2117; SSE2-NEXT:    movdqa %xmm4, %xmm5
2118; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2119; SSE2-NEXT:    movdqa %xmm0, %xmm7
2120; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2121; SSE2-NEXT:    pmullw %xmm5, %xmm7
2122; SSE2-NEXT:    movdqa %xmm0, %xmm5
2123; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2124; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2125; SSE2-NEXT:    pmullw %xmm3, %xmm5
2126; SSE2-NEXT:    pand %xmm9, %xmm7
2127; SSE2-NEXT:    pand %xmm9, %xmm5
2128; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2129; SSE2-NEXT:    packuswb %xmm7, %xmm5
2130; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2131; SSE2-NEXT:    psraw $8, %xmm3
2132; SSE2-NEXT:    psraw $8, %xmm7
2133; SSE2-NEXT:    pmullw %xmm3, %xmm7
2134; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2135; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2136; SSE2-NEXT:    psraw $8, %xmm3
2137; SSE2-NEXT:    psraw $8, %xmm0
2138; SSE2-NEXT:    pmullw %xmm3, %xmm0
2139; SSE2-NEXT:    psrlw $8, %xmm7
2140; SSE2-NEXT:    psrlw $8, %xmm0
2141; SSE2-NEXT:    packuswb %xmm7, %xmm0
2142; SSE2-NEXT:    pxor %xmm3, %xmm3
2143; SSE2-NEXT:    pcmpgtb %xmm5, %xmm3
2144; SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
2145; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
2146; SSE2-NEXT:    pxor %xmm3, %xmm12
2147; SSE2-NEXT:    pxor %xmm3, %xmm2
2148; SSE2-NEXT:    pxor %xmm3, %xmm1
2149; SSE2-NEXT:    pxor %xmm3, %xmm0
2150; SSE2-NEXT:    movdqa %xmm8, 48(%rsi)
2151; SSE2-NEXT:    movdqa %xmm11, 32(%rsi)
2152; SSE2-NEXT:    movdqa %xmm6, 16(%rsi)
2153; SSE2-NEXT:    movdqa %xmm12, %xmm3
2154; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2155; SSE2-NEXT:    movdqa %xmm5, (%rsi)
2156; SSE2-NEXT:    movdqa %xmm12, %xmm4
2157; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
2158; SSE2-NEXT:    pslld $31, %xmm12
2159; SSE2-NEXT:    psrad $31, %xmm12
2160; SSE2-NEXT:    movdqa %xmm12, 224(%rdi)
2161; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2162; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2163; SSE2-NEXT:    pslld $31, %xmm4
2164; SSE2-NEXT:    psrad $31, %xmm4
2165; SSE2-NEXT:    movdqa %xmm4, 240(%rdi)
2166; SSE2-NEXT:    movdqa %xmm3, %xmm4
2167; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2168; SSE2-NEXT:    pslld $31, %xmm3
2169; SSE2-NEXT:    psrad $31, %xmm3
2170; SSE2-NEXT:    movdqa %xmm3, 192(%rdi)
2171; SSE2-NEXT:    movdqa %xmm2, %xmm3
2172; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2173; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2174; SSE2-NEXT:    pslld $31, %xmm4
2175; SSE2-NEXT:    psrad $31, %xmm4
2176; SSE2-NEXT:    movdqa %xmm4, 208(%rdi)
2177; SSE2-NEXT:    movdqa %xmm2, %xmm4
2178; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2179; SSE2-NEXT:    pslld $31, %xmm2
2180; SSE2-NEXT:    psrad $31, %xmm2
2181; SSE2-NEXT:    movdqa %xmm2, 160(%rdi)
2182; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2183; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2184; SSE2-NEXT:    pslld $31, %xmm4
2185; SSE2-NEXT:    psrad $31, %xmm4
2186; SSE2-NEXT:    movdqa %xmm4, 176(%rdi)
2187; SSE2-NEXT:    movdqa %xmm3, %xmm2
2188; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2189; SSE2-NEXT:    pslld $31, %xmm3
2190; SSE2-NEXT:    psrad $31, %xmm3
2191; SSE2-NEXT:    movdqa %xmm3, 128(%rdi)
2192; SSE2-NEXT:    movdqa %xmm1, %xmm3
2193; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2194; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2195; SSE2-NEXT:    pslld $31, %xmm2
2196; SSE2-NEXT:    psrad $31, %xmm2
2197; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
2198; SSE2-NEXT:    movdqa %xmm1, %xmm2
2199; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2200; SSE2-NEXT:    pslld $31, %xmm1
2201; SSE2-NEXT:    psrad $31, %xmm1
2202; SSE2-NEXT:    movdqa %xmm1, 96(%rdi)
2203; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2204; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2205; SSE2-NEXT:    pslld $31, %xmm2
2206; SSE2-NEXT:    psrad $31, %xmm2
2207; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
2208; SSE2-NEXT:    movdqa %xmm3, %xmm1
2209; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2210; SSE2-NEXT:    pslld $31, %xmm3
2211; SSE2-NEXT:    psrad $31, %xmm3
2212; SSE2-NEXT:    movdqa %xmm3, 64(%rdi)
2213; SSE2-NEXT:    movdqa %xmm0, %xmm2
2214; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2215; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2216; SSE2-NEXT:    pslld $31, %xmm1
2217; SSE2-NEXT:    psrad $31, %xmm1
2218; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
2219; SSE2-NEXT:    movdqa %xmm0, %xmm1
2220; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2221; SSE2-NEXT:    pslld $31, %xmm0
2222; SSE2-NEXT:    psrad $31, %xmm0
2223; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
2224; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2225; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2226; SSE2-NEXT:    pslld $31, %xmm1
2227; SSE2-NEXT:    psrad $31, %xmm1
2228; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
2229; SSE2-NEXT:    movdqa %xmm2, %xmm0
2230; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2231; SSE2-NEXT:    pslld $31, %xmm2
2232; SSE2-NEXT:    psrad $31, %xmm2
2233; SSE2-NEXT:    movdqa %xmm2, (%rdi)
2234; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2235; SSE2-NEXT:    pslld $31, %xmm0
2236; SSE2-NEXT:    psrad $31, %xmm0
2237; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
2238; SSE2-NEXT:    retq
2239;
2240; SSSE3-LABEL: smulo_v64i8:
2241; SSSE3:       # %bb.0:
2242; SSSE3-NEXT:    movq %rdi, %rax
2243; SSSE3-NEXT:    movdqa %xmm7, %xmm9
2244; SSSE3-NEXT:    movdqa %xmm7, %xmm10
2245; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2246; SSSE3-NEXT:    movdqa %xmm3, %xmm8
2247; SSSE3-NEXT:    movdqa %xmm3, %xmm11
2248; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2249; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2250; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2251; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2252; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2253; SSSE3-NEXT:    pmullw %xmm9, %xmm8
2254; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2255; SSSE3-NEXT:    pand %xmm9, %xmm11
2256; SSSE3-NEXT:    pand %xmm9, %xmm8
2257; SSSE3-NEXT:    packuswb %xmm11, %xmm8
2258; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
2259; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2260; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
2261; SSSE3-NEXT:    psraw $8, %xmm10
2262; SSSE3-NEXT:    psraw $8, %xmm11
2263; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2264; SSSE3-NEXT:    pxor %xmm3, %xmm3
2265; SSSE3-NEXT:    pcmpgtb %xmm8, %xmm3
2266; SSSE3-NEXT:    psrlw $8, %xmm11
2267; SSSE3-NEXT:    psraw $8, %xmm7
2268; SSSE3-NEXT:    psraw $8, %xmm12
2269; SSSE3-NEXT:    pmullw %xmm7, %xmm12
2270; SSSE3-NEXT:    psrlw $8, %xmm12
2271; SSSE3-NEXT:    packuswb %xmm11, %xmm12
2272; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm12
2273; SSSE3-NEXT:    movdqa %xmm6, %xmm10
2274; SSSE3-NEXT:    movdqa %xmm6, %xmm7
2275; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2276; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2277; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2278; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2279; SSSE3-NEXT:    movdqa %xmm2, %xmm11
2280; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2281; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2282; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2283; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2284; SSSE3-NEXT:    pand %xmm9, %xmm3
2285; SSSE3-NEXT:    pand %xmm9, %xmm11
2286; SSSE3-NEXT:    packuswb %xmm3, %xmm11
2287; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2288; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2289; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2290; SSSE3-NEXT:    psraw $8, %xmm7
2291; SSSE3-NEXT:    psraw $8, %xmm3
2292; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2293; SSSE3-NEXT:    pxor %xmm7, %xmm7
2294; SSSE3-NEXT:    pcmpgtb %xmm11, %xmm7
2295; SSSE3-NEXT:    psrlw $8, %xmm3
2296; SSSE3-NEXT:    psraw $8, %xmm6
2297; SSSE3-NEXT:    psraw $8, %xmm2
2298; SSSE3-NEXT:    pmullw %xmm6, %xmm2
2299; SSSE3-NEXT:    psrlw $8, %xmm2
2300; SSSE3-NEXT:    packuswb %xmm3, %xmm2
2301; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm2
2302; SSSE3-NEXT:    movdqa %xmm5, %xmm3
2303; SSSE3-NEXT:    movdqa %xmm5, %xmm6
2304; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2305; SSSE3-NEXT:    movdqa %xmm1, %xmm7
2306; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2307; SSSE3-NEXT:    pmullw %xmm6, %xmm7
2308; SSSE3-NEXT:    movdqa %xmm1, %xmm6
2309; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2310; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2311; SSSE3-NEXT:    pmullw %xmm3, %xmm6
2312; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2313; SSSE3-NEXT:    pand %xmm9, %xmm7
2314; SSSE3-NEXT:    pand %xmm9, %xmm6
2315; SSSE3-NEXT:    packuswb %xmm7, %xmm6
2316; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2317; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2318; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2319; SSSE3-NEXT:    psraw $8, %xmm3
2320; SSSE3-NEXT:    psraw $8, %xmm7
2321; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2322; SSSE3-NEXT:    pxor %xmm3, %xmm3
2323; SSSE3-NEXT:    pcmpgtb %xmm6, %xmm3
2324; SSSE3-NEXT:    psrlw $8, %xmm7
2325; SSSE3-NEXT:    psraw $8, %xmm5
2326; SSSE3-NEXT:    psraw $8, %xmm1
2327; SSSE3-NEXT:    pmullw %xmm5, %xmm1
2328; SSSE3-NEXT:    psrlw $8, %xmm1
2329; SSSE3-NEXT:    packuswb %xmm7, %xmm1
2330; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm1
2331; SSSE3-NEXT:    movdqa %xmm4, %xmm3
2332; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2333; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2334; SSSE3-NEXT:    movdqa %xmm0, %xmm7
2335; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2336; SSSE3-NEXT:    pmullw %xmm5, %xmm7
2337; SSSE3-NEXT:    movdqa %xmm0, %xmm5
2338; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2339; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2340; SSSE3-NEXT:    pmullw %xmm3, %xmm5
2341; SSSE3-NEXT:    pand %xmm9, %xmm7
2342; SSSE3-NEXT:    pand %xmm9, %xmm5
2343; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2344; SSSE3-NEXT:    packuswb %xmm7, %xmm5
2345; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2346; SSSE3-NEXT:    psraw $8, %xmm3
2347; SSSE3-NEXT:    psraw $8, %xmm7
2348; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2349; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2350; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2351; SSSE3-NEXT:    psraw $8, %xmm3
2352; SSSE3-NEXT:    psraw $8, %xmm0
2353; SSSE3-NEXT:    pmullw %xmm3, %xmm0
2354; SSSE3-NEXT:    psrlw $8, %xmm7
2355; SSSE3-NEXT:    psrlw $8, %xmm0
2356; SSSE3-NEXT:    packuswb %xmm7, %xmm0
2357; SSSE3-NEXT:    pxor %xmm3, %xmm3
2358; SSSE3-NEXT:    pcmpgtb %xmm5, %xmm3
2359; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm0
2360; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
2361; SSSE3-NEXT:    pxor %xmm3, %xmm12
2362; SSSE3-NEXT:    pxor %xmm3, %xmm2
2363; SSSE3-NEXT:    pxor %xmm3, %xmm1
2364; SSSE3-NEXT:    pxor %xmm3, %xmm0
2365; SSSE3-NEXT:    movdqa %xmm8, 48(%rsi)
2366; SSSE3-NEXT:    movdqa %xmm11, 32(%rsi)
2367; SSSE3-NEXT:    movdqa %xmm6, 16(%rsi)
2368; SSSE3-NEXT:    movdqa %xmm12, %xmm3
2369; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2370; SSSE3-NEXT:    movdqa %xmm5, (%rsi)
2371; SSSE3-NEXT:    movdqa %xmm12, %xmm4
2372; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
2373; SSSE3-NEXT:    pslld $31, %xmm12
2374; SSSE3-NEXT:    psrad $31, %xmm12
2375; SSSE3-NEXT:    movdqa %xmm12, 224(%rdi)
2376; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2377; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2378; SSSE3-NEXT:    pslld $31, %xmm4
2379; SSSE3-NEXT:    psrad $31, %xmm4
2380; SSSE3-NEXT:    movdqa %xmm4, 240(%rdi)
2381; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2382; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2383; SSSE3-NEXT:    pslld $31, %xmm3
2384; SSSE3-NEXT:    psrad $31, %xmm3
2385; SSSE3-NEXT:    movdqa %xmm3, 192(%rdi)
2386; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2387; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2388; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2389; SSSE3-NEXT:    pslld $31, %xmm4
2390; SSSE3-NEXT:    psrad $31, %xmm4
2391; SSSE3-NEXT:    movdqa %xmm4, 208(%rdi)
2392; SSSE3-NEXT:    movdqa %xmm2, %xmm4
2393; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2394; SSSE3-NEXT:    pslld $31, %xmm2
2395; SSSE3-NEXT:    psrad $31, %xmm2
2396; SSSE3-NEXT:    movdqa %xmm2, 160(%rdi)
2397; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2398; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2399; SSSE3-NEXT:    pslld $31, %xmm4
2400; SSSE3-NEXT:    psrad $31, %xmm4
2401; SSSE3-NEXT:    movdqa %xmm4, 176(%rdi)
2402; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2403; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2404; SSSE3-NEXT:    pslld $31, %xmm3
2405; SSSE3-NEXT:    psrad $31, %xmm3
2406; SSSE3-NEXT:    movdqa %xmm3, 128(%rdi)
2407; SSSE3-NEXT:    movdqa %xmm1, %xmm3
2408; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2409; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2410; SSSE3-NEXT:    pslld $31, %xmm2
2411; SSSE3-NEXT:    psrad $31, %xmm2
2412; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
2413; SSSE3-NEXT:    movdqa %xmm1, %xmm2
2414; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2415; SSSE3-NEXT:    pslld $31, %xmm1
2416; SSSE3-NEXT:    psrad $31, %xmm1
2417; SSSE3-NEXT:    movdqa %xmm1, 96(%rdi)
2418; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2419; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2420; SSSE3-NEXT:    pslld $31, %xmm2
2421; SSSE3-NEXT:    psrad $31, %xmm2
2422; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
2423; SSSE3-NEXT:    movdqa %xmm3, %xmm1
2424; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2425; SSSE3-NEXT:    pslld $31, %xmm3
2426; SSSE3-NEXT:    psrad $31, %xmm3
2427; SSSE3-NEXT:    movdqa %xmm3, 64(%rdi)
2428; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2429; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2430; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2431; SSSE3-NEXT:    pslld $31, %xmm1
2432; SSSE3-NEXT:    psrad $31, %xmm1
2433; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
2434; SSSE3-NEXT:    movdqa %xmm0, %xmm1
2435; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2436; SSSE3-NEXT:    pslld $31, %xmm0
2437; SSSE3-NEXT:    psrad $31, %xmm0
2438; SSSE3-NEXT:    movdqa %xmm0, 32(%rdi)
2439; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2440; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2441; SSSE3-NEXT:    pslld $31, %xmm1
2442; SSSE3-NEXT:    psrad $31, %xmm1
2443; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
2444; SSSE3-NEXT:    movdqa %xmm2, %xmm0
2445; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2446; SSSE3-NEXT:    pslld $31, %xmm2
2447; SSSE3-NEXT:    psrad $31, %xmm2
2448; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
2449; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2450; SSSE3-NEXT:    pslld $31, %xmm0
2451; SSSE3-NEXT:    psrad $31, %xmm0
2452; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
2453; SSSE3-NEXT:    retq
2454;
2455; SSE41-LABEL: smulo_v64i8:
2456; SSE41:       # %bb.0:
2457; SSE41-NEXT:    movq %rdi, %rax
2458; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2459; SSE41-NEXT:    movdqa %xmm7, %xmm10
2460; SSE41-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2461; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2462; SSE41-NEXT:    movdqa %xmm3, %xmm11
2463; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2464; SSE41-NEXT:    pmullw %xmm10, %xmm11
2465; SSE41-NEXT:    pmovsxbw %xmm7, %xmm12
2466; SSE41-NEXT:    pmullw %xmm8, %xmm9
2467; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2468; SSE41-NEXT:    pand %xmm10, %xmm11
2469; SSE41-NEXT:    pand %xmm10, %xmm9
2470; SSE41-NEXT:    packuswb %xmm11, %xmm9
2471; SSE41-NEXT:    pmovsxbw %xmm3, %xmm8
2472; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,2,3]
2473; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,2,3]
2474; SSE41-NEXT:    pmullw %xmm12, %xmm8
2475; SSE41-NEXT:    pxor %xmm7, %xmm7
2476; SSE41-NEXT:    pcmpgtb %xmm9, %xmm7
2477; SSE41-NEXT:    psrlw $8, %xmm8
2478; SSE41-NEXT:    pmovsxbw %xmm11, %xmm11
2479; SSE41-NEXT:    pmovsxbw %xmm13, %xmm3
2480; SSE41-NEXT:    pmullw %xmm11, %xmm3
2481; SSE41-NEXT:    psrlw $8, %xmm3
2482; SSE41-NEXT:    packuswb %xmm3, %xmm8
2483; SSE41-NEXT:    pcmpeqb %xmm7, %xmm8
2484; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2485; SSE41-NEXT:    movdqa %xmm6, %xmm7
2486; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2487; SSE41-NEXT:    movdqa %xmm2, %xmm3
2488; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2489; SSE41-NEXT:    pmullw %xmm7, %xmm3
2490; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2491; SSE41-NEXT:    pmullw %xmm11, %xmm12
2492; SSE41-NEXT:    pmovsxbw %xmm6, %xmm7
2493; SSE41-NEXT:    pand %xmm10, %xmm3
2494; SSE41-NEXT:    pand %xmm10, %xmm12
2495; SSE41-NEXT:    packuswb %xmm3, %xmm12
2496; SSE41-NEXT:    pmovsxbw %xmm2, %xmm3
2497; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2498; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2499; SSE41-NEXT:    pmullw %xmm7, %xmm3
2500; SSE41-NEXT:    pxor %xmm7, %xmm7
2501; SSE41-NEXT:    pcmpgtb %xmm12, %xmm7
2502; SSE41-NEXT:    psrlw $8, %xmm3
2503; SSE41-NEXT:    pmovsxbw %xmm6, %xmm6
2504; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
2505; SSE41-NEXT:    pmullw %xmm6, %xmm2
2506; SSE41-NEXT:    psrlw $8, %xmm2
2507; SSE41-NEXT:    packuswb %xmm2, %xmm3
2508; SSE41-NEXT:    pcmpeqb %xmm7, %xmm3
2509; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2510; SSE41-NEXT:    movdqa %xmm5, %xmm6
2511; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2512; SSE41-NEXT:    movdqa %xmm1, %xmm7
2513; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2514; SSE41-NEXT:    pmullw %xmm6, %xmm7
2515; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2516; SSE41-NEXT:    pmullw %xmm2, %xmm11
2517; SSE41-NEXT:    pmovsxbw %xmm5, %xmm6
2518; SSE41-NEXT:    pand %xmm10, %xmm7
2519; SSE41-NEXT:    pand %xmm10, %xmm11
2520; SSE41-NEXT:    packuswb %xmm7, %xmm11
2521; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
2522; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2523; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2524; SSE41-NEXT:    pmullw %xmm6, %xmm2
2525; SSE41-NEXT:    pxor %xmm6, %xmm6
2526; SSE41-NEXT:    pcmpgtb %xmm11, %xmm6
2527; SSE41-NEXT:    psrlw $8, %xmm2
2528; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
2529; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
2530; SSE41-NEXT:    pmullw %xmm5, %xmm1
2531; SSE41-NEXT:    psrlw $8, %xmm1
2532; SSE41-NEXT:    packuswb %xmm1, %xmm2
2533; SSE41-NEXT:    pcmpeqb %xmm6, %xmm2
2534; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2535; SSE41-NEXT:    movdqa %xmm4, %xmm5
2536; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2537; SSE41-NEXT:    movdqa %xmm0, %xmm6
2538; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2539; SSE41-NEXT:    pmullw %xmm5, %xmm6
2540; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2541; SSE41-NEXT:    pmullw %xmm1, %xmm5
2542; SSE41-NEXT:    pand %xmm10, %xmm6
2543; SSE41-NEXT:    pand %xmm10, %xmm5
2544; SSE41-NEXT:    pmovsxbw %xmm4, %xmm7
2545; SSE41-NEXT:    packuswb %xmm6, %xmm5
2546; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
2547; SSE41-NEXT:    pmullw %xmm7, %xmm1
2548; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2549; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2550; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
2551; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
2552; SSE41-NEXT:    pmullw %xmm4, %xmm0
2553; SSE41-NEXT:    psrlw $8, %xmm1
2554; SSE41-NEXT:    psrlw $8, %xmm0
2555; SSE41-NEXT:    packuswb %xmm0, %xmm1
2556; SSE41-NEXT:    pxor %xmm0, %xmm0
2557; SSE41-NEXT:    pcmpgtb %xmm5, %xmm0
2558; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
2559; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2560; SSE41-NEXT:    pxor %xmm0, %xmm8
2561; SSE41-NEXT:    pxor %xmm0, %xmm3
2562; SSE41-NEXT:    pxor %xmm0, %xmm2
2563; SSE41-NEXT:    pxor %xmm0, %xmm1
2564; SSE41-NEXT:    movdqa %xmm9, 48(%rsi)
2565; SSE41-NEXT:    movdqa %xmm12, 32(%rsi)
2566; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)
2567; SSE41-NEXT:    movdqa %xmm5, (%rsi)
2568; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
2569; SSE41-NEXT:    pslld $31, %xmm0
2570; SSE41-NEXT:    psrad $31, %xmm0
2571; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
2572; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
2573; SSE41-NEXT:    pslld $31, %xmm0
2574; SSE41-NEXT:    psrad $31, %xmm0
2575; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
2576; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
2577; SSE41-NEXT:    pslld $31, %xmm0
2578; SSE41-NEXT:    psrad $31, %xmm0
2579; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
2580; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
2581; SSE41-NEXT:    pslld $31, %xmm0
2582; SSE41-NEXT:    psrad $31, %xmm0
2583; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2584; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
2585; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2586; SSE41-NEXT:    pslld $31, %xmm0
2587; SSE41-NEXT:    psrad $31, %xmm0
2588; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
2589; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
2590; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2591; SSE41-NEXT:    pslld $31, %xmm0
2592; SSE41-NEXT:    psrad $31, %xmm0
2593; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
2594; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
2595; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2596; SSE41-NEXT:    pslld $31, %xmm0
2597; SSE41-NEXT:    psrad $31, %xmm0
2598; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
2599; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
2600; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2601; SSE41-NEXT:    pslld $31, %xmm0
2602; SSE41-NEXT:    psrad $31, %xmm0
2603; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
2604; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
2605; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2606; SSE41-NEXT:    pslld $31, %xmm0
2607; SSE41-NEXT:    psrad $31, %xmm0
2608; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
2609; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
2610; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2611; SSE41-NEXT:    pslld $31, %xmm0
2612; SSE41-NEXT:    psrad $31, %xmm0
2613; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
2614; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
2615; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2616; SSE41-NEXT:    pslld $31, %xmm0
2617; SSE41-NEXT:    psrad $31, %xmm0
2618; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
2619; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
2620; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2621; SSE41-NEXT:    pslld $31, %xmm0
2622; SSE41-NEXT:    psrad $31, %xmm0
2623; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
2624; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
2625; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2626; SSE41-NEXT:    pslld $31, %xmm0
2627; SSE41-NEXT:    psrad $31, %xmm0
2628; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
2629; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
2630; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2631; SSE41-NEXT:    pslld $31, %xmm0
2632; SSE41-NEXT:    psrad $31, %xmm0
2633; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
2634; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
2635; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2636; SSE41-NEXT:    pslld $31, %xmm0
2637; SSE41-NEXT:    psrad $31, %xmm0
2638; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
2639; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
2640; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2641; SSE41-NEXT:    pslld $31, %xmm0
2642; SSE41-NEXT:    psrad $31, %xmm0
2643; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
2644; SSE41-NEXT:    retq
2645;
2646; AVX1-LABEL: smulo_v64i8:
2647; AVX1:       # %bb.0:
2648; AVX1-NEXT:    movq %rdi, %rax
2649; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
2650; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2651; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
2652; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2653; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2654; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2655; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm8
2656; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2657; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2658; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
2659; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2660; AVX1-NEXT:    vpackuswb %xmm8, %xmm4, %xmm9
2661; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
2662; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm11, %xmm8
2663; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
2664; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2665; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2666; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2667; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2668; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
2669; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2670; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2671; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2672; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2673; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
2674; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm8, %xmm8
2675; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2676; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2677; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2678; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2679; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2680; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2681; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2682; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2683; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm12
2684; AVX1-NEXT:    vpcmpgtb %xmm12, %xmm11, %xmm4
2685; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm6
2686; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm7
2687; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2688; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
2689; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
2690; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
2691; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2692; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
2693; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2694; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2695; AVX1-NEXT:    vpackuswb %xmm1, %xmm6, %xmm1
2696; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm3
2697; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
2698; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2699; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
2700; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2701; AVX1-NEXT:    vpmullw %xmm1, %xmm7, %xmm1
2702; AVX1-NEXT:    vpand %xmm1, %xmm10, %xmm1
2703; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2704; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2705; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
2706; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2707; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
2708; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm5
2709; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2710; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
2711; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm11, %xmm7
2712; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2713; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2714; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
2715; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2716; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2717; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
2718; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2719; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
2720; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm4
2721; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2722; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2723; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2724; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2725; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2726; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2727; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2728; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
2729; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm7
2730; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm5
2731; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm6
2732; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2733; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2734; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
2735; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2736; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
2737; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2738; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm2
2739; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
2740; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
2741; AVX1-NEXT:    vpcmpgtb %xmm7, %xmm11, %xmm2
2742; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2743; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
2744; AVX1-NEXT:    vpxor %xmm5, %xmm8, %xmm6
2745; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
2746; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
2747; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
2748; AVX1-NEXT:    vmovdqa %xmm9, 48(%rsi)
2749; AVX1-NEXT:    vmovdqa %xmm12, 32(%rsi)
2750; AVX1-NEXT:    vmovdqa %xmm1, 16(%rsi)
2751; AVX1-NEXT:    vmovdqa %xmm7, (%rsi)
2752; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
2753; AVX1-NEXT:    vmovdqa %xmm1, 192(%rdi)
2754; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm1
2755; AVX1-NEXT:    vmovdqa %xmm1, 128(%rdi)
2756; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm1
2757; AVX1-NEXT:    vmovdqa %xmm1, 64(%rdi)
2758; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
2759; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
2760; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
2761; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2762; AVX1-NEXT:    vmovdqa %xmm1, 224(%rdi)
2763; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
2764; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2765; AVX1-NEXT:    vmovdqa %xmm1, 240(%rdi)
2766; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
2767; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2768; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
2769; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
2770; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2771; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
2772; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
2773; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2774; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
2775; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
2776; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2777; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
2778; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
2779; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2780; AVX1-NEXT:    vmovdqa %xmm1, 96(%rdi)
2781; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
2782; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2783; AVX1-NEXT:    vmovdqa %xmm1, 112(%rdi)
2784; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
2785; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2786; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
2787; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
2788; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2789; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
2790; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2791; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2792; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
2793; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
2794; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
2795; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
2796; AVX1-NEXT:    vzeroupper
2797; AVX1-NEXT:    retq
2798;
2799; AVX2-LABEL: smulo_v64i8:
2800; AVX2:       # %bb.0:
2801; AVX2-NEXT:    movq %rdi, %rax
2802; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2803; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2804; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2805; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2806; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
2807; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2808; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2809; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2810; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2811; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm10
2812; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
2813; AVX2-NEXT:    vpcmpgtb %ymm10, %ymm6, %ymm7
2814; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2815; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2816; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2817; AVX2-NEXT:    vpsraw $8, %ymm9, %ymm9
2818; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2819; AVX2-NEXT:    vpsrlw $8, %ymm8, %ymm8
2820; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2821; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
2822; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2823; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
2824; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2825; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
2826; AVX2-NEXT:    vpackuswb %ymm8, %ymm1, %ymm1
2827; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm1
2828; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm7, %ymm7
2829; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm1
2830; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2831; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2832; AVX2-NEXT:    vpmullw %ymm3, %ymm8, %ymm3
2833; AVX2-NEXT:    vpand %ymm5, %ymm3, %ymm3
2834; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2835; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2836; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2837; AVX2-NEXT:    vpand %ymm5, %ymm8, %ymm5
2838; AVX2-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
2839; AVX2-NEXT:    vpcmpgtb %ymm3, %ymm6, %ymm5
2840; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2841; AVX2-NEXT:    vpsraw $8, %ymm6, %ymm6
2842; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2843; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2844; AVX2-NEXT:    vpmullw %ymm6, %ymm8, %ymm6
2845; AVX2-NEXT:    vpsrlw $8, %ymm6, %ymm6
2846; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2847; AVX2-NEXT:    vpsraw $8, %ymm2, %ymm2
2848; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2849; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
2850; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
2851; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
2852; AVX2-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
2853; AVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
2854; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm0
2855; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2856; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm8
2857; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
2858; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
2859; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
2860; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
2861; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
2862; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
2863; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
2864; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
2865; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
2866; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
2867; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
2868; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
2869; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
2870; AVX2-NEXT:    vmovdqa %ymm3, (%rsi)
2871; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
2872; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
2873; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)
2874; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
2875; AVX2-NEXT:    vmovdqa %ymm2, 224(%rdi)
2876; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
2877; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
2878; AVX2-NEXT:    vmovdqa %ymm8, 32(%rdi)
2879; AVX2-NEXT:    vzeroupper
2880; AVX2-NEXT:    retq
2881;
2882; AVX512F-LABEL: smulo_v64i8:
2883; AVX512F:       # %bb.0:
2884; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
2885; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm5
2886; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
2887; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
2888; AVX512F-NEXT:    vextracti128 $1, %ymm6, %xmm7
2889; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
2890; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
2891; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2892; AVX512F-NEXT:    vpmovdb %zmm2, %xmm4
2893; AVX512F-NEXT:    vpxor %xmm8, %xmm8, %xmm8
2894; AVX512F-NEXT:    vpcmpgtb %xmm4, %xmm8, %xmm2
2895; AVX512F-NEXT:    vpmovsxbw %xmm5, %ymm5
2896; AVX512F-NEXT:    vpmovsxbw %xmm7, %ymm7
2897; AVX512F-NEXT:    vpmullw %ymm5, %ymm7, %ymm5
2898; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
2899; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
2900; AVX512F-NEXT:    vpmovdb %zmm5, %xmm5
2901; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm5, %xmm2
2902; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2903; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2904; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k1
2905; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2906; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
2907; AVX512F-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
2908; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2909; AVX512F-NEXT:    vpmovdb %zmm2, %xmm5
2910; AVX512F-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
2911; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
2912; AVX512F-NEXT:    vpmovsxbw %xmm6, %ymm6
2913; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm3
2914; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
2915; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2916; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2917; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm2
2918; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2919; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2920; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k2
2921; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
2922; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2923; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm7
2924; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
2925; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm3
2926; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2927; AVX512F-NEXT:    vpmovdb %zmm3, %xmm6
2928; AVX512F-NEXT:    vpcmpgtb %xmm6, %xmm8, %xmm3
2929; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
2930; AVX512F-NEXT:    vpmovsxbw %xmm7, %ymm7
2931; AVX512F-NEXT:    vpmullw %ymm2, %ymm7, %ymm2
2932; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
2933; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2934; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
2935; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
2936; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2937; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2938; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k3
2939; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2940; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2941; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
2942; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2943; AVX512F-NEXT:    vpmovdb %zmm2, %xmm7
2944; AVX512F-NEXT:    vpcmpgtb %xmm7, %xmm8, %xmm2
2945; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
2946; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
2947; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2948; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
2949; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
2950; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2951; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2952; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2953; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
2954; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k4
2955; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
2956; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
2957; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
2958; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2959; AVX512F-NEXT:    vmovdqa %xmm4, 48(%rdi)
2960; AVX512F-NEXT:    vmovdqa %xmm5, 32(%rdi)
2961; AVX512F-NEXT:    vmovdqa %xmm6, 16(%rdi)
2962; AVX512F-NEXT:    vmovdqa %xmm7, (%rdi)
2963; AVX512F-NEXT:    retq
2964;
2965; AVX512BW-LABEL: smulo_v64i8:
2966; AVX512BW:       # %bb.0:
2967; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2968; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
2969; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2970; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
2971; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2972; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
2973; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2974; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
2975; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2976; AVX512BW-NEXT:    vpsraw $8, %zmm4, %zmm4
2977; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2978; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
2979; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
2980; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2981; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2982; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2983; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2984; AVX512BW-NEXT:    vpandq %zmm4, %zmm3, %zmm3
2985; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2986; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2987; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2988; AVX512BW-NEXT:    vpandq %zmm4, %zmm0, %zmm0
2989; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm4
2990; AVX512BW-NEXT:    vpmovb2m %zmm4, %k0
2991; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
2992; AVX512BW-NEXT:    vpcmpneqb %zmm0, %zmm2, %k1
2993; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2994; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
2995; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2996; AVX512BW-NEXT:    kshiftrq $32, %k1, %k1
2997; AVX512BW-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
2998; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2999; AVX512BW-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
3000; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdi)
3001; AVX512BW-NEXT:    retq
3002  %t = call {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)
3003  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 0
3004  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 1
3005  %res = sext <64 x i1> %obit to <64 x i32>
3006  store <64 x i8> %val, <64 x i8>* %p2
3007  ret <64 x i32> %res
3008}
3009
3010define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
3011; SSE2-LABEL: smulo_v8i16:
3012; SSE2:       # %bb.0:
3013; SSE2-NEXT:    movdqa %xmm0, %xmm2
3014; SSE2-NEXT:    pmulhw %xmm1, %xmm2
3015; SSE2-NEXT:    pmullw %xmm1, %xmm0
3016; SSE2-NEXT:    movdqa %xmm0, (%rdi)
3017; SSE2-NEXT:    psraw $15, %xmm0
3018; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
3019; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
3020; SSE2-NEXT:    pxor %xmm0, %xmm1
3021; SSE2-NEXT:    movdqa %xmm1, %xmm0
3022; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3023; SSE2-NEXT:    pslld $31, %xmm0
3024; SSE2-NEXT:    psrad $31, %xmm0
3025; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3026; SSE2-NEXT:    pslld $31, %xmm1
3027; SSE2-NEXT:    psrad $31, %xmm1
3028; SSE2-NEXT:    retq
3029;
3030; SSSE3-LABEL: smulo_v8i16:
3031; SSSE3:       # %bb.0:
3032; SSSE3-NEXT:    movdqa %xmm0, %xmm2
3033; SSSE3-NEXT:    pmulhw %xmm1, %xmm2
3034; SSSE3-NEXT:    pmullw %xmm1, %xmm0
3035; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
3036; SSSE3-NEXT:    psraw $15, %xmm0
3037; SSSE3-NEXT:    pcmpeqw %xmm2, %xmm0
3038; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
3039; SSSE3-NEXT:    pxor %xmm0, %xmm1
3040; SSSE3-NEXT:    movdqa %xmm1, %xmm0
3041; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3042; SSSE3-NEXT:    pslld $31, %xmm0
3043; SSSE3-NEXT:    psrad $31, %xmm0
3044; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3045; SSSE3-NEXT:    pslld $31, %xmm1
3046; SSSE3-NEXT:    psrad $31, %xmm1
3047; SSSE3-NEXT:    retq
3048;
3049; SSE41-LABEL: smulo_v8i16:
3050; SSE41:       # %bb.0:
3051; SSE41-NEXT:    movdqa %xmm0, %xmm2
3052; SSE41-NEXT:    pmulhw %xmm1, %xmm2
3053; SSE41-NEXT:    pmullw %xmm1, %xmm0
3054; SSE41-NEXT:    movdqa %xmm0, (%rdi)
3055; SSE41-NEXT:    psraw $15, %xmm0
3056; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
3057; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
3058; SSE41-NEXT:    pxor %xmm0, %xmm1
3059; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
3060; SSE41-NEXT:    pslld $31, %xmm0
3061; SSE41-NEXT:    psrad $31, %xmm0
3062; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3063; SSE41-NEXT:    pslld $31, %xmm1
3064; SSE41-NEXT:    psrad $31, %xmm1
3065; SSE41-NEXT:    retq
3066;
3067; AVX1-LABEL: smulo_v8i16:
3068; AVX1:       # %bb.0:
3069; AVX1-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3070; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3071; AVX1-NEXT:    vpsraw $15, %xmm1, %xmm0
3072; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3073; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3074; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3075; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
3076; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3077; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
3078; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
3079; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
3080; AVX1-NEXT:    retq
3081;
3082; AVX2-LABEL: smulo_v8i16:
3083; AVX2:       # %bb.0:
3084; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3085; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3086; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm0
3087; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3088; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3089; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3090; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
3091; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
3092; AVX2-NEXT:    retq
3093;
3094; AVX512F-LABEL: smulo_v8i16:
3095; AVX512F:       # %bb.0:
3096; AVX512F-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3097; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3098; AVX512F-NEXT:    vpsraw $15, %xmm1, %xmm0
3099; AVX512F-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3100; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
3101; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
3102; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
3103; AVX512F-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
3104; AVX512F-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
3105; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
3106; AVX512F-NEXT:    retq
3107;
3108; AVX512BW-LABEL: smulo_v8i16:
3109; AVX512BW:       # %bb.0:
3110; AVX512BW-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3111; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3112; AVX512BW-NEXT:    vpsraw $15, %xmm1, %xmm0
3113; AVX512BW-NEXT:    vpcmpneqw %xmm0, %xmm2, %k1
3114; AVX512BW-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
3115; AVX512BW-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
3116; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
3117; AVX512BW-NEXT:    retq
3118  %t = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
3119  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
3120  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
3121  %res = sext <8 x i1> %obit to <8 x i32>
3122  store <8 x i16> %val, <8 x i16>* %p2
3123  ret <8 x i32> %res
3124}
3125
3126define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
3127; SSE2-LABEL: smulo_v2i64:
3128; SSE2:       # %bb.0:
3129; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
3130; SSE2-NEXT:    movq %xmm2, %r8
3131; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
3132; SSE2-NEXT:    movq %xmm2, %rcx
3133; SSE2-NEXT:    movq %xmm1, %rdx
3134; SSE2-NEXT:    movq %xmm0, %rsi
3135; SSE2-NEXT:    xorl %eax, %eax
3136; SSE2-NEXT:    imulq %rdx, %rsi
3137; SSE2-NEXT:    movq $-1, %r9
3138; SSE2-NEXT:    movl $0, %edx
3139; SSE2-NEXT:    cmovoq %r9, %rdx
3140; SSE2-NEXT:    movq %rsi, %xmm1
3141; SSE2-NEXT:    imulq %r8, %rcx
3142; SSE2-NEXT:    movq %rcx, %xmm0
3143; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3144; SSE2-NEXT:    movq %rdx, %xmm0
3145; SSE2-NEXT:    cmovoq %r9, %rax
3146; SSE2-NEXT:    movq %rax, %xmm2
3147; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3148; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3149; SSE2-NEXT:    movdqa %xmm1, (%rdi)
3150; SSE2-NEXT:    retq
3151;
3152; SSSE3-LABEL: smulo_v2i64:
3153; SSSE3:       # %bb.0:
3154; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
3155; SSSE3-NEXT:    movq %xmm2, %r8
3156; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
3157; SSSE3-NEXT:    movq %xmm2, %rcx
3158; SSSE3-NEXT:    movq %xmm1, %rdx
3159; SSSE3-NEXT:    movq %xmm0, %rsi
3160; SSSE3-NEXT:    xorl %eax, %eax
3161; SSSE3-NEXT:    imulq %rdx, %rsi
3162; SSSE3-NEXT:    movq $-1, %r9
3163; SSSE3-NEXT:    movl $0, %edx
3164; SSSE3-NEXT:    cmovoq %r9, %rdx
3165; SSSE3-NEXT:    movq %rsi, %xmm1
3166; SSSE3-NEXT:    imulq %r8, %rcx
3167; SSSE3-NEXT:    movq %rcx, %xmm0
3168; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3169; SSSE3-NEXT:    movq %rdx, %xmm0
3170; SSSE3-NEXT:    cmovoq %r9, %rax
3171; SSSE3-NEXT:    movq %rax, %xmm2
3172; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3173; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3174; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
3175; SSSE3-NEXT:    retq
3176;
3177; SSE41-LABEL: smulo_v2i64:
3178; SSE41:       # %bb.0:
3179; SSE41-NEXT:    movq %xmm1, %r8
3180; SSE41-NEXT:    movq %xmm0, %rcx
3181; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
3182; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
3183; SSE41-NEXT:    xorl %eax, %eax
3184; SSE41-NEXT:    imulq %rdx, %rsi
3185; SSE41-NEXT:    movq $-1, %r9
3186; SSE41-NEXT:    movl $0, %edx
3187; SSE41-NEXT:    cmovoq %r9, %rdx
3188; SSE41-NEXT:    movq %rsi, %xmm0
3189; SSE41-NEXT:    imulq %r8, %rcx
3190; SSE41-NEXT:    movq %rcx, %xmm1
3191; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3192; SSE41-NEXT:    movq %rdx, %xmm0
3193; SSE41-NEXT:    cmovoq %r9, %rax
3194; SSE41-NEXT:    movq %rax, %xmm2
3195; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
3196; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
3197; SSE41-NEXT:    movdqa %xmm1, (%rdi)
3198; SSE41-NEXT:    retq
3199;
3200; AVX-LABEL: smulo_v2i64:
3201; AVX:       # %bb.0:
3202; AVX-NEXT:    vmovq %xmm1, %r8
3203; AVX-NEXT:    vmovq %xmm0, %rcx
3204; AVX-NEXT:    vpextrq $1, %xmm1, %rdx
3205; AVX-NEXT:    vpextrq $1, %xmm0, %rsi
3206; AVX-NEXT:    xorl %eax, %eax
3207; AVX-NEXT:    imulq %rdx, %rsi
3208; AVX-NEXT:    movq $-1, %r9
3209; AVX-NEXT:    movl $0, %edx
3210; AVX-NEXT:    cmovoq %r9, %rdx
3211; AVX-NEXT:    vmovq %rsi, %xmm0
3212; AVX-NEXT:    imulq %r8, %rcx
3213; AVX-NEXT:    vmovq %rcx, %xmm1
3214; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3215; AVX-NEXT:    vmovq %rdx, %xmm0
3216; AVX-NEXT:    cmovoq %r9, %rax
3217; AVX-NEXT:    vmovq %rax, %xmm2
3218; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
3219; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3220; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
3221; AVX-NEXT:    retq
3222;
3223; AVX512F-LABEL: smulo_v2i64:
3224; AVX512F:       # %bb.0:
3225; AVX512F-NEXT:    vmovq %xmm1, %rax
3226; AVX512F-NEXT:    vmovq %xmm0, %rcx
3227; AVX512F-NEXT:    vpextrq $1, %xmm1, %rdx
3228; AVX512F-NEXT:    vpextrq $1, %xmm0, %rsi
3229; AVX512F-NEXT:    imulq %rdx, %rsi
3230; AVX512F-NEXT:    seto %dl
3231; AVX512F-NEXT:    vmovq %rsi, %xmm0
3232; AVX512F-NEXT:    imulq %rax, %rcx
3233; AVX512F-NEXT:    vmovq %rcx, %xmm1
3234; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3235; AVX512F-NEXT:    seto %al
3236; AVX512F-NEXT:    movw $-3, %cx
3237; AVX512F-NEXT:    kmovw %ecx, %k0
3238; AVX512F-NEXT:    kmovw %eax, %k1
3239; AVX512F-NEXT:    kandw %k0, %k1, %k0
3240; AVX512F-NEXT:    kmovw %edx, %k1
3241; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
3242; AVX512F-NEXT:    kshiftrw $14, %k1, %k1
3243; AVX512F-NEXT:    korw %k1, %k0, %k1
3244; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3245; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3246; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
3247; AVX512F-NEXT:    retq
3248;
3249; AVX512BW-LABEL: smulo_v2i64:
3250; AVX512BW:       # %bb.0:
3251; AVX512BW-NEXT:    vmovq %xmm1, %rax
3252; AVX512BW-NEXT:    vmovq %xmm0, %rcx
3253; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rdx
3254; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rsi
3255; AVX512BW-NEXT:    imulq %rdx, %rsi
3256; AVX512BW-NEXT:    seto %dl
3257; AVX512BW-NEXT:    vmovq %rsi, %xmm0
3258; AVX512BW-NEXT:    imulq %rax, %rcx
3259; AVX512BW-NEXT:    vmovq %rcx, %xmm1
3260; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3261; AVX512BW-NEXT:    seto %al
3262; AVX512BW-NEXT:    movw $-3, %cx
3263; AVX512BW-NEXT:    kmovd %ecx, %k0
3264; AVX512BW-NEXT:    kmovd %eax, %k1
3265; AVX512BW-NEXT:    kandw %k0, %k1, %k0
3266; AVX512BW-NEXT:    kmovd %edx, %k1
3267; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
3268; AVX512BW-NEXT:    kshiftrw $14, %k1, %k1
3269; AVX512BW-NEXT:    korw %k1, %k0, %k1
3270; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3271; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3272; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
3273; AVX512BW-NEXT:    retq
3274  %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
3275  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
3276  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
3277  %res = sext <2 x i1> %obit to <2 x i32>
3278  store <2 x i64> %val, <2 x i64>* %p2
3279  ret <2 x i32> %res
3280}
3281
3282define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
3283; SSE2-LABEL: smulo_v4i24:
3284; SSE2:       # %bb.0:
3285; SSE2-NEXT:    pslld $8, %xmm0
3286; SSE2-NEXT:    psrad $8, %xmm0
3287; SSE2-NEXT:    pslld $8, %xmm1
3288; SSE2-NEXT:    psrad $8, %xmm1
3289; SSE2-NEXT:    pxor %xmm3, %xmm3
3290; SSE2-NEXT:    pxor %xmm2, %xmm2
3291; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3292; SSE2-NEXT:    pand %xmm0, %xmm2
3293; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3294; SSE2-NEXT:    pand %xmm1, %xmm3
3295; SSE2-NEXT:    paddd %xmm2, %xmm3
3296; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3297; SSE2-NEXT:    pmuludq %xmm1, %xmm0
3298; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3299; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3300; SSE2-NEXT:    pmuludq %xmm4, %xmm2
3301; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3302; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3303; SSE2-NEXT:    psubd %xmm3, %xmm5
3304; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3305; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3306; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3307; SSE2-NEXT:    movdqa %xmm3, %xmm1
3308; SSE2-NEXT:    pslld $8, %xmm1
3309; SSE2-NEXT:    psrad $8, %xmm1
3310; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
3311; SSE2-NEXT:    psrad $31, %xmm3
3312; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
3313; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
3314; SSE2-NEXT:    pxor %xmm4, %xmm3
3315; SSE2-NEXT:    pxor %xmm4, %xmm1
3316; SSE2-NEXT:    por %xmm3, %xmm1
3317; SSE2-NEXT:    movd %xmm0, %eax
3318; SSE2-NEXT:    movw %ax, (%rdi)
3319; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3320; SSE2-NEXT:    movd %xmm0, %ecx
3321; SSE2-NEXT:    movw %cx, 6(%rdi)
3322; SSE2-NEXT:    movd %xmm2, %edx
3323; SSE2-NEXT:    movw %dx, 3(%rdi)
3324; SSE2-NEXT:    shrl $16, %eax
3325; SSE2-NEXT:    movb %al, 2(%rdi)
3326; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3327; SSE2-NEXT:    movd %xmm0, %eax
3328; SSE2-NEXT:    movw %ax, 9(%rdi)
3329; SSE2-NEXT:    shrl $16, %ecx
3330; SSE2-NEXT:    movb %cl, 8(%rdi)
3331; SSE2-NEXT:    shrl $16, %edx
3332; SSE2-NEXT:    movb %dl, 5(%rdi)
3333; SSE2-NEXT:    shrl $16, %eax
3334; SSE2-NEXT:    movb %al, 11(%rdi)
3335; SSE2-NEXT:    movdqa %xmm1, %xmm0
3336; SSE2-NEXT:    retq
3337;
3338; SSSE3-LABEL: smulo_v4i24:
3339; SSSE3:       # %bb.0:
3340; SSSE3-NEXT:    pslld $8, %xmm0
3341; SSSE3-NEXT:    psrad $8, %xmm0
3342; SSSE3-NEXT:    pslld $8, %xmm1
3343; SSSE3-NEXT:    psrad $8, %xmm1
3344; SSSE3-NEXT:    pxor %xmm3, %xmm3
3345; SSSE3-NEXT:    pxor %xmm2, %xmm2
3346; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
3347; SSSE3-NEXT:    pand %xmm0, %xmm2
3348; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
3349; SSSE3-NEXT:    pand %xmm1, %xmm3
3350; SSSE3-NEXT:    paddd %xmm2, %xmm3
3351; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3352; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
3353; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3354; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3355; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
3356; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3357; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3358; SSSE3-NEXT:    psubd %xmm3, %xmm5
3359; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3360; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3361; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3362; SSSE3-NEXT:    movdqa %xmm3, %xmm1
3363; SSSE3-NEXT:    pslld $8, %xmm1
3364; SSSE3-NEXT:    psrad $8, %xmm1
3365; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm1
3366; SSSE3-NEXT:    psrad $31, %xmm3
3367; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
3368; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
3369; SSSE3-NEXT:    pxor %xmm4, %xmm3
3370; SSSE3-NEXT:    pxor %xmm4, %xmm1
3371; SSSE3-NEXT:    por %xmm3, %xmm1
3372; SSSE3-NEXT:    movd %xmm0, %eax
3373; SSSE3-NEXT:    movw %ax, (%rdi)
3374; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3375; SSSE3-NEXT:    movd %xmm0, %ecx
3376; SSSE3-NEXT:    movw %cx, 6(%rdi)
3377; SSSE3-NEXT:    movd %xmm2, %edx
3378; SSSE3-NEXT:    movw %dx, 3(%rdi)
3379; SSSE3-NEXT:    shrl $16, %eax
3380; SSSE3-NEXT:    movb %al, 2(%rdi)
3381; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3382; SSSE3-NEXT:    movd %xmm0, %eax
3383; SSSE3-NEXT:    movw %ax, 9(%rdi)
3384; SSSE3-NEXT:    shrl $16, %ecx
3385; SSSE3-NEXT:    movb %cl, 8(%rdi)
3386; SSSE3-NEXT:    shrl $16, %edx
3387; SSSE3-NEXT:    movb %dl, 5(%rdi)
3388; SSSE3-NEXT:    shrl $16, %eax
3389; SSSE3-NEXT:    movb %al, 11(%rdi)
3390; SSSE3-NEXT:    movdqa %xmm1, %xmm0
3391; SSSE3-NEXT:    retq
3392;
3393; SSE41-LABEL: smulo_v4i24:
3394; SSE41:       # %bb.0:
3395; SSE41-NEXT:    pslld $8, %xmm0
3396; SSE41-NEXT:    psrad $8, %xmm0
3397; SSE41-NEXT:    pslld $8, %xmm1
3398; SSE41-NEXT:    psrad $8, %xmm1
3399; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3400; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3401; SSE41-NEXT:    pmuldq %xmm2, %xmm3
3402; SSE41-NEXT:    movdqa %xmm0, %xmm2
3403; SSE41-NEXT:    pmuldq %xmm1, %xmm2
3404; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3405; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
3406; SSE41-NEXT:    pmulld %xmm0, %xmm1
3407; SSE41-NEXT:    movdqa %xmm1, %xmm3
3408; SSE41-NEXT:    pslld $8, %xmm3
3409; SSE41-NEXT:    psrad $8, %xmm3
3410; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
3411; SSE41-NEXT:    pextrd $3, %xmm1, %eax
3412; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
3413; SSE41-NEXT:    pextrd $1, %xmm1, %edx
3414; SSE41-NEXT:    movd %xmm1, %esi
3415; SSE41-NEXT:    psrad $31, %xmm1
3416; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
3417; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
3418; SSE41-NEXT:    pxor %xmm0, %xmm1
3419; SSE41-NEXT:    pxor %xmm3, %xmm0
3420; SSE41-NEXT:    por %xmm1, %xmm0
3421; SSE41-NEXT:    movw %ax, 9(%rdi)
3422; SSE41-NEXT:    movw %cx, 6(%rdi)
3423; SSE41-NEXT:    movw %dx, 3(%rdi)
3424; SSE41-NEXT:    movw %si, (%rdi)
3425; SSE41-NEXT:    shrl $16, %eax
3426; SSE41-NEXT:    movb %al, 11(%rdi)
3427; SSE41-NEXT:    shrl $16, %ecx
3428; SSE41-NEXT:    movb %cl, 8(%rdi)
3429; SSE41-NEXT:    shrl $16, %edx
3430; SSE41-NEXT:    movb %dl, 5(%rdi)
3431; SSE41-NEXT:    shrl $16, %esi
3432; SSE41-NEXT:    movb %sil, 2(%rdi)
3433; SSE41-NEXT:    retq
3434;
3435; AVX1-LABEL: smulo_v4i24:
3436; AVX1:       # %bb.0:
3437; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
3438; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
3439; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
3440; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
3441; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3442; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3443; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3444; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3445; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3446; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3447; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3448; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
3449; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3450; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3451; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3452; AVX1-NEXT:    vpslld $8, %xmm1, %xmm3
3453; AVX1-NEXT:    vpsrad $8, %xmm3, %xmm3
3454; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3455; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3456; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
3457; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
3458; AVX1-NEXT:    movw %ax, 9(%rdi)
3459; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
3460; AVX1-NEXT:    movw %cx, 6(%rdi)
3461; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
3462; AVX1-NEXT:    movw %dx, 3(%rdi)
3463; AVX1-NEXT:    vmovd %xmm1, %esi
3464; AVX1-NEXT:    movw %si, (%rdi)
3465; AVX1-NEXT:    shrl $16, %eax
3466; AVX1-NEXT:    movb %al, 11(%rdi)
3467; AVX1-NEXT:    shrl $16, %ecx
3468; AVX1-NEXT:    movb %cl, 8(%rdi)
3469; AVX1-NEXT:    shrl $16, %edx
3470; AVX1-NEXT:    movb %dl, 5(%rdi)
3471; AVX1-NEXT:    shrl $16, %esi
3472; AVX1-NEXT:    movb %sil, 2(%rdi)
3473; AVX1-NEXT:    retq
3474;
3475; AVX2-LABEL: smulo_v4i24:
3476; AVX2:       # %bb.0:
3477; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
3478; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
3479; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
3480; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
3481; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3482; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3483; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3484; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3485; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3486; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3487; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3488; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
3489; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3490; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3491; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3492; AVX2-NEXT:    vpslld $8, %xmm1, %xmm3
3493; AVX2-NEXT:    vpsrad $8, %xmm3, %xmm3
3494; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3495; AVX2-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3496; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
3497; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
3498; AVX2-NEXT:    movw %ax, 9(%rdi)
3499; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
3500; AVX2-NEXT:    movw %cx, 6(%rdi)
3501; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
3502; AVX2-NEXT:    movw %dx, 3(%rdi)
3503; AVX2-NEXT:    vmovd %xmm1, %esi
3504; AVX2-NEXT:    movw %si, (%rdi)
3505; AVX2-NEXT:    shrl $16, %eax
3506; AVX2-NEXT:    movb %al, 11(%rdi)
3507; AVX2-NEXT:    shrl $16, %ecx
3508; AVX2-NEXT:    movb %cl, 8(%rdi)
3509; AVX2-NEXT:    shrl $16, %edx
3510; AVX2-NEXT:    movb %dl, 5(%rdi)
3511; AVX2-NEXT:    shrl $16, %esi
3512; AVX2-NEXT:    movb %sil, 2(%rdi)
3513; AVX2-NEXT:    retq
3514;
3515; AVX512-LABEL: smulo_v4i24:
3516; AVX512:       # %bb.0:
3517; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
3518; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
3519; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
3520; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3521; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
3522; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
3523; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3524; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
3525; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
3526; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
3527; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3528; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
3529; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k0
3530; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
3531; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3532; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k1
3533; AVX512-NEXT:    korw %k0, %k1, %k1
3534; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3535; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3536; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
3537; AVX512-NEXT:    movw %ax, 9(%rdi)
3538; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
3539; AVX512-NEXT:    movw %cx, 6(%rdi)
3540; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
3541; AVX512-NEXT:    movw %dx, 3(%rdi)
3542; AVX512-NEXT:    vmovd %xmm1, %esi
3543; AVX512-NEXT:    movw %si, (%rdi)
3544; AVX512-NEXT:    shrl $16, %eax
3545; AVX512-NEXT:    movb %al, 11(%rdi)
3546; AVX512-NEXT:    shrl $16, %ecx
3547; AVX512-NEXT:    movb %cl, 8(%rdi)
3548; AVX512-NEXT:    shrl $16, %edx
3549; AVX512-NEXT:    movb %dl, 5(%rdi)
3550; AVX512-NEXT:    shrl $16, %esi
3551; AVX512-NEXT:    movb %sil, 2(%rdi)
3552; AVX512-NEXT:    retq
3553  %t = call {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
3554  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
3555  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
3556  %res = sext <4 x i1> %obit to <4 x i32>
3557  store <4 x i24> %val, <4 x i24>* %p2
3558  ret <4 x i32> %res
3559}
3560
3561define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
3562; SSE2-LABEL: smulo_v4i1:
3563; SSE2:       # %bb.0:
3564; SSE2-NEXT:    pslld $31, %xmm1
3565; SSE2-NEXT:    psrad $31, %xmm1
3566; SSE2-NEXT:    pslld $31, %xmm0
3567; SSE2-NEXT:    psrad $31, %xmm0
3568; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
3569; SSE2-NEXT:    pmuludq %xmm1, %xmm0
3570; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3571; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3572; SSE2-NEXT:    pmuludq %xmm2, %xmm1
3573; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3574; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3575; SSE2-NEXT:    movdqa %xmm0, %xmm1
3576; SSE2-NEXT:    pslld $31, %xmm1
3577; SSE2-NEXT:    movmskps %xmm1, %eax
3578; SSE2-NEXT:    psrad $31, %xmm1
3579; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
3580; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
3581; SSE2-NEXT:    pxor %xmm1, %xmm0
3582; SSE2-NEXT:    movb %al, (%rdi)
3583; SSE2-NEXT:    retq
3584;
3585; SSSE3-LABEL: smulo_v4i1:
3586; SSSE3:       # %bb.0:
3587; SSSE3-NEXT:    pslld $31, %xmm1
3588; SSSE3-NEXT:    psrad $31, %xmm1
3589; SSSE3-NEXT:    pslld $31, %xmm0
3590; SSSE3-NEXT:    psrad $31, %xmm0
3591; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
3592; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
3593; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3594; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3595; SSSE3-NEXT:    pmuludq %xmm2, %xmm1
3596; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3597; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3598; SSSE3-NEXT:    movdqa %xmm0, %xmm1
3599; SSSE3-NEXT:    pslld $31, %xmm1
3600; SSSE3-NEXT:    movmskps %xmm1, %eax
3601; SSSE3-NEXT:    psrad $31, %xmm1
3602; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
3603; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
3604; SSSE3-NEXT:    pxor %xmm1, %xmm0
3605; SSSE3-NEXT:    movb %al, (%rdi)
3606; SSSE3-NEXT:    retq
3607;
3608; SSE41-LABEL: smulo_v4i1:
3609; SSE41:       # %bb.0:
3610; SSE41-NEXT:    pslld $31, %xmm1
3611; SSE41-NEXT:    psrad $31, %xmm1
3612; SSE41-NEXT:    pslld $31, %xmm0
3613; SSE41-NEXT:    psrad $31, %xmm0
3614; SSE41-NEXT:    pmulld %xmm1, %xmm0
3615; SSE41-NEXT:    movdqa %xmm0, %xmm1
3616; SSE41-NEXT:    pslld $31, %xmm1
3617; SSE41-NEXT:    movmskps %xmm1, %eax
3618; SSE41-NEXT:    psrad $31, %xmm1
3619; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
3620; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
3621; SSE41-NEXT:    pxor %xmm0, %xmm1
3622; SSE41-NEXT:    movb %al, (%rdi)
3623; SSE41-NEXT:    movdqa %xmm1, %xmm0
3624; SSE41-NEXT:    retq
3625;
3626; AVX-LABEL: smulo_v4i1:
3627; AVX:       # %bb.0:
3628; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
3629; AVX-NEXT:    vpsrad $31, %xmm1, %xmm1
3630; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
3631; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
3632; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
3633; AVX-NEXT:    vpslld $31, %xmm0, %xmm1
3634; AVX-NEXT:    vpsrad $31, %xmm1, %xmm2
3635; AVX-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3636; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3637; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3638; AVX-NEXT:    vmovmskps %xmm1, %eax
3639; AVX-NEXT:    movb %al, (%rdi)
3640; AVX-NEXT:    retq
3641;
3642; AVX512F-LABEL: smulo_v4i1:
3643; AVX512F:       # %bb.0:
3644; AVX512F-NEXT:    pushq %rbx
3645; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
3646; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k0
3647; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
3648; AVX512F-NEXT:    kmovw %k1, %r8d
3649; AVX512F-NEXT:    andb $1, %r8b
3650; AVX512F-NEXT:    negb %r8b
3651; AVX512F-NEXT:    vpslld $31, %xmm1, %xmm0
3652; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k1
3653; AVX512F-NEXT:    kshiftrw $3, %k1, %k2
3654; AVX512F-NEXT:    kmovw %k2, %r9d
3655; AVX512F-NEXT:    andb $1, %r9b
3656; AVX512F-NEXT:    negb %r9b
3657; AVX512F-NEXT:    kshiftrw $2, %k0, %k2
3658; AVX512F-NEXT:    kmovw %k2, %r10d
3659; AVX512F-NEXT:    andb $1, %r10b
3660; AVX512F-NEXT:    negb %r10b
3661; AVX512F-NEXT:    kshiftrw $2, %k1, %k2
3662; AVX512F-NEXT:    kmovw %k2, %ebx
3663; AVX512F-NEXT:    andb $1, %bl
3664; AVX512F-NEXT:    negb %bl
3665; AVX512F-NEXT:    kshiftrw $1, %k0, %k2
3666; AVX512F-NEXT:    kmovw %k2, %ecx
3667; AVX512F-NEXT:    andb $1, %cl
3668; AVX512F-NEXT:    negb %cl
3669; AVX512F-NEXT:    kshiftrw $1, %k1, %k2
3670; AVX512F-NEXT:    kmovw %k2, %esi
3671; AVX512F-NEXT:    andb $1, %sil
3672; AVX512F-NEXT:    negb %sil
3673; AVX512F-NEXT:    kmovw %k0, %eax
3674; AVX512F-NEXT:    andb $1, %al
3675; AVX512F-NEXT:    negb %al
3676; AVX512F-NEXT:    kmovw %k1, %edx
3677; AVX512F-NEXT:    andb $1, %dl
3678; AVX512F-NEXT:    negb %dl
3679; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
3680; AVX512F-NEXT:    mulb %dl
3681; AVX512F-NEXT:    movl %eax, %r11d
3682; AVX512F-NEXT:    andb $1, %al
3683; AVX512F-NEXT:    negb %al
3684; AVX512F-NEXT:    cmpb %r11b, %al
3685; AVX512F-NEXT:    setne %al
3686; AVX512F-NEXT:    kmovw %eax, %k1
3687; AVX512F-NEXT:    movw $-3, %ax
3688; AVX512F-NEXT:    kmovw %eax, %k0
3689; AVX512F-NEXT:    kandw %k0, %k1, %k1
3690; AVX512F-NEXT:    movl %ecx, %eax
3691; AVX512F-NEXT:    mulb %sil
3692; AVX512F-NEXT:    movl %eax, %ecx
3693; AVX512F-NEXT:    andb $1, %al
3694; AVX512F-NEXT:    negb %al
3695; AVX512F-NEXT:    cmpb %cl, %al
3696; AVX512F-NEXT:    setne %al
3697; AVX512F-NEXT:    kmovw %eax, %k2
3698; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
3699; AVX512F-NEXT:    kshiftrw $14, %k2, %k2
3700; AVX512F-NEXT:    korw %k2, %k1, %k2
3701; AVX512F-NEXT:    movw $-5, %ax
3702; AVX512F-NEXT:    kmovw %eax, %k1
3703; AVX512F-NEXT:    kandw %k1, %k2, %k2
3704; AVX512F-NEXT:    movl %r10d, %eax
3705; AVX512F-NEXT:    mulb %bl
3706; AVX512F-NEXT:    movl %eax, %edx
3707; AVX512F-NEXT:    andb $1, %al
3708; AVX512F-NEXT:    negb %al
3709; AVX512F-NEXT:    cmpb %dl, %al
3710; AVX512F-NEXT:    setne %al
3711; AVX512F-NEXT:    kmovw %eax, %k3
3712; AVX512F-NEXT:    kshiftlw $2, %k3, %k3
3713; AVX512F-NEXT:    korw %k3, %k2, %k2
3714; AVX512F-NEXT:    kshiftlw $13, %k2, %k2
3715; AVX512F-NEXT:    kshiftrw $13, %k2, %k2
3716; AVX512F-NEXT:    movl %r8d, %eax
3717; AVX512F-NEXT:    mulb %r9b
3718; AVX512F-NEXT:    # kill: def $al killed $al def $eax
3719; AVX512F-NEXT:    movl %eax, %ebx
3720; AVX512F-NEXT:    andb $1, %bl
3721; AVX512F-NEXT:    negb %bl
3722; AVX512F-NEXT:    cmpb %al, %bl
3723; AVX512F-NEXT:    setne %sil
3724; AVX512F-NEXT:    kmovw %esi, %k3
3725; AVX512F-NEXT:    kshiftlw $3, %k3, %k3
3726; AVX512F-NEXT:    korw %k3, %k2, %k2
3727; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3728; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
3729; AVX512F-NEXT:    andl $1, %r11d
3730; AVX512F-NEXT:    kmovw %r11d, %k2
3731; AVX512F-NEXT:    kandw %k0, %k2, %k0
3732; AVX512F-NEXT:    kmovw %ecx, %k2
3733; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
3734; AVX512F-NEXT:    kshiftrw $14, %k2, %k2
3735; AVX512F-NEXT:    korw %k2, %k0, %k0
3736; AVX512F-NEXT:    kandw %k1, %k0, %k0
3737; AVX512F-NEXT:    kmovw %edx, %k1
3738; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
3739; AVX512F-NEXT:    kshiftrw $13, %k1, %k1
3740; AVX512F-NEXT:    korw %k1, %k0, %k0
3741; AVX512F-NEXT:    movw $-9, %cx
3742; AVX512F-NEXT:    kmovw %ecx, %k1
3743; AVX512F-NEXT:    kandw %k1, %k0, %k0
3744; AVX512F-NEXT:    kmovw %eax, %k1
3745; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
3746; AVX512F-NEXT:    kshiftrw $12, %k1, %k1
3747; AVX512F-NEXT:    korw %k1, %k0, %k0
3748; AVX512F-NEXT:    kmovw %k0, %eax
3749; AVX512F-NEXT:    movb %al, (%rdi)
3750; AVX512F-NEXT:    popq %rbx
3751; AVX512F-NEXT:    retq
3752;
3753; AVX512BW-LABEL: smulo_v4i1:
3754; AVX512BW:       # %bb.0:
3755; AVX512BW-NEXT:    pushq %rbx
3756; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
3757; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k0
3758; AVX512BW-NEXT:    kshiftrw $3, %k0, %k1
3759; AVX512BW-NEXT:    kmovd %k1, %r8d
3760; AVX512BW-NEXT:    andb $1, %r8b
3761; AVX512BW-NEXT:    negb %r8b
3762; AVX512BW-NEXT:    vpslld $31, %xmm1, %xmm0
3763; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k1
3764; AVX512BW-NEXT:    kshiftrw $3, %k1, %k2
3765; AVX512BW-NEXT:    kmovd %k2, %r9d
3766; AVX512BW-NEXT:    andb $1, %r9b
3767; AVX512BW-NEXT:    negb %r9b
3768; AVX512BW-NEXT:    kshiftrw $2, %k0, %k2
3769; AVX512BW-NEXT:    kmovd %k2, %r10d
3770; AVX512BW-NEXT:    andb $1, %r10b
3771; AVX512BW-NEXT:    negb %r10b
3772; AVX512BW-NEXT:    kshiftrw $2, %k1, %k2
3773; AVX512BW-NEXT:    kmovd %k2, %ebx
3774; AVX512BW-NEXT:    andb $1, %bl
3775; AVX512BW-NEXT:    negb %bl
3776; AVX512BW-NEXT:    kshiftrw $1, %k0, %k2
3777; AVX512BW-NEXT:    kmovd %k2, %ecx
3778; AVX512BW-NEXT:    andb $1, %cl
3779; AVX512BW-NEXT:    negb %cl
3780; AVX512BW-NEXT:    kshiftrw $1, %k1, %k2
3781; AVX512BW-NEXT:    kmovd %k2, %esi
3782; AVX512BW-NEXT:    andb $1, %sil
3783; AVX512BW-NEXT:    negb %sil
3784; AVX512BW-NEXT:    kmovd %k0, %eax
3785; AVX512BW-NEXT:    andb $1, %al
3786; AVX512BW-NEXT:    negb %al
3787; AVX512BW-NEXT:    kmovd %k1, %edx
3788; AVX512BW-NEXT:    andb $1, %dl
3789; AVX512BW-NEXT:    negb %dl
3790; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
3791; AVX512BW-NEXT:    mulb %dl
3792; AVX512BW-NEXT:    movl %eax, %r11d
3793; AVX512BW-NEXT:    andb $1, %al
3794; AVX512BW-NEXT:    negb %al
3795; AVX512BW-NEXT:    cmpb %r11b, %al
3796; AVX512BW-NEXT:    setne %al
3797; AVX512BW-NEXT:    kmovd %eax, %k1
3798; AVX512BW-NEXT:    movw $-3, %ax
3799; AVX512BW-NEXT:    kmovd %eax, %k0
3800; AVX512BW-NEXT:    kandw %k0, %k1, %k1
3801; AVX512BW-NEXT:    movl %ecx, %eax
3802; AVX512BW-NEXT:    mulb %sil
3803; AVX512BW-NEXT:    movl %eax, %ecx
3804; AVX512BW-NEXT:    andb $1, %al
3805; AVX512BW-NEXT:    negb %al
3806; AVX512BW-NEXT:    cmpb %cl, %al
3807; AVX512BW-NEXT:    setne %al
3808; AVX512BW-NEXT:    kmovd %eax, %k2
3809; AVX512BW-NEXT:    kshiftlw $15, %k2, %k2
3810; AVX512BW-NEXT:    kshiftrw $14, %k2, %k2
3811; AVX512BW-NEXT:    korw %k2, %k1, %k2
3812; AVX512BW-NEXT:    movw $-5, %ax
3813; AVX512BW-NEXT:    kmovd %eax, %k1
3814; AVX512BW-NEXT:    kandw %k1, %k2, %k2
3815; AVX512BW-NEXT:    movl %r10d, %eax
3816; AVX512BW-NEXT:    mulb %bl
3817; AVX512BW-NEXT:    movl %eax, %edx
3818; AVX512BW-NEXT:    andb $1, %al
3819; AVX512BW-NEXT:    negb %al
3820; AVX512BW-NEXT:    cmpb %dl, %al
3821; AVX512BW-NEXT:    setne %al
3822; AVX512BW-NEXT:    kmovd %eax, %k3
3823; AVX512BW-NEXT:    kshiftlw $2, %k3, %k3
3824; AVX512BW-NEXT:    korw %k3, %k2, %k2
3825; AVX512BW-NEXT:    kshiftlw $13, %k2, %k2
3826; AVX512BW-NEXT:    kshiftrw $13, %k2, %k2
3827; AVX512BW-NEXT:    movl %r8d, %eax
3828; AVX512BW-NEXT:    mulb %r9b
3829; AVX512BW-NEXT:    # kill: def $al killed $al def $eax
3830; AVX512BW-NEXT:    movl %eax, %ebx
3831; AVX512BW-NEXT:    andb $1, %bl
3832; AVX512BW-NEXT:    negb %bl
3833; AVX512BW-NEXT:    cmpb %al, %bl
3834; AVX512BW-NEXT:    setne %sil
3835; AVX512BW-NEXT:    kmovd %esi, %k3
3836; AVX512BW-NEXT:    kshiftlw $3, %k3, %k3
3837; AVX512BW-NEXT:    korw %k3, %k2, %k2
3838; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3839; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
3840; AVX512BW-NEXT:    andl $1, %r11d
3841; AVX512BW-NEXT:    kmovw %r11d, %k2
3842; AVX512BW-NEXT:    kandw %k0, %k2, %k0
3843; AVX512BW-NEXT:    kmovd %ecx, %k2
3844; AVX512BW-NEXT:    kshiftlw $15, %k2, %k2
3845; AVX512BW-NEXT:    kshiftrw $14, %k2, %k2
3846; AVX512BW-NEXT:    korw %k2, %k0, %k0
3847; AVX512BW-NEXT:    kandw %k1, %k0, %k0
3848; AVX512BW-NEXT:    kmovd %edx, %k1
3849; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
3850; AVX512BW-NEXT:    kshiftrw $13, %k1, %k1
3851; AVX512BW-NEXT:    korw %k1, %k0, %k0
3852; AVX512BW-NEXT:    movw $-9, %cx
3853; AVX512BW-NEXT:    kmovd %ecx, %k1
3854; AVX512BW-NEXT:    kandw %k1, %k0, %k0
3855; AVX512BW-NEXT:    kmovd %eax, %k1
3856; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
3857; AVX512BW-NEXT:    kshiftrw $12, %k1, %k1
3858; AVX512BW-NEXT:    korw %k1, %k0, %k0
3859; AVX512BW-NEXT:    kmovd %k0, %eax
3860; AVX512BW-NEXT:    movb %al, (%rdi)
3861; AVX512BW-NEXT:    popq %rbx
3862; AVX512BW-NEXT:    retq
3863  %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
3864  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
3865  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
3866  %res = sext <4 x i1> %obit to <4 x i32>
3867  store <4 x i1> %val, <4 x i1>* %p2
3868  ret <4 x i32> %res
3869}
3870
3871define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
3872; SSE2-LABEL: smulo_v2i128:
3873; SSE2:       # %bb.0:
3874; SSE2-NEXT:    pushq %rbp
3875; SSE2-NEXT:    pushq %r15
3876; SSE2-NEXT:    pushq %r14
3877; SSE2-NEXT:    pushq %r13
3878; SSE2-NEXT:    pushq %r12
3879; SSE2-NEXT:    pushq %rbx
3880; SSE2-NEXT:    subq $24, %rsp
3881; SSE2-NEXT:    movq %r8, %rax
3882; SSE2-NEXT:    movq %rcx, %r14
3883; SSE2-NEXT:    movq %rdx, %rbx
3884; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3885; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3886; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3887; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3888; SSE2-NEXT:    movq %rax, %rdx
3889; SSE2-NEXT:    movq %r9, %rcx
3890; SSE2-NEXT:    callq __muloti4@PLT
3891; SSE2-NEXT:    movq %rax, %r13
3892; SSE2-NEXT:    movq %rdx, %rbp
3893; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3894; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3895; SSE2-NEXT:    movq %rbx, %rdi
3896; SSE2-NEXT:    movq %r14, %rsi
3897; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3898; SSE2-NEXT:    movq %r12, %rcx
3899; SSE2-NEXT:    callq __muloti4@PLT
3900; SSE2-NEXT:    xorl %ecx, %ecx
3901; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3902; SSE2-NEXT:    sbbl %esi, %esi
3903; SSE2-NEXT:    movd %esi, %xmm1
3904; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3905; SSE2-NEXT:    sbbl %ecx, %ecx
3906; SSE2-NEXT:    movd %ecx, %xmm0
3907; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3908; SSE2-NEXT:    movq %rdx, 24(%r15)
3909; SSE2-NEXT:    movq %rax, 16(%r15)
3910; SSE2-NEXT:    movq %rbp, 8(%r15)
3911; SSE2-NEXT:    movq %r13, (%r15)
3912; SSE2-NEXT:    addq $24, %rsp
3913; SSE2-NEXT:    popq %rbx
3914; SSE2-NEXT:    popq %r12
3915; SSE2-NEXT:    popq %r13
3916; SSE2-NEXT:    popq %r14
3917; SSE2-NEXT:    popq %r15
3918; SSE2-NEXT:    popq %rbp
3919; SSE2-NEXT:    retq
3920;
3921; SSSE3-LABEL: smulo_v2i128:
3922; SSSE3:       # %bb.0:
3923; SSSE3-NEXT:    pushq %rbp
3924; SSSE3-NEXT:    pushq %r15
3925; SSSE3-NEXT:    pushq %r14
3926; SSSE3-NEXT:    pushq %r13
3927; SSSE3-NEXT:    pushq %r12
3928; SSSE3-NEXT:    pushq %rbx
3929; SSSE3-NEXT:    subq $24, %rsp
3930; SSSE3-NEXT:    movq %r8, %rax
3931; SSSE3-NEXT:    movq %rcx, %r14
3932; SSSE3-NEXT:    movq %rdx, %rbx
3933; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3934; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3935; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3936; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3937; SSSE3-NEXT:    movq %rax, %rdx
3938; SSSE3-NEXT:    movq %r9, %rcx
3939; SSSE3-NEXT:    callq __muloti4@PLT
3940; SSSE3-NEXT:    movq %rax, %r13
3941; SSSE3-NEXT:    movq %rdx, %rbp
3942; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3943; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3944; SSSE3-NEXT:    movq %rbx, %rdi
3945; SSSE3-NEXT:    movq %r14, %rsi
3946; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3947; SSSE3-NEXT:    movq %r12, %rcx
3948; SSSE3-NEXT:    callq __muloti4@PLT
3949; SSSE3-NEXT:    xorl %ecx, %ecx
3950; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3951; SSSE3-NEXT:    sbbl %esi, %esi
3952; SSSE3-NEXT:    movd %esi, %xmm1
3953; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3954; SSSE3-NEXT:    sbbl %ecx, %ecx
3955; SSSE3-NEXT:    movd %ecx, %xmm0
3956; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3957; SSSE3-NEXT:    movq %rdx, 24(%r15)
3958; SSSE3-NEXT:    movq %rax, 16(%r15)
3959; SSSE3-NEXT:    movq %rbp, 8(%r15)
3960; SSSE3-NEXT:    movq %r13, (%r15)
3961; SSSE3-NEXT:    addq $24, %rsp
3962; SSSE3-NEXT:    popq %rbx
3963; SSSE3-NEXT:    popq %r12
3964; SSSE3-NEXT:    popq %r13
3965; SSSE3-NEXT:    popq %r14
3966; SSSE3-NEXT:    popq %r15
3967; SSSE3-NEXT:    popq %rbp
3968; SSSE3-NEXT:    retq
3969;
3970; SSE41-LABEL: smulo_v2i128:
3971; SSE41:       # %bb.0:
3972; SSE41-NEXT:    pushq %rbp
3973; SSE41-NEXT:    pushq %r15
3974; SSE41-NEXT:    pushq %r14
3975; SSE41-NEXT:    pushq %r13
3976; SSE41-NEXT:    pushq %r12
3977; SSE41-NEXT:    pushq %rbx
3978; SSE41-NEXT:    subq $24, %rsp
3979; SSE41-NEXT:    movq %r8, %rax
3980; SSE41-NEXT:    movq %rcx, %r14
3981; SSE41-NEXT:    movq %rdx, %rbx
3982; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3983; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3984; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3985; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3986; SSE41-NEXT:    movq %rax, %rdx
3987; SSE41-NEXT:    movq %r9, %rcx
3988; SSE41-NEXT:    callq __muloti4@PLT
3989; SSE41-NEXT:    movq %rax, %r13
3990; SSE41-NEXT:    movq %rdx, %rbp
3991; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3992; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3993; SSE41-NEXT:    movq %rbx, %rdi
3994; SSE41-NEXT:    movq %r14, %rsi
3995; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3996; SSE41-NEXT:    movq %r12, %rcx
3997; SSE41-NEXT:    callq __muloti4@PLT
3998; SSE41-NEXT:    xorl %ecx, %ecx
3999; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
4000; SSE41-NEXT:    sbbl %esi, %esi
4001; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
4002; SSE41-NEXT:    sbbl %ecx, %ecx
4003; SSE41-NEXT:    movd %ecx, %xmm0
4004; SSE41-NEXT:    pinsrd $1, %esi, %xmm0
4005; SSE41-NEXT:    movq %rdx, 24(%r15)
4006; SSE41-NEXT:    movq %rax, 16(%r15)
4007; SSE41-NEXT:    movq %rbp, 8(%r15)
4008; SSE41-NEXT:    movq %r13, (%r15)
4009; SSE41-NEXT:    addq $24, %rsp
4010; SSE41-NEXT:    popq %rbx
4011; SSE41-NEXT:    popq %r12
4012; SSE41-NEXT:    popq %r13
4013; SSE41-NEXT:    popq %r14
4014; SSE41-NEXT:    popq %r15
4015; SSE41-NEXT:    popq %rbp
4016; SSE41-NEXT:    retq
4017;
4018; AVX-LABEL: smulo_v2i128:
4019; AVX:       # %bb.0:
4020; AVX-NEXT:    pushq %rbp
4021; AVX-NEXT:    pushq %r15
4022; AVX-NEXT:    pushq %r14
4023; AVX-NEXT:    pushq %r13
4024; AVX-NEXT:    pushq %r12
4025; AVX-NEXT:    pushq %rbx
4026; AVX-NEXT:    subq $24, %rsp
4027; AVX-NEXT:    movq %r8, %rax
4028; AVX-NEXT:    movq %rcx, %r14
4029; AVX-NEXT:    movq %rdx, %rbx
4030; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r15
4031; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r12
4032; AVX-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4033; AVX-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4034; AVX-NEXT:    movq %rax, %rdx
4035; AVX-NEXT:    movq %r9, %rcx
4036; AVX-NEXT:    callq __muloti4@PLT
4037; AVX-NEXT:    movq %rax, %r13
4038; AVX-NEXT:    movq %rdx, %rbp
4039; AVX-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4040; AVX-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4041; AVX-NEXT:    movq %rbx, %rdi
4042; AVX-NEXT:    movq %r14, %rsi
4043; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
4044; AVX-NEXT:    movq %r12, %rcx
4045; AVX-NEXT:    callq __muloti4@PLT
4046; AVX-NEXT:    xorl %ecx, %ecx
4047; AVX-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
4048; AVX-NEXT:    sbbl %esi, %esi
4049; AVX-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
4050; AVX-NEXT:    sbbl %ecx, %ecx
4051; AVX-NEXT:    vmovd %ecx, %xmm0
4052; AVX-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
4053; AVX-NEXT:    movq %rdx, 24(%r15)
4054; AVX-NEXT:    movq %rax, 16(%r15)
4055; AVX-NEXT:    movq %rbp, 8(%r15)
4056; AVX-NEXT:    movq %r13, (%r15)
4057; AVX-NEXT:    addq $24, %rsp
4058; AVX-NEXT:    popq %rbx
4059; AVX-NEXT:    popq %r12
4060; AVX-NEXT:    popq %r13
4061; AVX-NEXT:    popq %r14
4062; AVX-NEXT:    popq %r15
4063; AVX-NEXT:    popq %rbp
4064; AVX-NEXT:    retq
4065;
4066; AVX512F-LABEL: smulo_v2i128:
4067; AVX512F:       # %bb.0:
4068; AVX512F-NEXT:    pushq %rbp
4069; AVX512F-NEXT:    pushq %r15
4070; AVX512F-NEXT:    pushq %r14
4071; AVX512F-NEXT:    pushq %r13
4072; AVX512F-NEXT:    pushq %r12
4073; AVX512F-NEXT:    pushq %rbx
4074; AVX512F-NEXT:    subq $24, %rsp
4075; AVX512F-NEXT:    movq %r8, %rax
4076; AVX512F-NEXT:    movq %rcx, %r14
4077; AVX512F-NEXT:    movq %rdx, %rbx
4078; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r15
4079; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r12
4080; AVX512F-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4081; AVX512F-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4082; AVX512F-NEXT:    movq %rax, %rdx
4083; AVX512F-NEXT:    movq %r9, %rcx
4084; AVX512F-NEXT:    callq __muloti4@PLT
4085; AVX512F-NEXT:    movq %rax, %r13
4086; AVX512F-NEXT:    movq %rdx, %rbp
4087; AVX512F-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4088; AVX512F-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4089; AVX512F-NEXT:    movq %rbx, %rdi
4090; AVX512F-NEXT:    movq %r14, %rsi
4091; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
4092; AVX512F-NEXT:    movq %r12, %rcx
4093; AVX512F-NEXT:    callq __muloti4@PLT
4094; AVX512F-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
4095; AVX512F-NEXT:    setne %cl
4096; AVX512F-NEXT:    kmovw %ecx, %k0
4097; AVX512F-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
4098; AVX512F-NEXT:    setne %cl
4099; AVX512F-NEXT:    andl $1, %ecx
4100; AVX512F-NEXT:    kmovw %ecx, %k1
4101; AVX512F-NEXT:    kshiftlw $1, %k0, %k0
4102; AVX512F-NEXT:    korw %k0, %k1, %k1
4103; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
4104; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
4105; AVX512F-NEXT:    movq %rdx, 24(%r15)
4106; AVX512F-NEXT:    movq %rax, 16(%r15)
4107; AVX512F-NEXT:    movq %rbp, 8(%r15)
4108; AVX512F-NEXT:    movq %r13, (%r15)
4109; AVX512F-NEXT:    addq $24, %rsp
4110; AVX512F-NEXT:    popq %rbx
4111; AVX512F-NEXT:    popq %r12
4112; AVX512F-NEXT:    popq %r13
4113; AVX512F-NEXT:    popq %r14
4114; AVX512F-NEXT:    popq %r15
4115; AVX512F-NEXT:    popq %rbp
4116; AVX512F-NEXT:    retq
4117;
4118; AVX512BW-LABEL: smulo_v2i128:
4119; AVX512BW:       # %bb.0:
4120; AVX512BW-NEXT:    pushq %rbp
4121; AVX512BW-NEXT:    pushq %r15
4122; AVX512BW-NEXT:    pushq %r14
4123; AVX512BW-NEXT:    pushq %r13
4124; AVX512BW-NEXT:    pushq %r12
4125; AVX512BW-NEXT:    pushq %rbx
4126; AVX512BW-NEXT:    subq $24, %rsp
4127; AVX512BW-NEXT:    movq %r8, %rax
4128; AVX512BW-NEXT:    movq %rcx, %r14
4129; AVX512BW-NEXT:    movq %rdx, %rbx
4130; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r15
4131; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r12
4132; AVX512BW-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4133; AVX512BW-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4134; AVX512BW-NEXT:    movq %rax, %rdx
4135; AVX512BW-NEXT:    movq %r9, %rcx
4136; AVX512BW-NEXT:    callq __muloti4@PLT
4137; AVX512BW-NEXT:    movq %rax, %r13
4138; AVX512BW-NEXT:    movq %rdx, %rbp
4139; AVX512BW-NEXT:    movq $0, {{[0-9]+}}(%rsp)
4140; AVX512BW-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
4141; AVX512BW-NEXT:    movq %rbx, %rdi
4142; AVX512BW-NEXT:    movq %r14, %rsi
4143; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
4144; AVX512BW-NEXT:    movq %r12, %rcx
4145; AVX512BW-NEXT:    callq __muloti4@PLT
4146; AVX512BW-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
4147; AVX512BW-NEXT:    setne %cl
4148; AVX512BW-NEXT:    kmovd %ecx, %k0
4149; AVX512BW-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
4150; AVX512BW-NEXT:    setne %cl
4151; AVX512BW-NEXT:    andl $1, %ecx
4152; AVX512BW-NEXT:    kmovw %ecx, %k1
4153; AVX512BW-NEXT:    kshiftlw $1, %k0, %k0
4154; AVX512BW-NEXT:    korw %k0, %k1, %k1
4155; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
4156; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
4157; AVX512BW-NEXT:    movq %rdx, 24(%r15)
4158; AVX512BW-NEXT:    movq %rax, 16(%r15)
4159; AVX512BW-NEXT:    movq %rbp, 8(%r15)
4160; AVX512BW-NEXT:    movq %r13, (%r15)
4161; AVX512BW-NEXT:    addq $24, %rsp
4162; AVX512BW-NEXT:    popq %rbx
4163; AVX512BW-NEXT:    popq %r12
4164; AVX512BW-NEXT:    popq %r13
4165; AVX512BW-NEXT:    popq %r14
4166; AVX512BW-NEXT:    popq %r15
4167; AVX512BW-NEXT:    popq %rbp
4168; AVX512BW-NEXT:    retq
4169  %t = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
4170  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
4171  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
4172  %res = sext <2 x i1> %obit to <2 x i32>
4173  store <2 x i128> %val, <2 x i128>* %p2
4174  ret <2 x i32> %res
4175}
4176