1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
9
10declare {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
11declare {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
12declare {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
13declare {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
14declare {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
15declare {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
16declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
17
18declare {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
19declare {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
20declare {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
21declare {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
22declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
23
24declare {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
25declare {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
26declare {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
27
28define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
29; CHECK-LABEL: smulo_v1i32:
30; CHECK:       # %bb.0:
31; CHECK-NEXT:    xorl %eax, %eax
32; CHECK-NEXT:    imull %esi, %edi
33; CHECK-NEXT:    seto %al
34; CHECK-NEXT:    negl %eax
35; CHECK-NEXT:    movl %edi, (%rdx)
36; CHECK-NEXT:    retq
37  %t = call {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
38  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
39  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
40  %res = sext <1 x i1> %obit to <1 x i32>
41  store <1 x i32> %val, <1 x i32>* %p2
42  ret <1 x i32> %res
43}
44
45define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
46; SSE2-LABEL: smulo_v2i32:
47; SSE2:       # %bb.0:
48; SSE2-NEXT:    pxor %xmm2, %xmm2
49; SSE2-NEXT:    pxor %xmm3, %xmm3
50; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
51; SSE2-NEXT:    pand %xmm0, %xmm3
52; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
53; SSE2-NEXT:    pand %xmm1, %xmm2
54; SSE2-NEXT:    paddd %xmm3, %xmm2
55; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
56; SSE2-NEXT:    pmuludq %xmm1, %xmm0
57; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
58; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
59; SSE2-NEXT:    pmuludq %xmm3, %xmm1
60; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
61; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
62; SSE2-NEXT:    psubd %xmm2, %xmm4
63; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
64; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
65; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
66; SSE2-NEXT:    movq %xmm0, (%rdi)
67; SSE2-NEXT:    movdqa %xmm0, %xmm1
68; SSE2-NEXT:    psrad $31, %xmm1
69; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
70; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
71; SSE2-NEXT:    pxor %xmm1, %xmm0
72; SSE2-NEXT:    retq
73;
74; SSSE3-LABEL: smulo_v2i32:
75; SSSE3:       # %bb.0:
76; SSSE3-NEXT:    pxor %xmm2, %xmm2
77; SSSE3-NEXT:    pxor %xmm3, %xmm3
78; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
79; SSSE3-NEXT:    pand %xmm0, %xmm3
80; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
81; SSSE3-NEXT:    pand %xmm1, %xmm2
82; SSSE3-NEXT:    paddd %xmm3, %xmm2
83; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
84; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
85; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
86; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
87; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
88; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
89; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
90; SSSE3-NEXT:    psubd %xmm2, %xmm4
91; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
92; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
93; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
94; SSSE3-NEXT:    movq %xmm0, (%rdi)
95; SSSE3-NEXT:    movdqa %xmm0, %xmm1
96; SSSE3-NEXT:    psrad $31, %xmm1
97; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
98; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
99; SSSE3-NEXT:    pxor %xmm1, %xmm0
100; SSSE3-NEXT:    retq
101;
102; SSE41-LABEL: smulo_v2i32:
103; SSE41:       # %bb.0:
104; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
105; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
106; SSE41-NEXT:    pmuldq %xmm2, %xmm3
107; SSE41-NEXT:    movdqa %xmm0, %xmm2
108; SSE41-NEXT:    pmuldq %xmm1, %xmm2
109; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
110; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
111; SSE41-NEXT:    pmulld %xmm1, %xmm0
112; SSE41-NEXT:    movq %xmm0, (%rdi)
113; SSE41-NEXT:    psrad $31, %xmm0
114; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
115; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
116; SSE41-NEXT:    pxor %xmm1, %xmm0
117; SSE41-NEXT:    retq
118;
119; AVX1-LABEL: smulo_v2i32:
120; AVX1:       # %bb.0:
121; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
122; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
123; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
124; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
125; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
126; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
127; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
128; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
129; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
130; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
131; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
132; AVX1-NEXT:    vmovq %xmm1, (%rdi)
133; AVX1-NEXT:    retq
134;
135; AVX2-LABEL: smulo_v2i32:
136; AVX2:       # %bb.0:
137; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
138; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
139; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
140; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
141; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
142; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
143; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
144; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
145; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
146; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
147; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
148; AVX2-NEXT:    vmovq %xmm1, (%rdi)
149; AVX2-NEXT:    retq
150;
151; AVX512-LABEL: smulo_v2i32:
152; AVX512:       # %bb.0:
153; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
154; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
155; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
156; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
157; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
158; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
159; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
160; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
161; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
162; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
163; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
164; AVX512-NEXT:    vmovq %xmm1, (%rdi)
165; AVX512-NEXT:    retq
166  %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
167  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
168  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
169  %res = sext <2 x i1> %obit to <2 x i32>
170  store <2 x i32> %val, <2 x i32>* %p2
171  ret <2 x i32> %res
172}
173
174define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
175; SSE2-LABEL: smulo_v3i32:
176; SSE2:       # %bb.0:
177; SSE2-NEXT:    pxor %xmm2, %xmm2
178; SSE2-NEXT:    pxor %xmm3, %xmm3
179; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
180; SSE2-NEXT:    pand %xmm0, %xmm3
181; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
182; SSE2-NEXT:    pand %xmm1, %xmm2
183; SSE2-NEXT:    paddd %xmm3, %xmm2
184; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
185; SSE2-NEXT:    pmuludq %xmm1, %xmm0
186; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
187; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
188; SSE2-NEXT:    pmuludq %xmm3, %xmm1
189; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
190; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
191; SSE2-NEXT:    psubd %xmm2, %xmm4
192; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
193; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
194; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
195; SSE2-NEXT:    movq %xmm2, (%rdi)
196; SSE2-NEXT:    psrad $31, %xmm2
197; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
198; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
199; SSE2-NEXT:    pxor %xmm2, %xmm1
200; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
201; SSE2-NEXT:    movd %xmm0, 8(%rdi)
202; SSE2-NEXT:    movdqa %xmm1, %xmm0
203; SSE2-NEXT:    retq
204;
205; SSSE3-LABEL: smulo_v3i32:
206; SSSE3:       # %bb.0:
207; SSSE3-NEXT:    pxor %xmm2, %xmm2
208; SSSE3-NEXT:    pxor %xmm3, %xmm3
209; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
210; SSSE3-NEXT:    pand %xmm0, %xmm3
211; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
212; SSSE3-NEXT:    pand %xmm1, %xmm2
213; SSSE3-NEXT:    paddd %xmm3, %xmm2
214; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
215; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
216; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
217; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
218; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
219; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
220; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
221; SSSE3-NEXT:    psubd %xmm2, %xmm4
222; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
223; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
224; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
225; SSSE3-NEXT:    movq %xmm2, (%rdi)
226; SSSE3-NEXT:    psrad $31, %xmm2
227; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
228; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
229; SSSE3-NEXT:    pxor %xmm2, %xmm1
230; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
231; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
232; SSSE3-NEXT:    movdqa %xmm1, %xmm0
233; SSSE3-NEXT:    retq
234;
235; SSE41-LABEL: smulo_v3i32:
236; SSE41:       # %bb.0:
237; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
238; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
239; SSE41-NEXT:    pmuldq %xmm2, %xmm3
240; SSE41-NEXT:    movdqa %xmm0, %xmm2
241; SSE41-NEXT:    pmuldq %xmm1, %xmm2
242; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
243; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
244; SSE41-NEXT:    pmulld %xmm1, %xmm0
245; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
246; SSE41-NEXT:    movq %xmm0, (%rdi)
247; SSE41-NEXT:    psrad $31, %xmm0
248; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
249; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
250; SSE41-NEXT:    pxor %xmm1, %xmm0
251; SSE41-NEXT:    retq
252;
253; AVX1-LABEL: smulo_v3i32:
254; AVX1:       # %bb.0:
255; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
256; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
257; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
258; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
259; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
260; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
261; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
262; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
263; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
264; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
265; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
266; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
267; AVX1-NEXT:    vmovq %xmm1, (%rdi)
268; AVX1-NEXT:    retq
269;
270; AVX2-LABEL: smulo_v3i32:
271; AVX2:       # %bb.0:
272; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
273; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
274; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
275; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
276; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
277; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
278; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
279; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
280; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
281; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
282; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
283; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
284; AVX2-NEXT:    vmovq %xmm1, (%rdi)
285; AVX2-NEXT:    retq
286;
287; AVX512-LABEL: smulo_v3i32:
288; AVX512:       # %bb.0:
289; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
290; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
291; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
292; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
293; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
294; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
295; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
296; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
297; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
298; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
299; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
300; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
301; AVX512-NEXT:    vmovq %xmm1, (%rdi)
302; AVX512-NEXT:    retq
303  %t = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
304  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
305  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
306  %res = sext <3 x i1> %obit to <3 x i32>
307  store <3 x i32> %val, <3 x i32>* %p2
308  ret <3 x i32> %res
309}
310
311define <4 x i32> @smulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
312; SSE2-LABEL: smulo_v4i32:
313; SSE2:       # %bb.0:
314; SSE2-NEXT:    pxor %xmm2, %xmm2
315; SSE2-NEXT:    pxor %xmm3, %xmm3
316; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
317; SSE2-NEXT:    pand %xmm0, %xmm3
318; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
319; SSE2-NEXT:    pand %xmm1, %xmm2
320; SSE2-NEXT:    paddd %xmm3, %xmm2
321; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
322; SSE2-NEXT:    pmuludq %xmm1, %xmm0
323; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
324; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
325; SSE2-NEXT:    pmuludq %xmm3, %xmm1
326; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
327; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
328; SSE2-NEXT:    psubd %xmm2, %xmm4
329; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
330; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
331; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
332; SSE2-NEXT:    movdqa %xmm0, (%rdi)
333; SSE2-NEXT:    movdqa %xmm0, %xmm1
334; SSE2-NEXT:    psrad $31, %xmm1
335; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
336; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
337; SSE2-NEXT:    pxor %xmm1, %xmm0
338; SSE2-NEXT:    retq
339;
340; SSSE3-LABEL: smulo_v4i32:
341; SSSE3:       # %bb.0:
342; SSSE3-NEXT:    pxor %xmm2, %xmm2
343; SSSE3-NEXT:    pxor %xmm3, %xmm3
344; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
345; SSSE3-NEXT:    pand %xmm0, %xmm3
346; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
347; SSSE3-NEXT:    pand %xmm1, %xmm2
348; SSSE3-NEXT:    paddd %xmm3, %xmm2
349; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
350; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
351; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
352; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
353; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
354; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
355; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
356; SSSE3-NEXT:    psubd %xmm2, %xmm4
357; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
358; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
359; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
360; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
361; SSSE3-NEXT:    movdqa %xmm0, %xmm1
362; SSSE3-NEXT:    psrad $31, %xmm1
363; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
364; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
365; SSSE3-NEXT:    pxor %xmm1, %xmm0
366; SSSE3-NEXT:    retq
367;
368; SSE41-LABEL: smulo_v4i32:
369; SSE41:       # %bb.0:
370; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
371; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
372; SSE41-NEXT:    pmuldq %xmm2, %xmm3
373; SSE41-NEXT:    movdqa %xmm0, %xmm2
374; SSE41-NEXT:    pmuldq %xmm1, %xmm2
375; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
376; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
377; SSE41-NEXT:    pmulld %xmm1, %xmm0
378; SSE41-NEXT:    movdqa %xmm0, (%rdi)
379; SSE41-NEXT:    psrad $31, %xmm0
380; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
381; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
382; SSE41-NEXT:    pxor %xmm1, %xmm0
383; SSE41-NEXT:    retq
384;
385; AVX1-LABEL: smulo_v4i32:
386; AVX1:       # %bb.0:
387; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
388; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
389; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
390; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
391; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
392; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
393; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
394; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
395; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
396; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
397; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
398; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
399; AVX1-NEXT:    retq
400;
401; AVX2-LABEL: smulo_v4i32:
402; AVX2:       # %bb.0:
403; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
404; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
405; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
406; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
407; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
408; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
409; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
410; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
411; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
412; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
413; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
414; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
415; AVX2-NEXT:    retq
416;
417; AVX512-LABEL: smulo_v4i32:
418; AVX512:       # %bb.0:
419; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
420; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
421; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
422; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
423; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
424; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
425; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
426; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
427; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
428; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
429; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
430; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
431; AVX512-NEXT:    retq
432  %t = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
433  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
434  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
435  %res = sext <4 x i1> %obit to <4 x i32>
436  store <4 x i32> %val, <4 x i32>* %p2
437  ret <4 x i32> %res
438}
439
440define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
441; SSE2-LABEL: smulo_v6i32:
442; SSE2:       # %bb.0:
443; SSE2-NEXT:    movq %rdi, %rax
444; SSE2-NEXT:    movd %r8d, %xmm0
445; SSE2-NEXT:    movd %ecx, %xmm1
446; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
447; SSE2-NEXT:    movd %edx, %xmm0
448; SSE2-NEXT:    movd %esi, %xmm2
449; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
450; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
451; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
452; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
453; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
454; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
455; SSE2-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
456; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
457; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
458; SSE2-NEXT:    movd %r9d, %xmm10
459; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
460; SSE2-NEXT:    movdqa %xmm10, %xmm9
461; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
462; SSE2-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
463; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
464; SSE2-NEXT:    pmuludq %xmm6, %xmm10
465; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
466; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
467; SSE2-NEXT:    pxor %xmm7, %xmm7
468; SSE2-NEXT:    pxor %xmm4, %xmm4
469; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
470; SSE2-NEXT:    pand %xmm2, %xmm4
471; SSE2-NEXT:    pxor %xmm3, %xmm3
472; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
473; SSE2-NEXT:    pand %xmm5, %xmm3
474; SSE2-NEXT:    paddd %xmm4, %xmm3
475; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
476; SSE2-NEXT:    pmuludq %xmm5, %xmm2
477; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
478; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
479; SSE2-NEXT:    pmuludq %xmm4, %xmm5
480; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
481; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
482; SSE2-NEXT:    psubd %xmm3, %xmm0
483; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
484; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
485; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
486; SSE2-NEXT:    movdqa %xmm2, (%rcx)
487; SSE2-NEXT:    psrad $31, %xmm2
488; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
489; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
490; SSE2-NEXT:    pxor %xmm0, %xmm2
491; SSE2-NEXT:    pxor %xmm3, %xmm3
492; SSE2-NEXT:    pcmpgtd %xmm6, %xmm3
493; SSE2-NEXT:    pand %xmm9, %xmm3
494; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
495; SSE2-NEXT:    pand %xmm6, %xmm7
496; SSE2-NEXT:    paddd %xmm3, %xmm7
497; SSE2-NEXT:    pmuludq %xmm8, %xmm1
498; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
499; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
500; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
501; SSE2-NEXT:    psubd %xmm7, %xmm4
502; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
503; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
504; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
505; SSE2-NEXT:    movq %xmm3, 16(%rcx)
506; SSE2-NEXT:    psrad $31, %xmm3
507; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
508; SSE2-NEXT:    pxor %xmm0, %xmm3
509; SSE2-NEXT:    movq %xmm3, 16(%rdi)
510; SSE2-NEXT:    movdqa %xmm2, (%rdi)
511; SSE2-NEXT:    retq
512;
513; SSSE3-LABEL: smulo_v6i32:
514; SSSE3:       # %bb.0:
515; SSSE3-NEXT:    movq %rdi, %rax
516; SSSE3-NEXT:    movd %r8d, %xmm0
517; SSSE3-NEXT:    movd %ecx, %xmm1
518; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
519; SSSE3-NEXT:    movd %edx, %xmm0
520; SSSE3-NEXT:    movd %esi, %xmm2
521; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
522; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
523; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
524; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
525; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
526; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
527; SSSE3-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
528; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
529; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
530; SSSE3-NEXT:    movd %r9d, %xmm10
531; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
532; SSSE3-NEXT:    movdqa %xmm10, %xmm9
533; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
534; SSSE3-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
535; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
536; SSSE3-NEXT:    pmuludq %xmm6, %xmm10
537; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
538; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
539; SSSE3-NEXT:    pxor %xmm7, %xmm7
540; SSSE3-NEXT:    pxor %xmm4, %xmm4
541; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
542; SSSE3-NEXT:    pand %xmm2, %xmm4
543; SSSE3-NEXT:    pxor %xmm3, %xmm3
544; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
545; SSSE3-NEXT:    pand %xmm5, %xmm3
546; SSSE3-NEXT:    paddd %xmm4, %xmm3
547; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
548; SSSE3-NEXT:    pmuludq %xmm5, %xmm2
549; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
550; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
551; SSSE3-NEXT:    pmuludq %xmm4, %xmm5
552; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
553; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
554; SSSE3-NEXT:    psubd %xmm3, %xmm0
555; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
556; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
557; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
558; SSSE3-NEXT:    movdqa %xmm2, (%rcx)
559; SSSE3-NEXT:    psrad $31, %xmm2
560; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
561; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
562; SSSE3-NEXT:    pxor %xmm0, %xmm2
563; SSSE3-NEXT:    pxor %xmm3, %xmm3
564; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm3
565; SSSE3-NEXT:    pand %xmm9, %xmm3
566; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm7
567; SSSE3-NEXT:    pand %xmm6, %xmm7
568; SSSE3-NEXT:    paddd %xmm3, %xmm7
569; SSSE3-NEXT:    pmuludq %xmm8, %xmm1
570; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
571; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
572; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
573; SSSE3-NEXT:    psubd %xmm7, %xmm4
574; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
575; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
576; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
577; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
578; SSSE3-NEXT:    psrad $31, %xmm3
579; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
580; SSSE3-NEXT:    pxor %xmm0, %xmm3
581; SSSE3-NEXT:    movq %xmm3, 16(%rdi)
582; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
583; SSSE3-NEXT:    retq
584;
585; SSE41-LABEL: smulo_v6i32:
586; SSE41:       # %bb.0:
587; SSE41-NEXT:    movq %rdi, %rax
588; SSE41-NEXT:    movd %esi, %xmm2
589; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
590; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
591; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
592; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
593; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
594; SSE41-NEXT:    movdqa %xmm1, %xmm0
595; SSE41-NEXT:    pmuldq %xmm2, %xmm1
596; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
597; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
598; SSE41-NEXT:    movd %r9d, %xmm4
599; SSE41-NEXT:    movdqa %xmm4, %xmm5
600; SSE41-NEXT:    pmuldq %xmm3, %xmm4
601; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
602; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
603; SSE41-NEXT:    pmulld %xmm3, %xmm5
604; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
605; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
606; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
607; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
608; SSE41-NEXT:    pmuldq %xmm3, %xmm6
609; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
610; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
611; SSE41-NEXT:    pmulld %xmm2, %xmm0
612; SSE41-NEXT:    movdqa %xmm0, (%rcx)
613; SSE41-NEXT:    psrad $31, %xmm0
614; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
615; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
616; SSE41-NEXT:    pxor %xmm1, %xmm0
617; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
618; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
619; SSE41-NEXT:    pmuldq %xmm2, %xmm3
620; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
621; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
622; SSE41-NEXT:    movq %xmm5, 16(%rcx)
623; SSE41-NEXT:    psrad $31, %xmm5
624; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
625; SSE41-NEXT:    pxor %xmm1, %xmm5
626; SSE41-NEXT:    movq %xmm5, 16(%rdi)
627; SSE41-NEXT:    movdqa %xmm0, (%rdi)
628; SSE41-NEXT:    retq
629;
630; AVX1-LABEL: smulo_v6i32:
631; AVX1:       # %bb.0:
632; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
633; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
634; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
635; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
636; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
637; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
638; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
639; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
640; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
641; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
642; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
643; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
644; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
645; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
646; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
647; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
648; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
649; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
650; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
651; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
652; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
653; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
654; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
655; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
656; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
657; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
658; AVX1-NEXT:    retq
659;
660; AVX2-LABEL: smulo_v6i32:
661; AVX2:       # %bb.0:
662; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
663; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
664; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
665; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
666; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
667; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
668; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
669; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
670; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
671; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
672; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
673; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
674; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
675; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
676; AVX2-NEXT:    retq
677;
678; AVX512-LABEL: smulo_v6i32:
679; AVX512:       # %bb.0:
680; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
681; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
682; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
683; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
684; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
685; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
686; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
687; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
688; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
689; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
690; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
691; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
692; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
693; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
694; AVX512-NEXT:    retq
695  %t = call {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
696  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
697  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
698  %res = sext <6 x i1> %obit to <6 x i32>
699  store <6 x i32> %val, <6 x i32>* %p2
700  ret <6 x i32> %res
701}
702
703define <8 x i32> @smulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
704; SSE2-LABEL: smulo_v8i32:
705; SSE2:       # %bb.0:
706; SSE2-NEXT:    pxor %xmm4, %xmm4
707; SSE2-NEXT:    pxor %xmm5, %xmm5
708; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
709; SSE2-NEXT:    pand %xmm0, %xmm5
710; SSE2-NEXT:    pxor %xmm6, %xmm6
711; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
712; SSE2-NEXT:    pand %xmm2, %xmm6
713; SSE2-NEXT:    paddd %xmm5, %xmm6
714; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
715; SSE2-NEXT:    pmuludq %xmm2, %xmm0
716; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
717; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
718; SSE2-NEXT:    pmuludq %xmm7, %xmm2
719; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
720; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
721; SSE2-NEXT:    psubd %xmm6, %xmm5
722; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
723; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
724; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
725; SSE2-NEXT:    movdqa %xmm0, (%rdi)
726; SSE2-NEXT:    psrad $31, %xmm0
727; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
728; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
729; SSE2-NEXT:    pxor %xmm2, %xmm0
730; SSE2-NEXT:    pxor %xmm5, %xmm5
731; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
732; SSE2-NEXT:    pand %xmm1, %xmm5
733; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
734; SSE2-NEXT:    pand %xmm3, %xmm4
735; SSE2-NEXT:    paddd %xmm5, %xmm4
736; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
737; SSE2-NEXT:    pmuludq %xmm3, %xmm1
738; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
739; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
740; SSE2-NEXT:    pmuludq %xmm5, %xmm3
741; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
742; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
743; SSE2-NEXT:    psubd %xmm4, %xmm6
744; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
745; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
746; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
747; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
748; SSE2-NEXT:    psrad $31, %xmm1
749; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
750; SSE2-NEXT:    pxor %xmm2, %xmm1
751; SSE2-NEXT:    retq
752;
753; SSSE3-LABEL: smulo_v8i32:
754; SSSE3:       # %bb.0:
755; SSSE3-NEXT:    pxor %xmm4, %xmm4
756; SSSE3-NEXT:    pxor %xmm5, %xmm5
757; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
758; SSSE3-NEXT:    pand %xmm0, %xmm5
759; SSSE3-NEXT:    pxor %xmm6, %xmm6
760; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
761; SSSE3-NEXT:    pand %xmm2, %xmm6
762; SSSE3-NEXT:    paddd %xmm5, %xmm6
763; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
764; SSSE3-NEXT:    pmuludq %xmm2, %xmm0
765; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
766; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
767; SSSE3-NEXT:    pmuludq %xmm7, %xmm2
768; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
769; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
770; SSSE3-NEXT:    psubd %xmm6, %xmm5
771; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
772; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
773; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
774; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
775; SSSE3-NEXT:    psrad $31, %xmm0
776; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
777; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
778; SSSE3-NEXT:    pxor %xmm2, %xmm0
779; SSSE3-NEXT:    pxor %xmm5, %xmm5
780; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
781; SSSE3-NEXT:    pand %xmm1, %xmm5
782; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
783; SSSE3-NEXT:    pand %xmm3, %xmm4
784; SSSE3-NEXT:    paddd %xmm5, %xmm4
785; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
786; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
787; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
788; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
789; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
790; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
791; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
792; SSSE3-NEXT:    psubd %xmm4, %xmm6
793; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
794; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
795; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
796; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
797; SSSE3-NEXT:    psrad $31, %xmm1
798; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm1
799; SSSE3-NEXT:    pxor %xmm2, %xmm1
800; SSSE3-NEXT:    retq
801;
802; SSE41-LABEL: smulo_v8i32:
803; SSE41:       # %bb.0:
804; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
805; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
806; SSE41-NEXT:    pmuldq %xmm4, %xmm5
807; SSE41-NEXT:    movdqa %xmm0, %xmm4
808; SSE41-NEXT:    pmuldq %xmm2, %xmm4
809; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
810; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
811; SSE41-NEXT:    pmulld %xmm2, %xmm0
812; SSE41-NEXT:    movdqa %xmm0, (%rdi)
813; SSE41-NEXT:    psrad $31, %xmm0
814; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
815; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
816; SSE41-NEXT:    pxor %xmm2, %xmm0
817; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
818; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
819; SSE41-NEXT:    pmuldq %xmm4, %xmm5
820; SSE41-NEXT:    movdqa %xmm1, %xmm4
821; SSE41-NEXT:    pmuldq %xmm3, %xmm4
822; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
823; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
824; SSE41-NEXT:    pmulld %xmm3, %xmm1
825; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
826; SSE41-NEXT:    psrad $31, %xmm1
827; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
828; SSE41-NEXT:    pxor %xmm2, %xmm1
829; SSE41-NEXT:    retq
830;
831; AVX1-LABEL: smulo_v8i32:
832; AVX1:       # %bb.0:
833; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
834; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
835; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
836; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
837; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
838; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
839; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
840; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
841; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
842; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
843; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
844; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
845; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
846; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
847; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
848; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
849; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
850; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
851; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
852; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
853; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
854; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
855; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
856; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
857; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
858; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
859; AVX1-NEXT:    retq
860;
861; AVX2-LABEL: smulo_v8i32:
862; AVX2:       # %bb.0:
863; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
864; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
865; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
866; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
867; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
868; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
869; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
870; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
871; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
872; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
873; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
874; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
875; AVX2-NEXT:    retq
876;
877; AVX512-LABEL: smulo_v8i32:
878; AVX512:       # %bb.0:
879; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
880; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
881; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
882; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
883; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
884; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
885; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
886; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
887; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
888; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
889; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
890; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
891; AVX512-NEXT:    retq
892  %t = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
893  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
894  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
895  %res = sext <8 x i1> %obit to <8 x i32>
896  store <8 x i32> %val, <8 x i32>* %p2
897  ret <8 x i32> %res
898}
899
900define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
901; SSE2-LABEL: smulo_v16i32:
902; SSE2:       # %bb.0:
903; SSE2-NEXT:    pxor %xmm8, %xmm8
904; SSE2-NEXT:    pxor %xmm9, %xmm9
905; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
906; SSE2-NEXT:    pand %xmm0, %xmm9
907; SSE2-NEXT:    pxor %xmm10, %xmm10
908; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
909; SSE2-NEXT:    pand %xmm4, %xmm10
910; SSE2-NEXT:    paddd %xmm9, %xmm10
911; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
912; SSE2-NEXT:    pmuludq %xmm4, %xmm0
913; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
914; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
915; SSE2-NEXT:    pmuludq %xmm11, %xmm12
916; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
917; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
918; SSE2-NEXT:    psubd %xmm10, %xmm9
919; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
920; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
921; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
922; SSE2-NEXT:    movdqa %xmm0, (%rdi)
923; SSE2-NEXT:    psrad $31, %xmm0
924; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
925; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
926; SSE2-NEXT:    pxor %xmm9, %xmm0
927; SSE2-NEXT:    pxor %xmm10, %xmm10
928; SSE2-NEXT:    pcmpgtd %xmm5, %xmm10
929; SSE2-NEXT:    pand %xmm1, %xmm10
930; SSE2-NEXT:    pxor %xmm4, %xmm4
931; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
932; SSE2-NEXT:    pand %xmm5, %xmm4
933; SSE2-NEXT:    paddd %xmm10, %xmm4
934; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
935; SSE2-NEXT:    pmuludq %xmm5, %xmm1
936; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
937; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
938; SSE2-NEXT:    pmuludq %xmm10, %xmm12
939; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
940; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
941; SSE2-NEXT:    psubd %xmm4, %xmm11
942; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
943; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
944; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
945; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
946; SSE2-NEXT:    psrad $31, %xmm1
947; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
948; SSE2-NEXT:    pxor %xmm9, %xmm1
949; SSE2-NEXT:    pxor %xmm4, %xmm4
950; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
951; SSE2-NEXT:    pand %xmm2, %xmm4
952; SSE2-NEXT:    pxor %xmm5, %xmm5
953; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
954; SSE2-NEXT:    pand %xmm6, %xmm5
955; SSE2-NEXT:    paddd %xmm4, %xmm5
956; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
957; SSE2-NEXT:    pmuludq %xmm6, %xmm2
958; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
959; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
960; SSE2-NEXT:    pmuludq %xmm10, %xmm11
961; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
962; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
963; SSE2-NEXT:    psubd %xmm5, %xmm4
964; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
965; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
966; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
967; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
968; SSE2-NEXT:    psrad $31, %xmm2
969; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
970; SSE2-NEXT:    pxor %xmm9, %xmm2
971; SSE2-NEXT:    pxor %xmm4, %xmm4
972; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
973; SSE2-NEXT:    pand %xmm3, %xmm4
974; SSE2-NEXT:    pcmpgtd %xmm3, %xmm8
975; SSE2-NEXT:    pand %xmm7, %xmm8
976; SSE2-NEXT:    paddd %xmm4, %xmm8
977; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
978; SSE2-NEXT:    pmuludq %xmm7, %xmm3
979; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
980; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
981; SSE2-NEXT:    pmuludq %xmm4, %xmm6
982; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
983; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
984; SSE2-NEXT:    psubd %xmm8, %xmm5
985; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
986; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
987; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
988; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
989; SSE2-NEXT:    psrad $31, %xmm3
990; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
991; SSE2-NEXT:    pxor %xmm9, %xmm3
992; SSE2-NEXT:    retq
993;
994; SSSE3-LABEL: smulo_v16i32:
995; SSSE3:       # %bb.0:
996; SSSE3-NEXT:    pxor %xmm8, %xmm8
997; SSSE3-NEXT:    pxor %xmm9, %xmm9
998; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
999; SSSE3-NEXT:    pand %xmm0, %xmm9
1000; SSSE3-NEXT:    pxor %xmm10, %xmm10
1001; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm10
1002; SSSE3-NEXT:    pand %xmm4, %xmm10
1003; SSSE3-NEXT:    paddd %xmm9, %xmm10
1004; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
1005; SSSE3-NEXT:    pmuludq %xmm4, %xmm0
1006; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
1007; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
1008; SSSE3-NEXT:    pmuludq %xmm11, %xmm12
1009; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
1010; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
1011; SSSE3-NEXT:    psubd %xmm10, %xmm9
1012; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1013; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1014; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1015; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
1016; SSSE3-NEXT:    psrad $31, %xmm0
1017; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
1018; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
1019; SSSE3-NEXT:    pxor %xmm9, %xmm0
1020; SSSE3-NEXT:    pxor %xmm10, %xmm10
1021; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm10
1022; SSSE3-NEXT:    pand %xmm1, %xmm10
1023; SSSE3-NEXT:    pxor %xmm4, %xmm4
1024; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
1025; SSSE3-NEXT:    pand %xmm5, %xmm4
1026; SSSE3-NEXT:    paddd %xmm10, %xmm4
1027; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1028; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
1029; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
1030; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
1031; SSSE3-NEXT:    pmuludq %xmm10, %xmm12
1032; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
1033; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
1034; SSSE3-NEXT:    psubd %xmm4, %xmm11
1035; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1036; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1037; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1038; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
1039; SSSE3-NEXT:    psrad $31, %xmm1
1040; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
1041; SSSE3-NEXT:    pxor %xmm9, %xmm1
1042; SSSE3-NEXT:    pxor %xmm4, %xmm4
1043; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
1044; SSSE3-NEXT:    pand %xmm2, %xmm4
1045; SSSE3-NEXT:    pxor %xmm5, %xmm5
1046; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
1047; SSSE3-NEXT:    pand %xmm6, %xmm5
1048; SSSE3-NEXT:    paddd %xmm4, %xmm5
1049; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
1050; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
1051; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
1052; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
1053; SSSE3-NEXT:    pmuludq %xmm10, %xmm11
1054; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
1055; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
1056; SSSE3-NEXT:    psubd %xmm5, %xmm4
1057; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
1058; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
1059; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1060; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
1061; SSSE3-NEXT:    psrad $31, %xmm2
1062; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
1063; SSSE3-NEXT:    pxor %xmm9, %xmm2
1064; SSSE3-NEXT:    pxor %xmm4, %xmm4
1065; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
1066; SSSE3-NEXT:    pand %xmm3, %xmm4
1067; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm8
1068; SSSE3-NEXT:    pand %xmm7, %xmm8
1069; SSSE3-NEXT:    paddd %xmm4, %xmm8
1070; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
1071; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
1072; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
1073; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
1074; SSSE3-NEXT:    pmuludq %xmm4, %xmm6
1075; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
1076; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
1077; SSSE3-NEXT:    psubd %xmm8, %xmm5
1078; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
1079; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
1080; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1081; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
1082; SSSE3-NEXT:    psrad $31, %xmm3
1083; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
1084; SSSE3-NEXT:    pxor %xmm9, %xmm3
1085; SSSE3-NEXT:    retq
1086;
1087; SSE41-LABEL: smulo_v16i32:
1088; SSE41:       # %bb.0:
1089; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
1090; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
1091; SSE41-NEXT:    pmuldq %xmm8, %xmm9
1092; SSE41-NEXT:    movdqa %xmm0, %xmm8
1093; SSE41-NEXT:    pmuldq %xmm4, %xmm8
1094; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
1095; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
1096; SSE41-NEXT:    pmulld %xmm4, %xmm0
1097; SSE41-NEXT:    movdqa %xmm0, (%rdi)
1098; SSE41-NEXT:    psrad $31, %xmm0
1099; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
1100; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1101; SSE41-NEXT:    pxor %xmm8, %xmm0
1102; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
1103; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1104; SSE41-NEXT:    pmuldq %xmm9, %xmm10
1105; SSE41-NEXT:    movdqa %xmm1, %xmm4
1106; SSE41-NEXT:    pmuldq %xmm5, %xmm4
1107; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1108; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm10[2,3],xmm4[4,5],xmm10[6,7]
1109; SSE41-NEXT:    pmulld %xmm5, %xmm1
1110; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
1111; SSE41-NEXT:    psrad $31, %xmm1
1112; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
1113; SSE41-NEXT:    pxor %xmm8, %xmm1
1114; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
1115; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
1116; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1117; SSE41-NEXT:    movdqa %xmm2, %xmm4
1118; SSE41-NEXT:    pmuldq %xmm6, %xmm4
1119; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1120; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1121; SSE41-NEXT:    pmulld %xmm6, %xmm2
1122; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
1123; SSE41-NEXT:    psrad $31, %xmm2
1124; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
1125; SSE41-NEXT:    pxor %xmm8, %xmm2
1126; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
1127; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
1128; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1129; SSE41-NEXT:    movdqa %xmm3, %xmm4
1130; SSE41-NEXT:    pmuldq %xmm7, %xmm4
1131; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1132; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1133; SSE41-NEXT:    pmulld %xmm7, %xmm3
1134; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
1135; SSE41-NEXT:    psrad $31, %xmm3
1136; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
1137; SSE41-NEXT:    pxor %xmm8, %xmm3
1138; SSE41-NEXT:    retq
1139;
1140; AVX1-LABEL: smulo_v16i32:
1141; AVX1:       # %bb.0:
1142; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
1143; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
1144; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
1145; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
1146; AVX1-NEXT:    vpmuldq %xmm5, %xmm7, %xmm5
1147; AVX1-NEXT:    vpmuldq %xmm4, %xmm6, %xmm7
1148; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1149; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1150; AVX1-NEXT:    vpmulld %xmm4, %xmm6, %xmm8
1151; AVX1-NEXT:    vpsrad $31, %xmm8, %xmm6
1152; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm6
1153; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
1154; AVX1-NEXT:    vpxor %xmm6, %xmm9, %xmm6
1155; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
1156; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
1157; AVX1-NEXT:    vpmuldq %xmm7, %xmm4, %xmm4
1158; AVX1-NEXT:    vpmuldq %xmm3, %xmm1, %xmm7
1159; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1160; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3],xmm7[4,5],xmm4[6,7]
1161; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1162; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm1
1163; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
1164; AVX1-NEXT:    vpxor %xmm1, %xmm9, %xmm1
1165; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
1166; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1167; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
1168; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1169; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
1170; AVX1-NEXT:    vpmuldq %xmm6, %xmm5, %xmm5
1171; AVX1-NEXT:    vpmuldq %xmm4, %xmm7, %xmm6
1172; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
1173; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
1174; AVX1-NEXT:    vpmulld %xmm4, %xmm7, %xmm4
1175; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm6
1176; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm5
1177; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
1178; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
1179; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
1180; AVX1-NEXT:    vpmuldq %xmm6, %xmm7, %xmm6
1181; AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm7
1182; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1183; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
1184; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1185; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm0
1186; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
1187; AVX1-NEXT:    vpxor %xmm0, %xmm9, %xmm0
1188; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
1189; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
1190; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
1191; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1192; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
1193; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
1194; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1195; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1196; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1197; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1198; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1199; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
1200; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1201; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1202; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1203; AVX1-NEXT:    retq
1204;
1205; AVX2-LABEL: smulo_v16i32:
1206; AVX2:       # %bb.0:
1207; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1208; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1209; AVX2-NEXT:    vpmuldq %ymm4, %ymm5, %ymm4
1210; AVX2-NEXT:    vpmuldq %ymm3, %ymm1, %ymm5
1211; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1212; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1213; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1214; AVX2-NEXT:    vpsrad $31, %ymm3, %ymm1
1215; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm4, %ymm1
1216; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
1217; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
1218; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1219; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
1220; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[1,1,3,3,5,5,7,7]
1221; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm0[1,1,3,3,5,5,7,7]
1222; AVX2-NEXT:    vpmuldq %ymm5, %ymm6, %ymm5
1223; AVX2-NEXT:    vpmuldq %ymm2, %ymm0, %ymm6
1224; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,1,3,3,5,5,7,7]
1225; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
1226; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1227; AVX2-NEXT:    vpsrad $31, %ymm2, %ymm0
1228; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
1229; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
1230; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
1231; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
1232; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
1233; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
1234; AVX2-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1235; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1236; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1237; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1238; AVX2-NEXT:    retq
1239;
1240; AVX512-LABEL: smulo_v16i32:
1241; AVX512:       # %bb.0:
1242; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm2
1243; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1244; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1245; AVX512-NEXT:    vpmuldq %zmm3, %zmm4, %zmm3
1246; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1247; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1248; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1249; AVX512-NEXT:    vpsrad $31, %zmm1, %zmm0
1250; AVX512-NEXT:    vpcmpneqd %zmm0, %zmm4, %k1
1251; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1252; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1253; AVX512-NEXT:    retq
1254  %t = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1255  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1256  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1257  %res = sext <16 x i1> %obit to <16 x i32>
1258  store <16 x i32> %val, <16 x i32>* %p2
1259  ret <16 x i32> %res
1260}
1261
1262define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1263; SSE2-LABEL: smulo_v16i8:
1264; SSE2:       # %bb.0:
1265; SSE2-NEXT:    movdqa %xmm1, %xmm2
1266; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1267; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1268; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1269; SSE2-NEXT:    movdqa %xmm0, %xmm4
1270; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1271; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1272; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1273; SSE2-NEXT:    pmullw %xmm1, %xmm0
1274; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1275; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1276; SSE2-NEXT:    pmullw %xmm2, %xmm4
1277; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1278; SSE2-NEXT:    pand %xmm1, %xmm0
1279; SSE2-NEXT:    pand %xmm1, %xmm4
1280; SSE2-NEXT:    packuswb %xmm0, %xmm4
1281; SSE2-NEXT:    pxor %xmm0, %xmm0
1282; SSE2-NEXT:    pcmpgtb %xmm4, %xmm0
1283; SSE2-NEXT:    psraw $8, %xmm3
1284; SSE2-NEXT:    psraw $8, %xmm6
1285; SSE2-NEXT:    pmullw %xmm3, %xmm6
1286; SSE2-NEXT:    psrlw $8, %xmm6
1287; SSE2-NEXT:    psraw $8, %xmm5
1288; SSE2-NEXT:    psraw $8, %xmm7
1289; SSE2-NEXT:    pmullw %xmm5, %xmm7
1290; SSE2-NEXT:    psrlw $8, %xmm7
1291; SSE2-NEXT:    packuswb %xmm6, %xmm7
1292; SSE2-NEXT:    pcmpeqb %xmm0, %xmm7
1293; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1294; SSE2-NEXT:    pxor %xmm7, %xmm3
1295; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
1296; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1297; SSE2-NEXT:    psrad $24, %xmm0
1298; SSE2-NEXT:    movdqa %xmm3, %xmm1
1299; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1300; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1301; SSE2-NEXT:    pslld $31, %xmm1
1302; SSE2-NEXT:    psrad $31, %xmm1
1303; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1304; SSE2-NEXT:    movdqa %xmm3, %xmm2
1305; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1306; SSE2-NEXT:    pslld $31, %xmm2
1307; SSE2-NEXT:    psrad $31, %xmm2
1308; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1309; SSE2-NEXT:    pslld $31, %xmm3
1310; SSE2-NEXT:    psrad $31, %xmm3
1311; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1312; SSE2-NEXT:    retq
1313;
1314; SSSE3-LABEL: smulo_v16i8:
1315; SSSE3:       # %bb.0:
1316; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1317; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1318; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1319; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1320; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1321; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1322; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1323; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1324; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1325; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1326; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1327; SSSE3-NEXT:    pmullw %xmm2, %xmm4
1328; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1329; SSSE3-NEXT:    pand %xmm1, %xmm0
1330; SSSE3-NEXT:    pand %xmm1, %xmm4
1331; SSSE3-NEXT:    packuswb %xmm0, %xmm4
1332; SSSE3-NEXT:    pxor %xmm0, %xmm0
1333; SSSE3-NEXT:    pcmpgtb %xmm4, %xmm0
1334; SSSE3-NEXT:    psraw $8, %xmm3
1335; SSSE3-NEXT:    psraw $8, %xmm6
1336; SSSE3-NEXT:    pmullw %xmm3, %xmm6
1337; SSSE3-NEXT:    psrlw $8, %xmm6
1338; SSSE3-NEXT:    psraw $8, %xmm5
1339; SSSE3-NEXT:    psraw $8, %xmm7
1340; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1341; SSSE3-NEXT:    psrlw $8, %xmm7
1342; SSSE3-NEXT:    packuswb %xmm6, %xmm7
1343; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm7
1344; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1345; SSSE3-NEXT:    pxor %xmm7, %xmm3
1346; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
1347; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1348; SSSE3-NEXT:    psrad $24, %xmm0
1349; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1350; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1351; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1352; SSSE3-NEXT:    pslld $31, %xmm1
1353; SSSE3-NEXT:    psrad $31, %xmm1
1354; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1355; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1356; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1357; SSSE3-NEXT:    pslld $31, %xmm2
1358; SSSE3-NEXT:    psrad $31, %xmm2
1359; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1360; SSSE3-NEXT:    pslld $31, %xmm3
1361; SSSE3-NEXT:    psrad $31, %xmm3
1362; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1363; SSSE3-NEXT:    retq
1364;
1365; SSE41-LABEL: smulo_v16i8:
1366; SSE41:       # %bb.0:
1367; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1368; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
1369; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
1370; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1371; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1372; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
1373; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
1374; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1375; SSE41-NEXT:    pmullw %xmm1, %xmm0
1376; SSE41-NEXT:    pmullw %xmm2, %xmm4
1377; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1378; SSE41-NEXT:    pand %xmm1, %xmm0
1379; SSE41-NEXT:    pand %xmm1, %xmm4
1380; SSE41-NEXT:    packuswb %xmm0, %xmm4
1381; SSE41-NEXT:    pxor %xmm0, %xmm0
1382; SSE41-NEXT:    pcmpgtb %xmm4, %xmm0
1383; SSE41-NEXT:    pmullw %xmm3, %xmm6
1384; SSE41-NEXT:    psrlw $8, %xmm6
1385; SSE41-NEXT:    pmovsxbw %xmm5, %xmm1
1386; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1387; SSE41-NEXT:    pmullw %xmm1, %xmm2
1388; SSE41-NEXT:    psrlw $8, %xmm2
1389; SSE41-NEXT:    packuswb %xmm2, %xmm6
1390; SSE41-NEXT:    pcmpeqb %xmm0, %xmm6
1391; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1392; SSE41-NEXT:    pxor %xmm6, %xmm3
1393; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
1394; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
1395; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1396; SSE41-NEXT:    pslld $31, %xmm1
1397; SSE41-NEXT:    psrad $31, %xmm1
1398; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1399; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1400; SSE41-NEXT:    pslld $31, %xmm2
1401; SSE41-NEXT:    psrad $31, %xmm2
1402; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1403; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1404; SSE41-NEXT:    pslld $31, %xmm3
1405; SSE41-NEXT:    psrad $31, %xmm3
1406; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1407; SSE41-NEXT:    retq
1408;
1409; AVX1-LABEL: smulo_v16i8:
1410; AVX1:       # %bb.0:
1411; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1412; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1413; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1414; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1415; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1416; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1417; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1418; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1419; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1420; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1421; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1422; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1423; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm4
1424; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
1425; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1426; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1427; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1428; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1429; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1430; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1431; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1432; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1433; AVX1-NEXT:    vpackuswb %xmm0, %xmm4, %xmm0
1434; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1435; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1436; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1437; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1438; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
1439; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1440; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1441; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1442; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1443; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1444; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1445; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1446; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1447; AVX1-NEXT:    retq
1448;
1449; AVX2-LABEL: smulo_v16i8:
1450; AVX2:       # %bb.0:
1451; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm2
1452; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm3
1453; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1454; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1455; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
1456; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
1457; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1458; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1459; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1460; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1461; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1462; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm3
1463; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
1464; AVX2-NEXT:    vpcmpgtb %xmm3, %xmm0, %xmm0
1465; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
1466; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1467; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1468; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1469; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1470; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1471; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
1472; AVX2-NEXT:    retq
1473;
1474; AVX512F-LABEL: smulo_v16i8:
1475; AVX512F:       # %bb.0:
1476; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1477; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1478; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1479; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1480; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1481; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1482; AVX512F-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1483; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1484; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1485; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1486; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
1487; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1488; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1489; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1490; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1491; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1492; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
1493; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1494; AVX512F-NEXT:    vmovdqa %xmm2, (%rdi)
1495; AVX512F-NEXT:    retq
1496;
1497; AVX512BW-LABEL: smulo_v16i8:
1498; AVX512BW:       # %bb.0:
1499; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1500; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1501; AVX512BW-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1502; AVX512BW-NEXT:    vpmovwb %ymm2, %xmm2
1503; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1504; AVX512BW-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1505; AVX512BW-NEXT:    vpmovsxbw %xmm1, %ymm1
1506; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
1507; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1508; AVX512BW-NEXT:    vpsrlw $8, %ymm0, %ymm0
1509; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
1510; AVX512BW-NEXT:    vpcmpneqb %xmm3, %xmm0, %k1
1511; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1512; AVX512BW-NEXT:    vmovdqa %xmm2, (%rdi)
1513; AVX512BW-NEXT:    retq
1514  %t = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1515  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1516  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1517  %res = sext <16 x i1> %obit to <16 x i32>
1518  store <16 x i8> %val, <16 x i8>* %p2
1519  ret <16 x i32> %res
1520}
1521
1522define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nounwind {
1523; SSE2-LABEL: smulo_v32i8:
1524; SSE2:       # %bb.0:
1525; SSE2-NEXT:    movq %rdi, %rax
1526; SSE2-NEXT:    movdqa %xmm3, %xmm9
1527; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
1528; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
1529; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1530; SSE2-NEXT:    movdqa %xmm1, %xmm10
1531; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1532; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
1533; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1534; SSE2-NEXT:    pmullw %xmm3, %xmm1
1535; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1536; SSE2-NEXT:    pand %xmm8, %xmm1
1537; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1538; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1539; SSE2-NEXT:    pmullw %xmm9, %xmm10
1540; SSE2-NEXT:    pand %xmm8, %xmm10
1541; SSE2-NEXT:    packuswb %xmm1, %xmm10
1542; SSE2-NEXT:    pxor %xmm1, %xmm1
1543; SSE2-NEXT:    pcmpgtb %xmm10, %xmm1
1544; SSE2-NEXT:    psraw $8, %xmm4
1545; SSE2-NEXT:    psraw $8, %xmm6
1546; SSE2-NEXT:    pmullw %xmm4, %xmm6
1547; SSE2-NEXT:    psrlw $8, %xmm6
1548; SSE2-NEXT:    psraw $8, %xmm5
1549; SSE2-NEXT:    psraw $8, %xmm7
1550; SSE2-NEXT:    pmullw %xmm5, %xmm7
1551; SSE2-NEXT:    psrlw $8, %xmm7
1552; SSE2-NEXT:    packuswb %xmm6, %xmm7
1553; SSE2-NEXT:    pcmpeqb %xmm1, %xmm7
1554; SSE2-NEXT:    movdqa %xmm2, %xmm9
1555; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1556; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1557; SSE2-NEXT:    movdqa %xmm2, %xmm4
1558; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1559; SSE2-NEXT:    movdqa %xmm0, %xmm11
1560; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1561; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1562; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1563; SSE2-NEXT:    pmullw %xmm4, %xmm0
1564; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1565; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1566; SSE2-NEXT:    pmullw %xmm9, %xmm11
1567; SSE2-NEXT:    pxor %xmm4, %xmm4
1568; SSE2-NEXT:    pand %xmm8, %xmm0
1569; SSE2-NEXT:    pand %xmm8, %xmm11
1570; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
1571; SSE2-NEXT:    pxor %xmm8, %xmm7
1572; SSE2-NEXT:    packuswb %xmm0, %xmm11
1573; SSE2-NEXT:    pcmpgtb %xmm11, %xmm4
1574; SSE2-NEXT:    psraw $8, %xmm5
1575; SSE2-NEXT:    psraw $8, %xmm3
1576; SSE2-NEXT:    pmullw %xmm5, %xmm3
1577; SSE2-NEXT:    psrlw $8, %xmm3
1578; SSE2-NEXT:    psraw $8, %xmm6
1579; SSE2-NEXT:    psraw $8, %xmm2
1580; SSE2-NEXT:    pmullw %xmm6, %xmm2
1581; SSE2-NEXT:    psrlw $8, %xmm2
1582; SSE2-NEXT:    packuswb %xmm3, %xmm2
1583; SSE2-NEXT:    pcmpeqb %xmm4, %xmm2
1584; SSE2-NEXT:    pxor %xmm8, %xmm2
1585; SSE2-NEXT:    movdqa %xmm2, %xmm0
1586; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3],xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
1587; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1588; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1589; SSE2-NEXT:    pslld $31, %xmm2
1590; SSE2-NEXT:    psrad $31, %xmm2
1591; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1592; SSE2-NEXT:    movdqa %xmm0, %xmm5
1593; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1594; SSE2-NEXT:    pslld $31, %xmm5
1595; SSE2-NEXT:    psrad $31, %xmm5
1596; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1597; SSE2-NEXT:    pslld $31, %xmm0
1598; SSE2-NEXT:    psrad $31, %xmm0
1599; SSE2-NEXT:    movdqa %xmm7, %xmm6
1600; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1601; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
1602; SSE2-NEXT:    pslld $31, %xmm6
1603; SSE2-NEXT:    psrad $31, %xmm6
1604; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
1605; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1606; SSE2-NEXT:    movdqa %xmm7, %xmm4
1607; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1608; SSE2-NEXT:    pslld $31, %xmm4
1609; SSE2-NEXT:    psrad $31, %xmm4
1610; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]
1611; SSE2-NEXT:    pslld $31, %xmm7
1612; SSE2-NEXT:    psrad $31, %xmm7
1613; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
1614; SSE2-NEXT:    psrad $24, %xmm3
1615; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]
1616; SSE2-NEXT:    psrad $24, %xmm1
1617; SSE2-NEXT:    movdqa %xmm10, 16(%rsi)
1618; SSE2-NEXT:    movdqa %xmm11, (%rsi)
1619; SSE2-NEXT:    movdqa %xmm1, 64(%rdi)
1620; SSE2-NEXT:    movdqa %xmm3, (%rdi)
1621; SSE2-NEXT:    movdqa %xmm7, 112(%rdi)
1622; SSE2-NEXT:    movdqa %xmm4, 96(%rdi)
1623; SSE2-NEXT:    movdqa %xmm6, 80(%rdi)
1624; SSE2-NEXT:    movdqa %xmm0, 48(%rdi)
1625; SSE2-NEXT:    movdqa %xmm5, 32(%rdi)
1626; SSE2-NEXT:    movdqa %xmm2, 16(%rdi)
1627; SSE2-NEXT:    retq
1628;
1629; SSSE3-LABEL: smulo_v32i8:
1630; SSSE3:       # %bb.0:
1631; SSSE3-NEXT:    movq %rdi, %rax
1632; SSSE3-NEXT:    movdqa %xmm3, %xmm9
1633; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
1634; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
1635; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1636; SSSE3-NEXT:    movdqa %xmm1, %xmm10
1637; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1638; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
1639; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1640; SSSE3-NEXT:    pmullw %xmm3, %xmm1
1641; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1642; SSSE3-NEXT:    pand %xmm8, %xmm1
1643; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1644; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1645; SSSE3-NEXT:    pmullw %xmm9, %xmm10
1646; SSSE3-NEXT:    pand %xmm8, %xmm10
1647; SSSE3-NEXT:    packuswb %xmm1, %xmm10
1648; SSSE3-NEXT:    pxor %xmm1, %xmm1
1649; SSSE3-NEXT:    pcmpgtb %xmm10, %xmm1
1650; SSSE3-NEXT:    psraw $8, %xmm4
1651; SSSE3-NEXT:    psraw $8, %xmm6
1652; SSSE3-NEXT:    pmullw %xmm4, %xmm6
1653; SSSE3-NEXT:    psrlw $8, %xmm6
1654; SSSE3-NEXT:    psraw $8, %xmm5
1655; SSSE3-NEXT:    psraw $8, %xmm7
1656; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1657; SSSE3-NEXT:    psrlw $8, %xmm7
1658; SSSE3-NEXT:    packuswb %xmm6, %xmm7
1659; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm7
1660; SSSE3-NEXT:    movdqa %xmm2, %xmm9
1661; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1662; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1663; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1664; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1665; SSSE3-NEXT:    movdqa %xmm0, %xmm11
1666; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1667; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1668; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1669; SSSE3-NEXT:    pmullw %xmm4, %xmm0
1670; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1671; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1672; SSSE3-NEXT:    pmullw %xmm9, %xmm11
1673; SSSE3-NEXT:    pxor %xmm4, %xmm4
1674; SSSE3-NEXT:    pand %xmm8, %xmm0
1675; SSSE3-NEXT:    pand %xmm8, %xmm11
1676; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
1677; SSSE3-NEXT:    pxor %xmm8, %xmm7
1678; SSSE3-NEXT:    packuswb %xmm0, %xmm11
1679; SSSE3-NEXT:    pcmpgtb %xmm11, %xmm4
1680; SSSE3-NEXT:    psraw $8, %xmm5
1681; SSSE3-NEXT:    psraw $8, %xmm3
1682; SSSE3-NEXT:    pmullw %xmm5, %xmm3
1683; SSSE3-NEXT:    psrlw $8, %xmm3
1684; SSSE3-NEXT:    psraw $8, %xmm6
1685; SSSE3-NEXT:    psraw $8, %xmm2
1686; SSSE3-NEXT:    pmullw %xmm6, %xmm2
1687; SSSE3-NEXT:    psrlw $8, %xmm2
1688; SSSE3-NEXT:    packuswb %xmm3, %xmm2
1689; SSSE3-NEXT:    pcmpeqb %xmm4, %xmm2
1690; SSSE3-NEXT:    pxor %xmm8, %xmm2
1691; SSSE3-NEXT:    movdqa %xmm2, %xmm0
1692; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3],xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
1693; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1694; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1695; SSSE3-NEXT:    pslld $31, %xmm2
1696; SSSE3-NEXT:    psrad $31, %xmm2
1697; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1698; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1699; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1700; SSSE3-NEXT:    pslld $31, %xmm5
1701; SSSE3-NEXT:    psrad $31, %xmm5
1702; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1703; SSSE3-NEXT:    pslld $31, %xmm0
1704; SSSE3-NEXT:    psrad $31, %xmm0
1705; SSSE3-NEXT:    movdqa %xmm7, %xmm6
1706; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1707; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
1708; SSSE3-NEXT:    pslld $31, %xmm6
1709; SSSE3-NEXT:    psrad $31, %xmm6
1710; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]
1711; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1712; SSSE3-NEXT:    movdqa %xmm7, %xmm4
1713; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1714; SSSE3-NEXT:    pslld $31, %xmm4
1715; SSSE3-NEXT:    psrad $31, %xmm4
1716; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]
1717; SSSE3-NEXT:    pslld $31, %xmm7
1718; SSSE3-NEXT:    psrad $31, %xmm7
1719; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
1720; SSSE3-NEXT:    psrad $24, %xmm3
1721; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]
1722; SSSE3-NEXT:    psrad $24, %xmm1
1723; SSSE3-NEXT:    movdqa %xmm10, 16(%rsi)
1724; SSSE3-NEXT:    movdqa %xmm11, (%rsi)
1725; SSSE3-NEXT:    movdqa %xmm1, 64(%rdi)
1726; SSSE3-NEXT:    movdqa %xmm3, (%rdi)
1727; SSSE3-NEXT:    movdqa %xmm7, 112(%rdi)
1728; SSSE3-NEXT:    movdqa %xmm4, 96(%rdi)
1729; SSSE3-NEXT:    movdqa %xmm6, 80(%rdi)
1730; SSSE3-NEXT:    movdqa %xmm0, 48(%rdi)
1731; SSSE3-NEXT:    movdqa %xmm5, 32(%rdi)
1732; SSSE3-NEXT:    movdqa %xmm2, 16(%rdi)
1733; SSSE3-NEXT:    retq
1734;
1735; SSE41-LABEL: smulo_v32i8:
1736; SSE41:       # %bb.0:
1737; SSE41-NEXT:    movq %rdi, %rax
1738; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1739; SSE41-NEXT:    pmovsxbw %xmm3, %xmm10
1740; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
1741; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1742; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1743; SSE41-NEXT:    pmovsxbw %xmm1, %xmm4
1744; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
1745; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1746; SSE41-NEXT:    pmullw %xmm3, %xmm1
1747; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1748; SSE41-NEXT:    pand %xmm8, %xmm1
1749; SSE41-NEXT:    pmullw %xmm9, %xmm11
1750; SSE41-NEXT:    pand %xmm8, %xmm11
1751; SSE41-NEXT:    packuswb %xmm1, %xmm11
1752; SSE41-NEXT:    pxor %xmm1, %xmm1
1753; SSE41-NEXT:    pcmpgtb %xmm11, %xmm1
1754; SSE41-NEXT:    pmullw %xmm10, %xmm4
1755; SSE41-NEXT:    psrlw $8, %xmm4
1756; SSE41-NEXT:    pmovsxbw %xmm5, %xmm3
1757; SSE41-NEXT:    pmovsxbw %xmm7, %xmm5
1758; SSE41-NEXT:    pmullw %xmm3, %xmm5
1759; SSE41-NEXT:    psrlw $8, %xmm5
1760; SSE41-NEXT:    packuswb %xmm5, %xmm4
1761; SSE41-NEXT:    pcmpeqb %xmm1, %xmm4
1762; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1763; SSE41-NEXT:    pmovsxbw %xmm2, %xmm10
1764; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,2,3]
1765; SSE41-NEXT:    movdqa %xmm2, %xmm3
1766; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1767; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1768; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
1769; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
1770; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1771; SSE41-NEXT:    pmullw %xmm3, %xmm0
1772; SSE41-NEXT:    pmullw %xmm9, %xmm1
1773; SSE41-NEXT:    pxor %xmm3, %xmm3
1774; SSE41-NEXT:    pand %xmm8, %xmm0
1775; SSE41-NEXT:    pand %xmm8, %xmm1
1776; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1777; SSE41-NEXT:    pxor %xmm8, %xmm4
1778; SSE41-NEXT:    packuswb %xmm0, %xmm1
1779; SSE41-NEXT:    pcmpgtb %xmm1, %xmm3
1780; SSE41-NEXT:    pmullw %xmm10, %xmm2
1781; SSE41-NEXT:    psrlw $8, %xmm2
1782; SSE41-NEXT:    pmovsxbw %xmm7, %xmm0
1783; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
1784; SSE41-NEXT:    pmullw %xmm0, %xmm5
1785; SSE41-NEXT:    psrlw $8, %xmm5
1786; SSE41-NEXT:    packuswb %xmm5, %xmm2
1787; SSE41-NEXT:    pcmpeqb %xmm3, %xmm2
1788; SSE41-NEXT:    pxor %xmm8, %xmm2
1789; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
1790; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1791; SSE41-NEXT:    pslld $31, %xmm8
1792; SSE41-NEXT:    psrad $31, %xmm8
1793; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
1794; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1795; SSE41-NEXT:    pslld $31, %xmm5
1796; SSE41-NEXT:    psrad $31, %xmm5
1797; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
1798; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1799; SSE41-NEXT:    pslld $31, %xmm3
1800; SSE41-NEXT:    psrad $31, %xmm3
1801; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
1802; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
1803; SSE41-NEXT:    pslld $31, %xmm7
1804; SSE41-NEXT:    psrad $31, %xmm7
1805; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
1806; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
1807; SSE41-NEXT:    pslld $31, %xmm6
1808; SSE41-NEXT:    psrad $31, %xmm6
1809; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
1810; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1811; SSE41-NEXT:    pslld $31, %xmm0
1812; SSE41-NEXT:    psrad $31, %xmm0
1813; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
1814; SSE41-NEXT:    pmovsxbd %xmm4, %xmm4
1815; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)
1816; SSE41-NEXT:    movdqa %xmm1, (%rsi)
1817; SSE41-NEXT:    movdqa %xmm4, 64(%rdi)
1818; SSE41-NEXT:    movdqa %xmm2, (%rdi)
1819; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
1820; SSE41-NEXT:    movdqa %xmm6, 96(%rdi)
1821; SSE41-NEXT:    movdqa %xmm7, 80(%rdi)
1822; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
1823; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)
1824; SSE41-NEXT:    movdqa %xmm8, 16(%rdi)
1825; SSE41-NEXT:    retq
1826;
1827; AVX1-LABEL: smulo_v32i8:
1828; AVX1:       # %bb.0:
1829; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1830; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1831; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
1832; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1833; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
1834; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
1835; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
1836; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1837; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
1838; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
1839; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
1840; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm9
1841; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
1842; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm8, %xmm3
1843; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm7
1844; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
1845; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
1846; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1847; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
1848; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
1849; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
1850; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
1851; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1852; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1853; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
1854; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
1855; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1856; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm3
1857; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1858; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1859; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1860; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
1861; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1862; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1863; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
1864; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
1865; AVX1-NEXT:    vpackuswb %xmm2, %xmm5, %xmm5
1866; AVX1-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
1867; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm6
1868; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm7
1869; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
1870; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
1871; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1872; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1873; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1874; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1875; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1876; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1877; AVX1-NEXT:    vpackuswb %xmm0, %xmm6, %xmm0
1878; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
1879; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
1880; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1881; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1882; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
1883; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1884; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm2
1885; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
1886; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1887; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
1888; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
1889; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1890; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1891; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1892; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
1893; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
1894; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1895; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1896; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1897; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1898; AVX1-NEXT:    vmovdqa %xmm9, 16(%rdi)
1899; AVX1-NEXT:    vmovdqa %xmm5, (%rdi)
1900; AVX1-NEXT:    retq
1901;
1902; AVX2-LABEL: smulo_v32i8:
1903; AVX2:       # %bb.0:
1904; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1905; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1906; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1907; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1908; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
1909; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1910; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1911; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1912; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
1913; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
1914; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1915; AVX2-NEXT:    vpcmpgtb %ymm4, %ymm2, %ymm2
1916; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1917; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
1918; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1919; AVX2-NEXT:    vpsraw $8, %ymm5, %ymm5
1920; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
1921; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
1922; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1923; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
1924; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1925; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
1926; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1927; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1928; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
1929; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
1930; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
1931; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
1932; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1933; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1934; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
1935; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1936; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1937; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1938; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
1939; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
1940; AVX2-NEXT:    retq
1941;
1942; AVX512F-LABEL: smulo_v32i8:
1943; AVX512F:       # %bb.0:
1944; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
1945; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
1946; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm4
1947; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1948; AVX512F-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
1949; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1950; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1951; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1952; AVX512F-NEXT:    vpcmpgtb %xmm2, %xmm5, %xmm6
1953; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
1954; AVX512F-NEXT:    vpmovsxbw %xmm4, %ymm4
1955; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
1956; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
1957; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1958; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1959; AVX512F-NEXT:    vpcmpeqb %xmm6, %xmm3, %xmm3
1960; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
1961; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
1962; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k1
1963; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1964; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1965; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
1966; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1967; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1968; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm5, %xmm4
1969; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1970; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1971; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1972; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
1973; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1974; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1975; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
1976; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1977; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1978; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
1979; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
1980; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1981; AVX512F-NEXT:    vmovdqa %xmm2, 16(%rdi)
1982; AVX512F-NEXT:    vmovdqa %xmm3, (%rdi)
1983; AVX512F-NEXT:    retq
1984;
1985; AVX512BW-LABEL: smulo_v32i8:
1986; AVX512BW:       # %bb.0:
1987; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
1988; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1989; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
1990; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
1991; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1992; AVX512BW-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm3
1993; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
1994; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
1995; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1996; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
1997; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
1998; AVX512BW-NEXT:    vpcmpneqb %ymm3, %ymm0, %k1
1999; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2000; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2001; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
2002; AVX512BW-NEXT:    vmovdqa %ymm2, (%rdi)
2003; AVX512BW-NEXT:    retq
2004  %t = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)
2005  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 0
2006  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 1
2007  %res = sext <32 x i1> %obit to <32 x i32>
2008  store <32 x i8> %val, <32 x i8>* %p2
2009  ret <32 x i32> %res
2010}
2011
2012define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nounwind {
2013; SSE2-LABEL: smulo_v64i8:
2014; SSE2:       # %bb.0:
2015; SSE2-NEXT:    movq %rdi, %rax
2016; SSE2-NEXT:    movdqa %xmm7, %xmm9
2017; SSE2-NEXT:    movdqa %xmm7, %xmm10
2018; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2019; SSE2-NEXT:    movdqa %xmm3, %xmm8
2020; SSE2-NEXT:    movdqa %xmm3, %xmm11
2021; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2022; SSE2-NEXT:    pmullw %xmm10, %xmm11
2023; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2024; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2025; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2026; SSE2-NEXT:    pmullw %xmm9, %xmm8
2027; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2028; SSE2-NEXT:    pand %xmm9, %xmm11
2029; SSE2-NEXT:    pand %xmm9, %xmm8
2030; SSE2-NEXT:    packuswb %xmm11, %xmm8
2031; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
2032; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2033; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
2034; SSE2-NEXT:    psraw $8, %xmm10
2035; SSE2-NEXT:    psraw $8, %xmm11
2036; SSE2-NEXT:    pmullw %xmm10, %xmm11
2037; SSE2-NEXT:    pxor %xmm3, %xmm3
2038; SSE2-NEXT:    pcmpgtb %xmm8, %xmm3
2039; SSE2-NEXT:    psrlw $8, %xmm11
2040; SSE2-NEXT:    psraw $8, %xmm7
2041; SSE2-NEXT:    psraw $8, %xmm12
2042; SSE2-NEXT:    pmullw %xmm7, %xmm12
2043; SSE2-NEXT:    psrlw $8, %xmm12
2044; SSE2-NEXT:    packuswb %xmm11, %xmm12
2045; SSE2-NEXT:    pcmpeqb %xmm3, %xmm12
2046; SSE2-NEXT:    movdqa %xmm6, %xmm10
2047; SSE2-NEXT:    movdqa %xmm6, %xmm7
2048; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2049; SSE2-NEXT:    movdqa %xmm2, %xmm3
2050; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2051; SSE2-NEXT:    pmullw %xmm7, %xmm3
2052; SSE2-NEXT:    movdqa %xmm2, %xmm11
2053; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2054; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2055; SSE2-NEXT:    pmullw %xmm10, %xmm11
2056; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2057; SSE2-NEXT:    pand %xmm9, %xmm3
2058; SSE2-NEXT:    pand %xmm9, %xmm11
2059; SSE2-NEXT:    packuswb %xmm3, %xmm11
2060; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2061; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2062; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2063; SSE2-NEXT:    psraw $8, %xmm7
2064; SSE2-NEXT:    psraw $8, %xmm3
2065; SSE2-NEXT:    pmullw %xmm7, %xmm3
2066; SSE2-NEXT:    pxor %xmm7, %xmm7
2067; SSE2-NEXT:    pcmpgtb %xmm11, %xmm7
2068; SSE2-NEXT:    psrlw $8, %xmm3
2069; SSE2-NEXT:    psraw $8, %xmm6
2070; SSE2-NEXT:    psraw $8, %xmm2
2071; SSE2-NEXT:    pmullw %xmm6, %xmm2
2072; SSE2-NEXT:    psrlw $8, %xmm2
2073; SSE2-NEXT:    packuswb %xmm3, %xmm2
2074; SSE2-NEXT:    pcmpeqb %xmm7, %xmm2
2075; SSE2-NEXT:    movdqa %xmm5, %xmm3
2076; SSE2-NEXT:    movdqa %xmm5, %xmm6
2077; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2078; SSE2-NEXT:    movdqa %xmm1, %xmm7
2079; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2080; SSE2-NEXT:    pmullw %xmm6, %xmm7
2081; SSE2-NEXT:    movdqa %xmm1, %xmm6
2082; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2083; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2084; SSE2-NEXT:    pmullw %xmm3, %xmm6
2085; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2086; SSE2-NEXT:    pand %xmm9, %xmm7
2087; SSE2-NEXT:    pand %xmm9, %xmm6
2088; SSE2-NEXT:    packuswb %xmm7, %xmm6
2089; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2090; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2091; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2092; SSE2-NEXT:    psraw $8, %xmm3
2093; SSE2-NEXT:    psraw $8, %xmm7
2094; SSE2-NEXT:    pmullw %xmm3, %xmm7
2095; SSE2-NEXT:    pxor %xmm3, %xmm3
2096; SSE2-NEXT:    pcmpgtb %xmm6, %xmm3
2097; SSE2-NEXT:    psrlw $8, %xmm7
2098; SSE2-NEXT:    psraw $8, %xmm5
2099; SSE2-NEXT:    psraw $8, %xmm1
2100; SSE2-NEXT:    pmullw %xmm5, %xmm1
2101; SSE2-NEXT:    psrlw $8, %xmm1
2102; SSE2-NEXT:    packuswb %xmm7, %xmm1
2103; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
2104; SSE2-NEXT:    movdqa %xmm4, %xmm3
2105; SSE2-NEXT:    movdqa %xmm4, %xmm5
2106; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2107; SSE2-NEXT:    movdqa %xmm0, %xmm7
2108; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2109; SSE2-NEXT:    pmullw %xmm5, %xmm7
2110; SSE2-NEXT:    movdqa %xmm0, %xmm5
2111; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2112; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2113; SSE2-NEXT:    pmullw %xmm3, %xmm5
2114; SSE2-NEXT:    pand %xmm9, %xmm7
2115; SSE2-NEXT:    pand %xmm9, %xmm5
2116; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2117; SSE2-NEXT:    packuswb %xmm7, %xmm5
2118; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2119; SSE2-NEXT:    psraw $8, %xmm3
2120; SSE2-NEXT:    psraw $8, %xmm7
2121; SSE2-NEXT:    pmullw %xmm3, %xmm7
2122; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2123; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2124; SSE2-NEXT:    psraw $8, %xmm3
2125; SSE2-NEXT:    psraw $8, %xmm0
2126; SSE2-NEXT:    pmullw %xmm3, %xmm0
2127; SSE2-NEXT:    psrlw $8, %xmm7
2128; SSE2-NEXT:    psrlw $8, %xmm0
2129; SSE2-NEXT:    packuswb %xmm7, %xmm0
2130; SSE2-NEXT:    pxor %xmm3, %xmm3
2131; SSE2-NEXT:    pcmpgtb %xmm5, %xmm3
2132; SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
2133; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
2134; SSE2-NEXT:    pxor %xmm3, %xmm12
2135; SSE2-NEXT:    pxor %xmm3, %xmm2
2136; SSE2-NEXT:    pxor %xmm3, %xmm1
2137; SSE2-NEXT:    pxor %xmm3, %xmm0
2138; SSE2-NEXT:    movdqa %xmm8, 48(%rsi)
2139; SSE2-NEXT:    movdqa %xmm11, 32(%rsi)
2140; SSE2-NEXT:    movdqa %xmm6, 16(%rsi)
2141; SSE2-NEXT:    movdqa %xmm12, %xmm3
2142; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2143; SSE2-NEXT:    movdqa %xmm5, (%rsi)
2144; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
2145; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2146; SSE2-NEXT:    psrad $24, %xmm4
2147; SSE2-NEXT:    movdqa %xmm4, 192(%rdi)
2148; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
2149; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2150; SSE2-NEXT:    psrad $24, %xmm4
2151; SSE2-NEXT:    movdqa %xmm4, 128(%rdi)
2152; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
2153; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2154; SSE2-NEXT:    psrad $24, %xmm4
2155; SSE2-NEXT:    movdqa %xmm4, 64(%rdi)
2156; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2157; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2158; SSE2-NEXT:    psrad $24, %xmm4
2159; SSE2-NEXT:    movdqa %xmm4, (%rdi)
2160; SSE2-NEXT:    movdqa %xmm3, %xmm4
2161; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2162; SSE2-NEXT:    pslld $31, %xmm3
2163; SSE2-NEXT:    psrad $31, %xmm3
2164; SSE2-NEXT:    movdqa %xmm3, 224(%rdi)
2165; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2166; SSE2-NEXT:    pslld $31, %xmm4
2167; SSE2-NEXT:    psrad $31, %xmm4
2168; SSE2-NEXT:    movdqa %xmm4, 240(%rdi)
2169; SSE2-NEXT:    movdqa %xmm2, %xmm3
2170; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2171; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2172; SSE2-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4,4,5,5,6,6,7,7]
2173; SSE2-NEXT:    pslld $31, %xmm12
2174; SSE2-NEXT:    psrad $31, %xmm12
2175; SSE2-NEXT:    movdqa %xmm12, 208(%rdi)
2176; SSE2-NEXT:    movdqa %xmm3, %xmm4
2177; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2178; SSE2-NEXT:    pslld $31, %xmm3
2179; SSE2-NEXT:    psrad $31, %xmm3
2180; SSE2-NEXT:    movdqa %xmm3, 160(%rdi)
2181; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2182; SSE2-NEXT:    pslld $31, %xmm4
2183; SSE2-NEXT:    psrad $31, %xmm4
2184; SSE2-NEXT:    movdqa %xmm4, 176(%rdi)
2185; SSE2-NEXT:    movdqa %xmm1, %xmm3
2186; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2187; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2188; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2189; SSE2-NEXT:    pslld $31, %xmm2
2190; SSE2-NEXT:    psrad $31, %xmm2
2191; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
2192; SSE2-NEXT:    movdqa %xmm3, %xmm2
2193; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2194; SSE2-NEXT:    pslld $31, %xmm3
2195; SSE2-NEXT:    psrad $31, %xmm3
2196; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
2197; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2198; SSE2-NEXT:    pslld $31, %xmm2
2199; SSE2-NEXT:    psrad $31, %xmm2
2200; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
2201; SSE2-NEXT:    movdqa %xmm0, %xmm2
2202; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2203; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2204; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2205; SSE2-NEXT:    pslld $31, %xmm1
2206; SSE2-NEXT:    psrad $31, %xmm1
2207; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
2208; SSE2-NEXT:    movdqa %xmm2, %xmm1
2209; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2210; SSE2-NEXT:    pslld $31, %xmm2
2211; SSE2-NEXT:    psrad $31, %xmm2
2212; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
2213; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2214; SSE2-NEXT:    pslld $31, %xmm1
2215; SSE2-NEXT:    psrad $31, %xmm1
2216; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
2217; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2218; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2219; SSE2-NEXT:    pslld $31, %xmm0
2220; SSE2-NEXT:    psrad $31, %xmm0
2221; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
2222; SSE2-NEXT:    retq
2223;
2224; SSSE3-LABEL: smulo_v64i8:
2225; SSSE3:       # %bb.0:
2226; SSSE3-NEXT:    movq %rdi, %rax
2227; SSSE3-NEXT:    movdqa %xmm7, %xmm9
2228; SSSE3-NEXT:    movdqa %xmm7, %xmm10
2229; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2230; SSSE3-NEXT:    movdqa %xmm3, %xmm8
2231; SSSE3-NEXT:    movdqa %xmm3, %xmm11
2232; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2233; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2234; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2235; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2236; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2237; SSSE3-NEXT:    pmullw %xmm9, %xmm8
2238; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2239; SSSE3-NEXT:    pand %xmm9, %xmm11
2240; SSSE3-NEXT:    pand %xmm9, %xmm8
2241; SSSE3-NEXT:    packuswb %xmm11, %xmm8
2242; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
2243; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2244; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
2245; SSSE3-NEXT:    psraw $8, %xmm10
2246; SSSE3-NEXT:    psraw $8, %xmm11
2247; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2248; SSSE3-NEXT:    pxor %xmm3, %xmm3
2249; SSSE3-NEXT:    pcmpgtb %xmm8, %xmm3
2250; SSSE3-NEXT:    psrlw $8, %xmm11
2251; SSSE3-NEXT:    psraw $8, %xmm7
2252; SSSE3-NEXT:    psraw $8, %xmm12
2253; SSSE3-NEXT:    pmullw %xmm7, %xmm12
2254; SSSE3-NEXT:    psrlw $8, %xmm12
2255; SSSE3-NEXT:    packuswb %xmm11, %xmm12
2256; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm12
2257; SSSE3-NEXT:    movdqa %xmm6, %xmm10
2258; SSSE3-NEXT:    movdqa %xmm6, %xmm7
2259; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2260; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2261; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2262; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2263; SSSE3-NEXT:    movdqa %xmm2, %xmm11
2264; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2265; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2266; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2267; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2268; SSSE3-NEXT:    pand %xmm9, %xmm3
2269; SSSE3-NEXT:    pand %xmm9, %xmm11
2270; SSSE3-NEXT:    packuswb %xmm3, %xmm11
2271; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2272; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2273; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2274; SSSE3-NEXT:    psraw $8, %xmm7
2275; SSSE3-NEXT:    psraw $8, %xmm3
2276; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2277; SSSE3-NEXT:    pxor %xmm7, %xmm7
2278; SSSE3-NEXT:    pcmpgtb %xmm11, %xmm7
2279; SSSE3-NEXT:    psrlw $8, %xmm3
2280; SSSE3-NEXT:    psraw $8, %xmm6
2281; SSSE3-NEXT:    psraw $8, %xmm2
2282; SSSE3-NEXT:    pmullw %xmm6, %xmm2
2283; SSSE3-NEXT:    psrlw $8, %xmm2
2284; SSSE3-NEXT:    packuswb %xmm3, %xmm2
2285; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm2
2286; SSSE3-NEXT:    movdqa %xmm5, %xmm3
2287; SSSE3-NEXT:    movdqa %xmm5, %xmm6
2288; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2289; SSSE3-NEXT:    movdqa %xmm1, %xmm7
2290; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2291; SSSE3-NEXT:    pmullw %xmm6, %xmm7
2292; SSSE3-NEXT:    movdqa %xmm1, %xmm6
2293; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2294; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2295; SSSE3-NEXT:    pmullw %xmm3, %xmm6
2296; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2297; SSSE3-NEXT:    pand %xmm9, %xmm7
2298; SSSE3-NEXT:    pand %xmm9, %xmm6
2299; SSSE3-NEXT:    packuswb %xmm7, %xmm6
2300; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2301; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2302; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2303; SSSE3-NEXT:    psraw $8, %xmm3
2304; SSSE3-NEXT:    psraw $8, %xmm7
2305; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2306; SSSE3-NEXT:    pxor %xmm3, %xmm3
2307; SSSE3-NEXT:    pcmpgtb %xmm6, %xmm3
2308; SSSE3-NEXT:    psrlw $8, %xmm7
2309; SSSE3-NEXT:    psraw $8, %xmm5
2310; SSSE3-NEXT:    psraw $8, %xmm1
2311; SSSE3-NEXT:    pmullw %xmm5, %xmm1
2312; SSSE3-NEXT:    psrlw $8, %xmm1
2313; SSSE3-NEXT:    packuswb %xmm7, %xmm1
2314; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm1
2315; SSSE3-NEXT:    movdqa %xmm4, %xmm3
2316; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2317; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2318; SSSE3-NEXT:    movdqa %xmm0, %xmm7
2319; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2320; SSSE3-NEXT:    pmullw %xmm5, %xmm7
2321; SSSE3-NEXT:    movdqa %xmm0, %xmm5
2322; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2323; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2324; SSSE3-NEXT:    pmullw %xmm3, %xmm5
2325; SSSE3-NEXT:    pand %xmm9, %xmm7
2326; SSSE3-NEXT:    pand %xmm9, %xmm5
2327; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2328; SSSE3-NEXT:    packuswb %xmm7, %xmm5
2329; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2330; SSSE3-NEXT:    psraw $8, %xmm3
2331; SSSE3-NEXT:    psraw $8, %xmm7
2332; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2333; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2334; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2335; SSSE3-NEXT:    psraw $8, %xmm3
2336; SSSE3-NEXT:    psraw $8, %xmm0
2337; SSSE3-NEXT:    pmullw %xmm3, %xmm0
2338; SSSE3-NEXT:    psrlw $8, %xmm7
2339; SSSE3-NEXT:    psrlw $8, %xmm0
2340; SSSE3-NEXT:    packuswb %xmm7, %xmm0
2341; SSSE3-NEXT:    pxor %xmm3, %xmm3
2342; SSSE3-NEXT:    pcmpgtb %xmm5, %xmm3
2343; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm0
2344; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
2345; SSSE3-NEXT:    pxor %xmm3, %xmm12
2346; SSSE3-NEXT:    pxor %xmm3, %xmm2
2347; SSSE3-NEXT:    pxor %xmm3, %xmm1
2348; SSSE3-NEXT:    pxor %xmm3, %xmm0
2349; SSSE3-NEXT:    movdqa %xmm8, 48(%rsi)
2350; SSSE3-NEXT:    movdqa %xmm11, 32(%rsi)
2351; SSSE3-NEXT:    movdqa %xmm6, 16(%rsi)
2352; SSSE3-NEXT:    movdqa %xmm12, %xmm3
2353; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2354; SSSE3-NEXT:    movdqa %xmm5, (%rsi)
2355; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
2356; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2357; SSSE3-NEXT:    psrad $24, %xmm4
2358; SSSE3-NEXT:    movdqa %xmm4, 192(%rdi)
2359; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
2360; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2361; SSSE3-NEXT:    psrad $24, %xmm4
2362; SSSE3-NEXT:    movdqa %xmm4, 128(%rdi)
2363; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
2364; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2365; SSSE3-NEXT:    psrad $24, %xmm4
2366; SSSE3-NEXT:    movdqa %xmm4, 64(%rdi)
2367; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2368; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2369; SSSE3-NEXT:    psrad $24, %xmm4
2370; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
2371; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2372; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2373; SSSE3-NEXT:    pslld $31, %xmm3
2374; SSSE3-NEXT:    psrad $31, %xmm3
2375; SSSE3-NEXT:    movdqa %xmm3, 224(%rdi)
2376; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2377; SSSE3-NEXT:    pslld $31, %xmm4
2378; SSSE3-NEXT:    psrad $31, %xmm4
2379; SSSE3-NEXT:    movdqa %xmm4, 240(%rdi)
2380; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2381; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2382; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2383; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4,4,5,5,6,6,7,7]
2384; SSSE3-NEXT:    pslld $31, %xmm12
2385; SSSE3-NEXT:    psrad $31, %xmm12
2386; SSSE3-NEXT:    movdqa %xmm12, 208(%rdi)
2387; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2388; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2389; SSSE3-NEXT:    pslld $31, %xmm3
2390; SSSE3-NEXT:    psrad $31, %xmm3
2391; SSSE3-NEXT:    movdqa %xmm3, 160(%rdi)
2392; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2393; SSSE3-NEXT:    pslld $31, %xmm4
2394; SSSE3-NEXT:    psrad $31, %xmm4
2395; SSSE3-NEXT:    movdqa %xmm4, 176(%rdi)
2396; SSSE3-NEXT:    movdqa %xmm1, %xmm3
2397; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2398; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2399; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2400; SSSE3-NEXT:    pslld $31, %xmm2
2401; SSSE3-NEXT:    psrad $31, %xmm2
2402; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
2403; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2404; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2405; SSSE3-NEXT:    pslld $31, %xmm3
2406; SSSE3-NEXT:    psrad $31, %xmm3
2407; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
2408; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2409; SSSE3-NEXT:    pslld $31, %xmm2
2410; SSSE3-NEXT:    psrad $31, %xmm2
2411; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
2412; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2413; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2414; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2415; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2416; SSSE3-NEXT:    pslld $31, %xmm1
2417; SSSE3-NEXT:    psrad $31, %xmm1
2418; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
2419; SSSE3-NEXT:    movdqa %xmm2, %xmm1
2420; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2421; SSSE3-NEXT:    pslld $31, %xmm2
2422; SSSE3-NEXT:    psrad $31, %xmm2
2423; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
2424; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2425; SSSE3-NEXT:    pslld $31, %xmm1
2426; SSSE3-NEXT:    psrad $31, %xmm1
2427; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
2428; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2429; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2430; SSSE3-NEXT:    pslld $31, %xmm0
2431; SSSE3-NEXT:    psrad $31, %xmm0
2432; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
2433; SSSE3-NEXT:    retq
2434;
2435; SSE41-LABEL: smulo_v64i8:
2436; SSE41:       # %bb.0:
2437; SSE41-NEXT:    movq %rdi, %rax
2438; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2439; SSE41-NEXT:    movdqa %xmm7, %xmm10
2440; SSE41-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2441; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2442; SSE41-NEXT:    movdqa %xmm3, %xmm11
2443; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2444; SSE41-NEXT:    pmullw %xmm10, %xmm11
2445; SSE41-NEXT:    pmovsxbw %xmm7, %xmm12
2446; SSE41-NEXT:    pmullw %xmm9, %xmm8
2447; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2448; SSE41-NEXT:    pand %xmm10, %xmm11
2449; SSE41-NEXT:    pand %xmm10, %xmm8
2450; SSE41-NEXT:    packuswb %xmm11, %xmm8
2451; SSE41-NEXT:    pmovsxbw %xmm3, %xmm9
2452; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,2,3]
2453; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,2,3]
2454; SSE41-NEXT:    pmullw %xmm12, %xmm9
2455; SSE41-NEXT:    pxor %xmm7, %xmm7
2456; SSE41-NEXT:    pcmpgtb %xmm8, %xmm7
2457; SSE41-NEXT:    psrlw $8, %xmm9
2458; SSE41-NEXT:    pmovsxbw %xmm11, %xmm11
2459; SSE41-NEXT:    pmovsxbw %xmm13, %xmm3
2460; SSE41-NEXT:    pmullw %xmm11, %xmm3
2461; SSE41-NEXT:    psrlw $8, %xmm3
2462; SSE41-NEXT:    packuswb %xmm3, %xmm9
2463; SSE41-NEXT:    pcmpeqb %xmm7, %xmm9
2464; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2465; SSE41-NEXT:    movdqa %xmm6, %xmm7
2466; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2467; SSE41-NEXT:    movdqa %xmm2, %xmm3
2468; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2469; SSE41-NEXT:    pmullw %xmm7, %xmm3
2470; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2471; SSE41-NEXT:    pmullw %xmm11, %xmm12
2472; SSE41-NEXT:    pmovsxbw %xmm6, %xmm7
2473; SSE41-NEXT:    pand %xmm10, %xmm3
2474; SSE41-NEXT:    pand %xmm10, %xmm12
2475; SSE41-NEXT:    packuswb %xmm3, %xmm12
2476; SSE41-NEXT:    pmovsxbw %xmm2, %xmm3
2477; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2478; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2479; SSE41-NEXT:    pmullw %xmm7, %xmm3
2480; SSE41-NEXT:    pxor %xmm7, %xmm7
2481; SSE41-NEXT:    pcmpgtb %xmm12, %xmm7
2482; SSE41-NEXT:    psrlw $8, %xmm3
2483; SSE41-NEXT:    pmovsxbw %xmm6, %xmm6
2484; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
2485; SSE41-NEXT:    pmullw %xmm6, %xmm2
2486; SSE41-NEXT:    psrlw $8, %xmm2
2487; SSE41-NEXT:    packuswb %xmm2, %xmm3
2488; SSE41-NEXT:    pcmpeqb %xmm7, %xmm3
2489; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2490; SSE41-NEXT:    movdqa %xmm5, %xmm6
2491; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2492; SSE41-NEXT:    movdqa %xmm1, %xmm7
2493; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2494; SSE41-NEXT:    pmullw %xmm6, %xmm7
2495; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2496; SSE41-NEXT:    pmullw %xmm2, %xmm11
2497; SSE41-NEXT:    pmovsxbw %xmm5, %xmm6
2498; SSE41-NEXT:    pand %xmm10, %xmm7
2499; SSE41-NEXT:    pand %xmm10, %xmm11
2500; SSE41-NEXT:    packuswb %xmm7, %xmm11
2501; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
2502; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2503; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2504; SSE41-NEXT:    pmullw %xmm6, %xmm2
2505; SSE41-NEXT:    pxor %xmm6, %xmm6
2506; SSE41-NEXT:    pcmpgtb %xmm11, %xmm6
2507; SSE41-NEXT:    psrlw $8, %xmm2
2508; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
2509; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
2510; SSE41-NEXT:    pmullw %xmm5, %xmm1
2511; SSE41-NEXT:    psrlw $8, %xmm1
2512; SSE41-NEXT:    packuswb %xmm1, %xmm2
2513; SSE41-NEXT:    pcmpeqb %xmm6, %xmm2
2514; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2515; SSE41-NEXT:    movdqa %xmm4, %xmm5
2516; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2517; SSE41-NEXT:    movdqa %xmm0, %xmm6
2518; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2519; SSE41-NEXT:    pmullw %xmm5, %xmm6
2520; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2521; SSE41-NEXT:    pmullw %xmm1, %xmm5
2522; SSE41-NEXT:    pand %xmm10, %xmm6
2523; SSE41-NEXT:    pand %xmm10, %xmm5
2524; SSE41-NEXT:    pmovsxbw %xmm4, %xmm7
2525; SSE41-NEXT:    packuswb %xmm6, %xmm5
2526; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
2527; SSE41-NEXT:    pmullw %xmm7, %xmm1
2528; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2529; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2530; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
2531; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
2532; SSE41-NEXT:    pmullw %xmm4, %xmm0
2533; SSE41-NEXT:    psrlw $8, %xmm1
2534; SSE41-NEXT:    psrlw $8, %xmm0
2535; SSE41-NEXT:    packuswb %xmm0, %xmm1
2536; SSE41-NEXT:    pxor %xmm0, %xmm0
2537; SSE41-NEXT:    pcmpgtb %xmm5, %xmm0
2538; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
2539; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2540; SSE41-NEXT:    pxor %xmm0, %xmm9
2541; SSE41-NEXT:    pxor %xmm0, %xmm3
2542; SSE41-NEXT:    pxor %xmm0, %xmm2
2543; SSE41-NEXT:    pxor %xmm0, %xmm1
2544; SSE41-NEXT:    movdqa %xmm8, 48(%rsi)
2545; SSE41-NEXT:    movdqa %xmm12, 32(%rsi)
2546; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)
2547; SSE41-NEXT:    movdqa %xmm5, (%rsi)
2548; SSE41-NEXT:    pmovsxbd %xmm9, %xmm0
2549; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
2550; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
2551; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
2552; SSE41-NEXT:    pmovsxbd %xmm2, %xmm0
2553; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
2554; SSE41-NEXT:    pmovsxbd %xmm1, %xmm0
2555; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2556; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
2557; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2558; SSE41-NEXT:    pslld $31, %xmm0
2559; SSE41-NEXT:    psrad $31, %xmm0
2560; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
2561; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[3,3,3,3]
2562; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2563; SSE41-NEXT:    pslld $31, %xmm0
2564; SSE41-NEXT:    psrad $31, %xmm0
2565; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
2566; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,1,1]
2567; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2568; SSE41-NEXT:    pslld $31, %xmm0
2569; SSE41-NEXT:    psrad $31, %xmm0
2570; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
2571; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
2572; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2573; SSE41-NEXT:    pslld $31, %xmm0
2574; SSE41-NEXT:    psrad $31, %xmm0
2575; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
2576; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
2577; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2578; SSE41-NEXT:    pslld $31, %xmm0
2579; SSE41-NEXT:    psrad $31, %xmm0
2580; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
2581; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
2582; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2583; SSE41-NEXT:    pslld $31, %xmm0
2584; SSE41-NEXT:    psrad $31, %xmm0
2585; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
2586; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
2587; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2588; SSE41-NEXT:    pslld $31, %xmm0
2589; SSE41-NEXT:    psrad $31, %xmm0
2590; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
2591; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
2592; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2593; SSE41-NEXT:    pslld $31, %xmm0
2594; SSE41-NEXT:    psrad $31, %xmm0
2595; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
2596; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
2597; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2598; SSE41-NEXT:    pslld $31, %xmm0
2599; SSE41-NEXT:    psrad $31, %xmm0
2600; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
2601; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
2602; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2603; SSE41-NEXT:    pslld $31, %xmm0
2604; SSE41-NEXT:    psrad $31, %xmm0
2605; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
2606; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
2607; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2608; SSE41-NEXT:    pslld $31, %xmm0
2609; SSE41-NEXT:    psrad $31, %xmm0
2610; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
2611; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
2612; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2613; SSE41-NEXT:    pslld $31, %xmm0
2614; SSE41-NEXT:    psrad $31, %xmm0
2615; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
2616; SSE41-NEXT:    retq
2617;
2618; AVX1-LABEL: smulo_v64i8:
2619; AVX1:       # %bb.0:
2620; AVX1-NEXT:    movq %rdi, %rax
2621; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
2622; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2623; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
2624; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2625; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2626; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2627; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm8
2628; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2629; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2630; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
2631; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2632; AVX1-NEXT:    vpackuswb %xmm8, %xmm4, %xmm9
2633; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
2634; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm11, %xmm8
2635; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
2636; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2637; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2638; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2639; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2640; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
2641; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2642; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2643; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2644; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2645; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
2646; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm8, %xmm8
2647; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2648; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2649; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2650; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2651; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2652; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2653; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2654; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2655; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm12
2656; AVX1-NEXT:    vpcmpgtb %xmm12, %xmm11, %xmm4
2657; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm6
2658; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm7
2659; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2660; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
2661; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
2662; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
2663; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2664; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
2665; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2666; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2667; AVX1-NEXT:    vpackuswb %xmm1, %xmm6, %xmm1
2668; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm3
2669; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
2670; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2671; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
2672; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2673; AVX1-NEXT:    vpmullw %xmm1, %xmm7, %xmm1
2674; AVX1-NEXT:    vpand %xmm1, %xmm10, %xmm1
2675; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2676; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2677; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
2678; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2679; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
2680; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm5
2681; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2682; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
2683; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm11, %xmm7
2684; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2685; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2686; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
2687; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2688; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2689; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
2690; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2691; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
2692; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm4
2693; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2694; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2695; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2696; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2697; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2698; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2699; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2700; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
2701; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm7
2702; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm5
2703; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm6
2704; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2705; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2706; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
2707; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2708; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
2709; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2710; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm2
2711; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
2712; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
2713; AVX1-NEXT:    vpcmpgtb %xmm7, %xmm11, %xmm2
2714; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2715; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
2716; AVX1-NEXT:    vpxor %xmm5, %xmm8, %xmm6
2717; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
2718; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
2719; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
2720; AVX1-NEXT:    vmovdqa %xmm9, 48(%rsi)
2721; AVX1-NEXT:    vmovdqa %xmm12, 32(%rsi)
2722; AVX1-NEXT:    vmovdqa %xmm1, 16(%rsi)
2723; AVX1-NEXT:    vmovdqa %xmm7, (%rsi)
2724; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
2725; AVX1-NEXT:    vmovdqa %xmm1, 192(%rdi)
2726; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm1
2727; AVX1-NEXT:    vmovdqa %xmm1, 128(%rdi)
2728; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm1
2729; AVX1-NEXT:    vmovdqa %xmm1, 64(%rdi)
2730; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
2731; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
2732; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
2733; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2734; AVX1-NEXT:    vmovdqa %xmm1, 224(%rdi)
2735; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
2736; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2737; AVX1-NEXT:    vmovdqa %xmm1, 240(%rdi)
2738; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
2739; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2740; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
2741; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
2742; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2743; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
2744; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
2745; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2746; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
2747; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
2748; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2749; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
2750; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
2751; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2752; AVX1-NEXT:    vmovdqa %xmm1, 96(%rdi)
2753; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
2754; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2755; AVX1-NEXT:    vmovdqa %xmm1, 112(%rdi)
2756; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
2757; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2758; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
2759; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
2760; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2761; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
2762; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2763; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2764; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
2765; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
2766; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
2767; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
2768; AVX1-NEXT:    vzeroupper
2769; AVX1-NEXT:    retq
2770;
2771; AVX2-LABEL: smulo_v64i8:
2772; AVX2:       # %bb.0:
2773; AVX2-NEXT:    movq %rdi, %rax
2774; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2775; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2776; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2777; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2778; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
2779; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2780; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2781; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2782; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2783; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm10
2784; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
2785; AVX2-NEXT:    vpcmpgtb %ymm10, %ymm6, %ymm7
2786; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2787; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2788; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2789; AVX2-NEXT:    vpsraw $8, %ymm9, %ymm9
2790; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2791; AVX2-NEXT:    vpsrlw $8, %ymm8, %ymm8
2792; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2793; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
2794; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2795; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
2796; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2797; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
2798; AVX2-NEXT:    vpackuswb %ymm8, %ymm1, %ymm1
2799; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm1
2800; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm7, %ymm7
2801; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm1
2802; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2803; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2804; AVX2-NEXT:    vpmullw %ymm3, %ymm8, %ymm3
2805; AVX2-NEXT:    vpand %ymm5, %ymm3, %ymm3
2806; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2807; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2808; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2809; AVX2-NEXT:    vpand %ymm5, %ymm8, %ymm5
2810; AVX2-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
2811; AVX2-NEXT:    vpcmpgtb %ymm3, %ymm6, %ymm5
2812; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2813; AVX2-NEXT:    vpsraw $8, %ymm6, %ymm6
2814; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2815; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2816; AVX2-NEXT:    vpmullw %ymm6, %ymm8, %ymm6
2817; AVX2-NEXT:    vpsrlw $8, %ymm6, %ymm6
2818; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2819; AVX2-NEXT:    vpsraw $8, %ymm2, %ymm2
2820; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2821; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
2822; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
2823; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
2824; AVX2-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
2825; AVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
2826; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm0
2827; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2828; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm8
2829; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
2830; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
2831; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
2832; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
2833; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
2834; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
2835; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
2836; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
2837; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
2838; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
2839; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
2840; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
2841; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
2842; AVX2-NEXT:    vmovdqa %ymm3, (%rsi)
2843; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
2844; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
2845; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)
2846; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
2847; AVX2-NEXT:    vmovdqa %ymm2, 224(%rdi)
2848; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
2849; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
2850; AVX2-NEXT:    vmovdqa %ymm8, 32(%rdi)
2851; AVX2-NEXT:    vzeroupper
2852; AVX2-NEXT:    retq
2853;
2854; AVX512F-LABEL: smulo_v64i8:
2855; AVX512F:       # %bb.0:
2856; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
2857; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm5
2858; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
2859; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
2860; AVX512F-NEXT:    vextracti128 $1, %ymm6, %xmm7
2861; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
2862; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
2863; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2864; AVX512F-NEXT:    vpmovdb %zmm2, %xmm4
2865; AVX512F-NEXT:    vpxor %xmm8, %xmm8, %xmm8
2866; AVX512F-NEXT:    vpcmpgtb %xmm4, %xmm8, %xmm2
2867; AVX512F-NEXT:    vpmovsxbw %xmm5, %ymm5
2868; AVX512F-NEXT:    vpmovsxbw %xmm7, %ymm7
2869; AVX512F-NEXT:    vpmullw %ymm5, %ymm7, %ymm5
2870; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
2871; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
2872; AVX512F-NEXT:    vpmovdb %zmm5, %xmm5
2873; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm5, %xmm2
2874; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2875; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2876; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k1
2877; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2878; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
2879; AVX512F-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
2880; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2881; AVX512F-NEXT:    vpmovdb %zmm2, %xmm5
2882; AVX512F-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
2883; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
2884; AVX512F-NEXT:    vpmovsxbw %xmm6, %ymm6
2885; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm3
2886; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
2887; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2888; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2889; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm2
2890; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2891; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2892; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k2
2893; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
2894; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2895; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm7
2896; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
2897; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm3
2898; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2899; AVX512F-NEXT:    vpmovdb %zmm3, %xmm6
2900; AVX512F-NEXT:    vpcmpgtb %xmm6, %xmm8, %xmm3
2901; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
2902; AVX512F-NEXT:    vpmovsxbw %xmm7, %ymm7
2903; AVX512F-NEXT:    vpmullw %ymm2, %ymm7, %ymm2
2904; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
2905; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2906; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
2907; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
2908; AVX512F-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
2909; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
2910; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k3
2911; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2912; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2913; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
2914; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2915; AVX512F-NEXT:    vpmovdb %zmm2, %xmm7
2916; AVX512F-NEXT:    vpcmpgtb %xmm7, %xmm8, %xmm2
2917; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
2918; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
2919; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
2920; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
2921; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
2922; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2923; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2924; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2925; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
2926; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k4
2927; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
2928; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
2929; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
2930; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2931; AVX512F-NEXT:    vmovdqa %xmm4, 48(%rdi)
2932; AVX512F-NEXT:    vmovdqa %xmm5, 32(%rdi)
2933; AVX512F-NEXT:    vmovdqa %xmm6, 16(%rdi)
2934; AVX512F-NEXT:    vmovdqa %xmm7, (%rdi)
2935; AVX512F-NEXT:    retq
2936;
2937; AVX512BW-LABEL: smulo_v64i8:
2938; AVX512BW:       # %bb.0:
2939; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2940; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
2941; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2942; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
2943; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2944; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
2945; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2946; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
2947; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2948; AVX512BW-NEXT:    vpsraw $8, %zmm4, %zmm4
2949; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2950; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
2951; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
2952; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2953; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2954; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2955; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2956; AVX512BW-NEXT:    vpandq %zmm4, %zmm3, %zmm3
2957; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2958; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2959; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2960; AVX512BW-NEXT:    vpandq %zmm4, %zmm0, %zmm0
2961; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm4
2962; AVX512BW-NEXT:    vpmovb2m %zmm4, %k0
2963; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
2964; AVX512BW-NEXT:    vpcmpneqb %zmm0, %zmm2, %k1
2965; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2966; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
2967; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2968; AVX512BW-NEXT:    kshiftrq $32, %k1, %k1
2969; AVX512BW-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
2970; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2971; AVX512BW-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2972; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdi)
2973; AVX512BW-NEXT:    retq
2974  %t = call {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)
2975  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 0
2976  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 1
2977  %res = sext <64 x i1> %obit to <64 x i32>
2978  store <64 x i8> %val, <64 x i8>* %p2
2979  ret <64 x i32> %res
2980}
2981
2982define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
2983; SSE2-LABEL: smulo_v8i16:
2984; SSE2:       # %bb.0:
2985; SSE2-NEXT:    movdqa %xmm0, %xmm2
2986; SSE2-NEXT:    pmulhw %xmm1, %xmm2
2987; SSE2-NEXT:    pmullw %xmm1, %xmm0
2988; SSE2-NEXT:    movdqa %xmm0, (%rdi)
2989; SSE2-NEXT:    psraw $15, %xmm0
2990; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
2991; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
2992; SSE2-NEXT:    pxor %xmm0, %xmm1
2993; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2994; SSE2-NEXT:    psrad $16, %xmm0
2995; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2996; SSE2-NEXT:    pslld $31, %xmm1
2997; SSE2-NEXT:    psrad $31, %xmm1
2998; SSE2-NEXT:    retq
2999;
3000; SSSE3-LABEL: smulo_v8i16:
3001; SSSE3:       # %bb.0:
3002; SSSE3-NEXT:    movdqa %xmm0, %xmm2
3003; SSSE3-NEXT:    pmulhw %xmm1, %xmm2
3004; SSSE3-NEXT:    pmullw %xmm1, %xmm0
3005; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
3006; SSSE3-NEXT:    psraw $15, %xmm0
3007; SSSE3-NEXT:    pcmpeqw %xmm2, %xmm0
3008; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
3009; SSSE3-NEXT:    pxor %xmm0, %xmm1
3010; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3011; SSSE3-NEXT:    psrad $16, %xmm0
3012; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3013; SSSE3-NEXT:    pslld $31, %xmm1
3014; SSSE3-NEXT:    psrad $31, %xmm1
3015; SSSE3-NEXT:    retq
3016;
3017; SSE41-LABEL: smulo_v8i16:
3018; SSE41:       # %bb.0:
3019; SSE41-NEXT:    movdqa %xmm0, %xmm2
3020; SSE41-NEXT:    pmulhw %xmm1, %xmm2
3021; SSE41-NEXT:    pmullw %xmm1, %xmm0
3022; SSE41-NEXT:    movdqa %xmm0, (%rdi)
3023; SSE41-NEXT:    psraw $15, %xmm0
3024; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
3025; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
3026; SSE41-NEXT:    pxor %xmm0, %xmm1
3027; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
3028; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3029; SSE41-NEXT:    pslld $31, %xmm1
3030; SSE41-NEXT:    psrad $31, %xmm1
3031; SSE41-NEXT:    retq
3032;
3033; AVX1-LABEL: smulo_v8i16:
3034; AVX1:       # %bb.0:
3035; AVX1-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3036; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3037; AVX1-NEXT:    vpsraw $15, %xmm1, %xmm0
3038; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3039; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3040; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3041; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
3042; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3043; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
3044; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
3045; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
3046; AVX1-NEXT:    retq
3047;
3048; AVX2-LABEL: smulo_v8i16:
3049; AVX2:       # %bb.0:
3050; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3051; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3052; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm0
3053; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3054; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3055; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3056; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
3057; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
3058; AVX2-NEXT:    retq
3059;
3060; AVX512F-LABEL: smulo_v8i16:
3061; AVX512F:       # %bb.0:
3062; AVX512F-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3063; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3064; AVX512F-NEXT:    vpsraw $15, %xmm1, %xmm0
3065; AVX512F-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
3066; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
3067; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
3068; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
3069; AVX512F-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
3070; AVX512F-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
3071; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
3072; AVX512F-NEXT:    retq
3073;
3074; AVX512BW-LABEL: smulo_v8i16:
3075; AVX512BW:       # %bb.0:
3076; AVX512BW-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
3077; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
3078; AVX512BW-NEXT:    vpsraw $15, %xmm1, %xmm0
3079; AVX512BW-NEXT:    vpcmpneqw %xmm0, %xmm2, %k1
3080; AVX512BW-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
3081; AVX512BW-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
3082; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
3083; AVX512BW-NEXT:    retq
3084  %t = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
3085  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
3086  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
3087  %res = sext <8 x i1> %obit to <8 x i32>
3088  store <8 x i16> %val, <8 x i16>* %p2
3089  ret <8 x i32> %res
3090}
3091
3092define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
3093; SSE2-LABEL: smulo_v2i64:
3094; SSE2:       # %bb.0:
3095; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
3096; SSE2-NEXT:    movq %xmm2, %r8
3097; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
3098; SSE2-NEXT:    movq %xmm2, %rcx
3099; SSE2-NEXT:    movq %xmm1, %rdx
3100; SSE2-NEXT:    movq %xmm0, %rsi
3101; SSE2-NEXT:    xorl %eax, %eax
3102; SSE2-NEXT:    imulq %rdx, %rsi
3103; SSE2-NEXT:    movq $-1, %r9
3104; SSE2-NEXT:    movl $0, %edx
3105; SSE2-NEXT:    cmovoq %r9, %rdx
3106; SSE2-NEXT:    movq %rsi, %xmm1
3107; SSE2-NEXT:    imulq %r8, %rcx
3108; SSE2-NEXT:    movq %rcx, %xmm0
3109; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3110; SSE2-NEXT:    movq %rdx, %xmm0
3111; SSE2-NEXT:    cmovoq %r9, %rax
3112; SSE2-NEXT:    movq %rax, %xmm2
3113; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3114; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3115; SSE2-NEXT:    movdqa %xmm1, (%rdi)
3116; SSE2-NEXT:    retq
3117;
3118; SSSE3-LABEL: smulo_v2i64:
3119; SSSE3:       # %bb.0:
3120; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
3121; SSSE3-NEXT:    movq %xmm2, %r8
3122; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
3123; SSSE3-NEXT:    movq %xmm2, %rcx
3124; SSSE3-NEXT:    movq %xmm1, %rdx
3125; SSSE3-NEXT:    movq %xmm0, %rsi
3126; SSSE3-NEXT:    xorl %eax, %eax
3127; SSSE3-NEXT:    imulq %rdx, %rsi
3128; SSSE3-NEXT:    movq $-1, %r9
3129; SSSE3-NEXT:    movl $0, %edx
3130; SSSE3-NEXT:    cmovoq %r9, %rdx
3131; SSSE3-NEXT:    movq %rsi, %xmm1
3132; SSSE3-NEXT:    imulq %r8, %rcx
3133; SSSE3-NEXT:    movq %rcx, %xmm0
3134; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3135; SSSE3-NEXT:    movq %rdx, %xmm0
3136; SSSE3-NEXT:    cmovoq %r9, %rax
3137; SSSE3-NEXT:    movq %rax, %xmm2
3138; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3139; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3140; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
3141; SSSE3-NEXT:    retq
3142;
3143; SSE41-LABEL: smulo_v2i64:
3144; SSE41:       # %bb.0:
3145; SSE41-NEXT:    movq %xmm1, %r8
3146; SSE41-NEXT:    movq %xmm0, %rcx
3147; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
3148; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
3149; SSE41-NEXT:    xorl %eax, %eax
3150; SSE41-NEXT:    imulq %rdx, %rsi
3151; SSE41-NEXT:    movq $-1, %r9
3152; SSE41-NEXT:    movl $0, %edx
3153; SSE41-NEXT:    cmovoq %r9, %rdx
3154; SSE41-NEXT:    movq %rsi, %xmm0
3155; SSE41-NEXT:    imulq %r8, %rcx
3156; SSE41-NEXT:    movq %rcx, %xmm1
3157; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3158; SSE41-NEXT:    movq %rdx, %xmm0
3159; SSE41-NEXT:    cmovoq %r9, %rax
3160; SSE41-NEXT:    movq %rax, %xmm2
3161; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
3162; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
3163; SSE41-NEXT:    movdqa %xmm1, (%rdi)
3164; SSE41-NEXT:    retq
3165;
3166; AVX-LABEL: smulo_v2i64:
3167; AVX:       # %bb.0:
3168; AVX-NEXT:    vmovq %xmm1, %r8
3169; AVX-NEXT:    vmovq %xmm0, %rcx
3170; AVX-NEXT:    vpextrq $1, %xmm1, %rdx
3171; AVX-NEXT:    vpextrq $1, %xmm0, %rsi
3172; AVX-NEXT:    xorl %eax, %eax
3173; AVX-NEXT:    imulq %rdx, %rsi
3174; AVX-NEXT:    movq $-1, %r9
3175; AVX-NEXT:    movl $0, %edx
3176; AVX-NEXT:    cmovoq %r9, %rdx
3177; AVX-NEXT:    vmovq %rsi, %xmm0
3178; AVX-NEXT:    imulq %r8, %rcx
3179; AVX-NEXT:    vmovq %rcx, %xmm1
3180; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3181; AVX-NEXT:    vmovq %rdx, %xmm0
3182; AVX-NEXT:    cmovoq %r9, %rax
3183; AVX-NEXT:    vmovq %rax, %xmm2
3184; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
3185; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3186; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
3187; AVX-NEXT:    retq
3188;
3189; AVX512F-LABEL: smulo_v2i64:
3190; AVX512F:       # %bb.0:
3191; AVX512F-NEXT:    vmovq %xmm1, %rax
3192; AVX512F-NEXT:    vmovq %xmm0, %rcx
3193; AVX512F-NEXT:    vpextrq $1, %xmm1, %rdx
3194; AVX512F-NEXT:    vpextrq $1, %xmm0, %rsi
3195; AVX512F-NEXT:    imulq %rdx, %rsi
3196; AVX512F-NEXT:    seto %dl
3197; AVX512F-NEXT:    vmovq %rsi, %xmm0
3198; AVX512F-NEXT:    imulq %rax, %rcx
3199; AVX512F-NEXT:    vmovq %rcx, %xmm1
3200; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3201; AVX512F-NEXT:    seto %al
3202; AVX512F-NEXT:    movw $-3, %cx
3203; AVX512F-NEXT:    kmovw %ecx, %k0
3204; AVX512F-NEXT:    kmovw %eax, %k1
3205; AVX512F-NEXT:    kandw %k0, %k1, %k0
3206; AVX512F-NEXT:    kmovw %edx, %k1
3207; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
3208; AVX512F-NEXT:    kshiftrw $14, %k1, %k1
3209; AVX512F-NEXT:    korw %k1, %k0, %k1
3210; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3211; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3212; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
3213; AVX512F-NEXT:    retq
3214;
3215; AVX512BW-LABEL: smulo_v2i64:
3216; AVX512BW:       # %bb.0:
3217; AVX512BW-NEXT:    vmovq %xmm1, %rax
3218; AVX512BW-NEXT:    vmovq %xmm0, %rcx
3219; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rdx
3220; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rsi
3221; AVX512BW-NEXT:    imulq %rdx, %rsi
3222; AVX512BW-NEXT:    seto %dl
3223; AVX512BW-NEXT:    vmovq %rsi, %xmm0
3224; AVX512BW-NEXT:    imulq %rax, %rcx
3225; AVX512BW-NEXT:    vmovq %rcx, %xmm1
3226; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3227; AVX512BW-NEXT:    seto %al
3228; AVX512BW-NEXT:    movw $-3, %cx
3229; AVX512BW-NEXT:    kmovd %ecx, %k0
3230; AVX512BW-NEXT:    kmovd %eax, %k1
3231; AVX512BW-NEXT:    kandw %k0, %k1, %k0
3232; AVX512BW-NEXT:    kmovd %edx, %k1
3233; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
3234; AVX512BW-NEXT:    kshiftrw $14, %k1, %k1
3235; AVX512BW-NEXT:    korw %k1, %k0, %k1
3236; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3237; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3238; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
3239; AVX512BW-NEXT:    retq
3240  %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
3241  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
3242  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
3243  %res = sext <2 x i1> %obit to <2 x i32>
3244  store <2 x i64> %val, <2 x i64>* %p2
3245  ret <2 x i32> %res
3246}
3247
3248define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
3249; SSE2-LABEL: smulo_v4i24:
3250; SSE2:       # %bb.0:
3251; SSE2-NEXT:    pslld $8, %xmm0
3252; SSE2-NEXT:    psrad $8, %xmm0
3253; SSE2-NEXT:    pslld $8, %xmm1
3254; SSE2-NEXT:    psrad $8, %xmm1
3255; SSE2-NEXT:    pxor %xmm3, %xmm3
3256; SSE2-NEXT:    pxor %xmm2, %xmm2
3257; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3258; SSE2-NEXT:    pand %xmm0, %xmm2
3259; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3260; SSE2-NEXT:    pand %xmm1, %xmm3
3261; SSE2-NEXT:    paddd %xmm2, %xmm3
3262; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3263; SSE2-NEXT:    pmuludq %xmm1, %xmm0
3264; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3265; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3266; SSE2-NEXT:    pmuludq %xmm4, %xmm2
3267; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3268; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3269; SSE2-NEXT:    psubd %xmm3, %xmm5
3270; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3271; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3272; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3273; SSE2-NEXT:    movdqa %xmm3, %xmm1
3274; SSE2-NEXT:    pslld $8, %xmm1
3275; SSE2-NEXT:    psrad $8, %xmm1
3276; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
3277; SSE2-NEXT:    psrad $31, %xmm3
3278; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
3279; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
3280; SSE2-NEXT:    pxor %xmm4, %xmm3
3281; SSE2-NEXT:    pxor %xmm4, %xmm1
3282; SSE2-NEXT:    por %xmm3, %xmm1
3283; SSE2-NEXT:    movd %xmm0, %eax
3284; SSE2-NEXT:    movw %ax, (%rdi)
3285; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3286; SSE2-NEXT:    movd %xmm0, %ecx
3287; SSE2-NEXT:    movw %cx, 6(%rdi)
3288; SSE2-NEXT:    movd %xmm2, %edx
3289; SSE2-NEXT:    movw %dx, 3(%rdi)
3290; SSE2-NEXT:    shrl $16, %eax
3291; SSE2-NEXT:    movb %al, 2(%rdi)
3292; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3293; SSE2-NEXT:    movd %xmm0, %eax
3294; SSE2-NEXT:    movw %ax, 9(%rdi)
3295; SSE2-NEXT:    shrl $16, %ecx
3296; SSE2-NEXT:    movb %cl, 8(%rdi)
3297; SSE2-NEXT:    shrl $16, %edx
3298; SSE2-NEXT:    movb %dl, 5(%rdi)
3299; SSE2-NEXT:    shrl $16, %eax
3300; SSE2-NEXT:    movb %al, 11(%rdi)
3301; SSE2-NEXT:    movdqa %xmm1, %xmm0
3302; SSE2-NEXT:    retq
3303;
3304; SSSE3-LABEL: smulo_v4i24:
3305; SSSE3:       # %bb.0:
3306; SSSE3-NEXT:    pslld $8, %xmm0
3307; SSSE3-NEXT:    psrad $8, %xmm0
3308; SSSE3-NEXT:    pslld $8, %xmm1
3309; SSSE3-NEXT:    psrad $8, %xmm1
3310; SSSE3-NEXT:    pxor %xmm3, %xmm3
3311; SSSE3-NEXT:    pxor %xmm2, %xmm2
3312; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
3313; SSSE3-NEXT:    pand %xmm0, %xmm2
3314; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
3315; SSSE3-NEXT:    pand %xmm1, %xmm3
3316; SSSE3-NEXT:    paddd %xmm2, %xmm3
3317; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3318; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
3319; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3320; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3321; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
3322; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3323; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3324; SSSE3-NEXT:    psubd %xmm3, %xmm5
3325; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3326; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3327; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3328; SSSE3-NEXT:    movdqa %xmm3, %xmm1
3329; SSSE3-NEXT:    pslld $8, %xmm1
3330; SSSE3-NEXT:    psrad $8, %xmm1
3331; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm1
3332; SSSE3-NEXT:    psrad $31, %xmm3
3333; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
3334; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
3335; SSSE3-NEXT:    pxor %xmm4, %xmm3
3336; SSSE3-NEXT:    pxor %xmm4, %xmm1
3337; SSSE3-NEXT:    por %xmm3, %xmm1
3338; SSSE3-NEXT:    movd %xmm0, %eax
3339; SSSE3-NEXT:    movw %ax, (%rdi)
3340; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3341; SSSE3-NEXT:    movd %xmm0, %ecx
3342; SSSE3-NEXT:    movw %cx, 6(%rdi)
3343; SSSE3-NEXT:    movd %xmm2, %edx
3344; SSSE3-NEXT:    movw %dx, 3(%rdi)
3345; SSSE3-NEXT:    shrl $16, %eax
3346; SSSE3-NEXT:    movb %al, 2(%rdi)
3347; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3348; SSSE3-NEXT:    movd %xmm0, %eax
3349; SSSE3-NEXT:    movw %ax, 9(%rdi)
3350; SSSE3-NEXT:    shrl $16, %ecx
3351; SSSE3-NEXT:    movb %cl, 8(%rdi)
3352; SSSE3-NEXT:    shrl $16, %edx
3353; SSSE3-NEXT:    movb %dl, 5(%rdi)
3354; SSSE3-NEXT:    shrl $16, %eax
3355; SSSE3-NEXT:    movb %al, 11(%rdi)
3356; SSSE3-NEXT:    movdqa %xmm1, %xmm0
3357; SSSE3-NEXT:    retq
3358;
3359; SSE41-LABEL: smulo_v4i24:
3360; SSE41:       # %bb.0:
3361; SSE41-NEXT:    pslld $8, %xmm0
3362; SSE41-NEXT:    psrad $8, %xmm0
3363; SSE41-NEXT:    pslld $8, %xmm1
3364; SSE41-NEXT:    psrad $8, %xmm1
3365; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3366; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3367; SSE41-NEXT:    pmuldq %xmm2, %xmm3
3368; SSE41-NEXT:    movdqa %xmm0, %xmm2
3369; SSE41-NEXT:    pmuldq %xmm1, %xmm2
3370; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3371; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
3372; SSE41-NEXT:    pmulld %xmm0, %xmm1
3373; SSE41-NEXT:    movdqa %xmm1, %xmm3
3374; SSE41-NEXT:    pslld $8, %xmm3
3375; SSE41-NEXT:    psrad $8, %xmm3
3376; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
3377; SSE41-NEXT:    pextrd $3, %xmm1, %eax
3378; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
3379; SSE41-NEXT:    pextrd $1, %xmm1, %edx
3380; SSE41-NEXT:    movd %xmm1, %esi
3381; SSE41-NEXT:    psrad $31, %xmm1
3382; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
3383; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
3384; SSE41-NEXT:    pxor %xmm0, %xmm1
3385; SSE41-NEXT:    pxor %xmm3, %xmm0
3386; SSE41-NEXT:    por %xmm1, %xmm0
3387; SSE41-NEXT:    movw %ax, 9(%rdi)
3388; SSE41-NEXT:    movw %cx, 6(%rdi)
3389; SSE41-NEXT:    movw %dx, 3(%rdi)
3390; SSE41-NEXT:    movw %si, (%rdi)
3391; SSE41-NEXT:    shrl $16, %eax
3392; SSE41-NEXT:    movb %al, 11(%rdi)
3393; SSE41-NEXT:    shrl $16, %ecx
3394; SSE41-NEXT:    movb %cl, 8(%rdi)
3395; SSE41-NEXT:    shrl $16, %edx
3396; SSE41-NEXT:    movb %dl, 5(%rdi)
3397; SSE41-NEXT:    shrl $16, %esi
3398; SSE41-NEXT:    movb %sil, 2(%rdi)
3399; SSE41-NEXT:    retq
3400;
3401; AVX1-LABEL: smulo_v4i24:
3402; AVX1:       # %bb.0:
3403; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
3404; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
3405; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
3406; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
3407; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3408; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3409; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3410; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3411; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3412; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3413; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3414; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
3415; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3416; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3417; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3418; AVX1-NEXT:    vpslld $8, %xmm1, %xmm3
3419; AVX1-NEXT:    vpsrad $8, %xmm3, %xmm3
3420; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3421; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3422; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
3423; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
3424; AVX1-NEXT:    movw %ax, 9(%rdi)
3425; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
3426; AVX1-NEXT:    movw %cx, 6(%rdi)
3427; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
3428; AVX1-NEXT:    movw %dx, 3(%rdi)
3429; AVX1-NEXT:    vmovd %xmm1, %esi
3430; AVX1-NEXT:    movw %si, (%rdi)
3431; AVX1-NEXT:    shrl $16, %eax
3432; AVX1-NEXT:    movb %al, 11(%rdi)
3433; AVX1-NEXT:    shrl $16, %ecx
3434; AVX1-NEXT:    movb %cl, 8(%rdi)
3435; AVX1-NEXT:    shrl $16, %edx
3436; AVX1-NEXT:    movb %dl, 5(%rdi)
3437; AVX1-NEXT:    shrl $16, %esi
3438; AVX1-NEXT:    movb %sil, 2(%rdi)
3439; AVX1-NEXT:    retq
3440;
3441; AVX2-LABEL: smulo_v4i24:
3442; AVX2:       # %bb.0:
3443; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
3444; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
3445; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
3446; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
3447; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3448; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3449; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3450; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3451; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3452; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3453; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3454; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
3455; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3456; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3457; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3458; AVX2-NEXT:    vpslld $8, %xmm1, %xmm3
3459; AVX2-NEXT:    vpsrad $8, %xmm3, %xmm3
3460; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3461; AVX2-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3462; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
3463; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
3464; AVX2-NEXT:    movw %ax, 9(%rdi)
3465; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
3466; AVX2-NEXT:    movw %cx, 6(%rdi)
3467; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
3468; AVX2-NEXT:    movw %dx, 3(%rdi)
3469; AVX2-NEXT:    vmovd %xmm1, %esi
3470; AVX2-NEXT:    movw %si, (%rdi)
3471; AVX2-NEXT:    shrl $16, %eax
3472; AVX2-NEXT:    movb %al, 11(%rdi)
3473; AVX2-NEXT:    shrl $16, %ecx
3474; AVX2-NEXT:    movb %cl, 8(%rdi)
3475; AVX2-NEXT:    shrl $16, %edx
3476; AVX2-NEXT:    movb %dl, 5(%rdi)
3477; AVX2-NEXT:    shrl $16, %esi
3478; AVX2-NEXT:    movb %sil, 2(%rdi)
3479; AVX2-NEXT:    retq
3480;
3481; AVX512-LABEL: smulo_v4i24:
3482; AVX512:       # %bb.0:
3483; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
3484; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
3485; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
3486; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3487; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
3488; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
3489; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3490; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
3491; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
3492; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
3493; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3494; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
3495; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k0
3496; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
3497; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3498; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k1
3499; AVX512-NEXT:    korw %k0, %k1, %k1
3500; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3501; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3502; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
3503; AVX512-NEXT:    movw %ax, 9(%rdi)
3504; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
3505; AVX512-NEXT:    movw %cx, 6(%rdi)
3506; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
3507; AVX512-NEXT:    movw %dx, 3(%rdi)
3508; AVX512-NEXT:    vmovd %xmm1, %esi
3509; AVX512-NEXT:    movw %si, (%rdi)
3510; AVX512-NEXT:    shrl $16, %eax
3511; AVX512-NEXT:    movb %al, 11(%rdi)
3512; AVX512-NEXT:    shrl $16, %ecx
3513; AVX512-NEXT:    movb %cl, 8(%rdi)
3514; AVX512-NEXT:    shrl $16, %edx
3515; AVX512-NEXT:    movb %dl, 5(%rdi)
3516; AVX512-NEXT:    shrl $16, %esi
3517; AVX512-NEXT:    movb %sil, 2(%rdi)
3518; AVX512-NEXT:    retq
3519  %t = call {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
3520  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
3521  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
3522  %res = sext <4 x i1> %obit to <4 x i32>
3523  store <4 x i24> %val, <4 x i24>* %p2
3524  ret <4 x i32> %res
3525}
3526
3527define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
3528; SSE2-LABEL: smulo_v4i1:
3529; SSE2:       # %bb.0:
3530; SSE2-NEXT:    pand %xmm1, %xmm0
3531; SSE2-NEXT:    pslld $31, %xmm0
3532; SSE2-NEXT:    movmskps %xmm0, %eax
3533; SSE2-NEXT:    psrad $31, %xmm0
3534; SSE2-NEXT:    movb %al, (%rdi)
3535; SSE2-NEXT:    retq
3536;
3537; SSSE3-LABEL: smulo_v4i1:
3538; SSSE3:       # %bb.0:
3539; SSSE3-NEXT:    pand %xmm1, %xmm0
3540; SSSE3-NEXT:    pslld $31, %xmm0
3541; SSSE3-NEXT:    movmskps %xmm0, %eax
3542; SSSE3-NEXT:    psrad $31, %xmm0
3543; SSSE3-NEXT:    movb %al, (%rdi)
3544; SSSE3-NEXT:    retq
3545;
3546; SSE41-LABEL: smulo_v4i1:
3547; SSE41:       # %bb.0:
3548; SSE41-NEXT:    pand %xmm1, %xmm0
3549; SSE41-NEXT:    pslld $31, %xmm0
3550; SSE41-NEXT:    movmskps %xmm0, %eax
3551; SSE41-NEXT:    psrad $31, %xmm0
3552; SSE41-NEXT:    movb %al, (%rdi)
3553; SSE41-NEXT:    retq
3554;
3555; AVX-LABEL: smulo_v4i1:
3556; AVX:       # %bb.0:
3557; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
3558; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
3559; AVX-NEXT:    vmovmskps %xmm0, %eax
3560; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
3561; AVX-NEXT:    movb %al, (%rdi)
3562; AVX-NEXT:    retq
3563;
3564; AVX512F-LABEL: smulo_v4i1:
3565; AVX512F:       # %bb.0:
3566; AVX512F-NEXT:    vpand %xmm1, %xmm0, %xmm0
3567; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
3568; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k0
3569; AVX512F-NEXT:    vpsrad $31, %xmm0, %xmm0
3570; AVX512F-NEXT:    kmovw %k0, %eax
3571; AVX512F-NEXT:    movb %al, (%rdi)
3572; AVX512F-NEXT:    retq
3573;
3574; AVX512BW-LABEL: smulo_v4i1:
3575; AVX512BW:       # %bb.0:
3576; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
3577; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
3578; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k0
3579; AVX512BW-NEXT:    vpsrad $31, %xmm0, %xmm0
3580; AVX512BW-NEXT:    kmovd %k0, %eax
3581; AVX512BW-NEXT:    movb %al, (%rdi)
3582; AVX512BW-NEXT:    retq
3583  %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
3584  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
3585  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
3586  %res = sext <4 x i1> %obit to <4 x i32>
3587  store <4 x i1> %val, <4 x i1>* %p2
3588  ret <4 x i32> %res
3589}
3590
3591define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
3592; SSE2-LABEL: smulo_v2i128:
3593; SSE2:       # %bb.0:
3594; SSE2-NEXT:    pushq %rbp
3595; SSE2-NEXT:    pushq %r15
3596; SSE2-NEXT:    pushq %r14
3597; SSE2-NEXT:    pushq %r13
3598; SSE2-NEXT:    pushq %r12
3599; SSE2-NEXT:    pushq %rbx
3600; SSE2-NEXT:    subq $24, %rsp
3601; SSE2-NEXT:    movq %r8, %rax
3602; SSE2-NEXT:    movq %rcx, %r14
3603; SSE2-NEXT:    movq %rdx, %rbx
3604; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3605; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3606; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3607; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3608; SSE2-NEXT:    movq %rax, %rdx
3609; SSE2-NEXT:    movq %r9, %rcx
3610; SSE2-NEXT:    callq __muloti4@PLT
3611; SSE2-NEXT:    movq %rax, %r13
3612; SSE2-NEXT:    movq %rdx, %rbp
3613; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3614; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3615; SSE2-NEXT:    movq %rbx, %rdi
3616; SSE2-NEXT:    movq %r14, %rsi
3617; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3618; SSE2-NEXT:    movq %r12, %rcx
3619; SSE2-NEXT:    callq __muloti4@PLT
3620; SSE2-NEXT:    xorl %ecx, %ecx
3621; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3622; SSE2-NEXT:    sbbl %esi, %esi
3623; SSE2-NEXT:    movd %esi, %xmm1
3624; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3625; SSE2-NEXT:    sbbl %ecx, %ecx
3626; SSE2-NEXT:    movd %ecx, %xmm0
3627; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3628; SSE2-NEXT:    movq %rdx, 24(%r15)
3629; SSE2-NEXT:    movq %rax, 16(%r15)
3630; SSE2-NEXT:    movq %rbp, 8(%r15)
3631; SSE2-NEXT:    movq %r13, (%r15)
3632; SSE2-NEXT:    addq $24, %rsp
3633; SSE2-NEXT:    popq %rbx
3634; SSE2-NEXT:    popq %r12
3635; SSE2-NEXT:    popq %r13
3636; SSE2-NEXT:    popq %r14
3637; SSE2-NEXT:    popq %r15
3638; SSE2-NEXT:    popq %rbp
3639; SSE2-NEXT:    retq
3640;
3641; SSSE3-LABEL: smulo_v2i128:
3642; SSSE3:       # %bb.0:
3643; SSSE3-NEXT:    pushq %rbp
3644; SSSE3-NEXT:    pushq %r15
3645; SSSE3-NEXT:    pushq %r14
3646; SSSE3-NEXT:    pushq %r13
3647; SSSE3-NEXT:    pushq %r12
3648; SSSE3-NEXT:    pushq %rbx
3649; SSSE3-NEXT:    subq $24, %rsp
3650; SSSE3-NEXT:    movq %r8, %rax
3651; SSSE3-NEXT:    movq %rcx, %r14
3652; SSSE3-NEXT:    movq %rdx, %rbx
3653; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3654; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3655; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3656; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3657; SSSE3-NEXT:    movq %rax, %rdx
3658; SSSE3-NEXT:    movq %r9, %rcx
3659; SSSE3-NEXT:    callq __muloti4@PLT
3660; SSSE3-NEXT:    movq %rax, %r13
3661; SSSE3-NEXT:    movq %rdx, %rbp
3662; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3663; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3664; SSSE3-NEXT:    movq %rbx, %rdi
3665; SSSE3-NEXT:    movq %r14, %rsi
3666; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3667; SSSE3-NEXT:    movq %r12, %rcx
3668; SSSE3-NEXT:    callq __muloti4@PLT
3669; SSSE3-NEXT:    xorl %ecx, %ecx
3670; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3671; SSSE3-NEXT:    sbbl %esi, %esi
3672; SSSE3-NEXT:    movd %esi, %xmm1
3673; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3674; SSSE3-NEXT:    sbbl %ecx, %ecx
3675; SSSE3-NEXT:    movd %ecx, %xmm0
3676; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3677; SSSE3-NEXT:    movq %rdx, 24(%r15)
3678; SSSE3-NEXT:    movq %rax, 16(%r15)
3679; SSSE3-NEXT:    movq %rbp, 8(%r15)
3680; SSSE3-NEXT:    movq %r13, (%r15)
3681; SSSE3-NEXT:    addq $24, %rsp
3682; SSSE3-NEXT:    popq %rbx
3683; SSSE3-NEXT:    popq %r12
3684; SSSE3-NEXT:    popq %r13
3685; SSSE3-NEXT:    popq %r14
3686; SSSE3-NEXT:    popq %r15
3687; SSSE3-NEXT:    popq %rbp
3688; SSSE3-NEXT:    retq
3689;
3690; SSE41-LABEL: smulo_v2i128:
3691; SSE41:       # %bb.0:
3692; SSE41-NEXT:    pushq %rbp
3693; SSE41-NEXT:    pushq %r15
3694; SSE41-NEXT:    pushq %r14
3695; SSE41-NEXT:    pushq %r13
3696; SSE41-NEXT:    pushq %r12
3697; SSE41-NEXT:    pushq %rbx
3698; SSE41-NEXT:    subq $24, %rsp
3699; SSE41-NEXT:    movq %r8, %rax
3700; SSE41-NEXT:    movq %rcx, %r14
3701; SSE41-NEXT:    movq %rdx, %rbx
3702; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3703; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3704; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3705; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3706; SSE41-NEXT:    movq %rax, %rdx
3707; SSE41-NEXT:    movq %r9, %rcx
3708; SSE41-NEXT:    callq __muloti4@PLT
3709; SSE41-NEXT:    movq %rax, %r13
3710; SSE41-NEXT:    movq %rdx, %rbp
3711; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3712; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3713; SSE41-NEXT:    movq %rbx, %rdi
3714; SSE41-NEXT:    movq %r14, %rsi
3715; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3716; SSE41-NEXT:    movq %r12, %rcx
3717; SSE41-NEXT:    callq __muloti4@PLT
3718; SSE41-NEXT:    xorl %ecx, %ecx
3719; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3720; SSE41-NEXT:    sbbl %esi, %esi
3721; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3722; SSE41-NEXT:    sbbl %ecx, %ecx
3723; SSE41-NEXT:    movd %ecx, %xmm0
3724; SSE41-NEXT:    pinsrd $1, %esi, %xmm0
3725; SSE41-NEXT:    movq %rdx, 24(%r15)
3726; SSE41-NEXT:    movq %rax, 16(%r15)
3727; SSE41-NEXT:    movq %rbp, 8(%r15)
3728; SSE41-NEXT:    movq %r13, (%r15)
3729; SSE41-NEXT:    addq $24, %rsp
3730; SSE41-NEXT:    popq %rbx
3731; SSE41-NEXT:    popq %r12
3732; SSE41-NEXT:    popq %r13
3733; SSE41-NEXT:    popq %r14
3734; SSE41-NEXT:    popq %r15
3735; SSE41-NEXT:    popq %rbp
3736; SSE41-NEXT:    retq
3737;
3738; AVX-LABEL: smulo_v2i128:
3739; AVX:       # %bb.0:
3740; AVX-NEXT:    pushq %rbp
3741; AVX-NEXT:    pushq %r15
3742; AVX-NEXT:    pushq %r14
3743; AVX-NEXT:    pushq %r13
3744; AVX-NEXT:    pushq %r12
3745; AVX-NEXT:    pushq %rbx
3746; AVX-NEXT:    subq $24, %rsp
3747; AVX-NEXT:    movq %r8, %rax
3748; AVX-NEXT:    movq %rcx, %r14
3749; AVX-NEXT:    movq %rdx, %rbx
3750; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3751; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3752; AVX-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3753; AVX-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3754; AVX-NEXT:    movq %rax, %rdx
3755; AVX-NEXT:    movq %r9, %rcx
3756; AVX-NEXT:    callq __muloti4@PLT
3757; AVX-NEXT:    movq %rax, %r13
3758; AVX-NEXT:    movq %rdx, %rbp
3759; AVX-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3760; AVX-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3761; AVX-NEXT:    movq %rbx, %rdi
3762; AVX-NEXT:    movq %r14, %rsi
3763; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3764; AVX-NEXT:    movq %r12, %rcx
3765; AVX-NEXT:    callq __muloti4@PLT
3766; AVX-NEXT:    xorl %ecx, %ecx
3767; AVX-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3768; AVX-NEXT:    sbbl %esi, %esi
3769; AVX-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3770; AVX-NEXT:    sbbl %ecx, %ecx
3771; AVX-NEXT:    vmovd %ecx, %xmm0
3772; AVX-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
3773; AVX-NEXT:    movq %rdx, 24(%r15)
3774; AVX-NEXT:    movq %rax, 16(%r15)
3775; AVX-NEXT:    movq %rbp, 8(%r15)
3776; AVX-NEXT:    movq %r13, (%r15)
3777; AVX-NEXT:    addq $24, %rsp
3778; AVX-NEXT:    popq %rbx
3779; AVX-NEXT:    popq %r12
3780; AVX-NEXT:    popq %r13
3781; AVX-NEXT:    popq %r14
3782; AVX-NEXT:    popq %r15
3783; AVX-NEXT:    popq %rbp
3784; AVX-NEXT:    retq
3785;
3786; AVX512F-LABEL: smulo_v2i128:
3787; AVX512F:       # %bb.0:
3788; AVX512F-NEXT:    pushq %rbp
3789; AVX512F-NEXT:    pushq %r15
3790; AVX512F-NEXT:    pushq %r14
3791; AVX512F-NEXT:    pushq %r13
3792; AVX512F-NEXT:    pushq %r12
3793; AVX512F-NEXT:    pushq %rbx
3794; AVX512F-NEXT:    subq $24, %rsp
3795; AVX512F-NEXT:    movq %r8, %rax
3796; AVX512F-NEXT:    movq %rcx, %r14
3797; AVX512F-NEXT:    movq %rdx, %rbx
3798; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3799; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3800; AVX512F-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3801; AVX512F-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3802; AVX512F-NEXT:    movq %rax, %rdx
3803; AVX512F-NEXT:    movq %r9, %rcx
3804; AVX512F-NEXT:    callq __muloti4@PLT
3805; AVX512F-NEXT:    movq %rax, %r13
3806; AVX512F-NEXT:    movq %rdx, %rbp
3807; AVX512F-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3808; AVX512F-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3809; AVX512F-NEXT:    movq %rbx, %rdi
3810; AVX512F-NEXT:    movq %r14, %rsi
3811; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3812; AVX512F-NEXT:    movq %r12, %rcx
3813; AVX512F-NEXT:    callq __muloti4@PLT
3814; AVX512F-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3815; AVX512F-NEXT:    setne %cl
3816; AVX512F-NEXT:    kmovw %ecx, %k0
3817; AVX512F-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3818; AVX512F-NEXT:    setne %cl
3819; AVX512F-NEXT:    andl $1, %ecx
3820; AVX512F-NEXT:    kmovw %ecx, %k1
3821; AVX512F-NEXT:    kshiftlw $1, %k0, %k0
3822; AVX512F-NEXT:    korw %k0, %k1, %k1
3823; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3824; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3825; AVX512F-NEXT:    movq %rdx, 24(%r15)
3826; AVX512F-NEXT:    movq %rax, 16(%r15)
3827; AVX512F-NEXT:    movq %rbp, 8(%r15)
3828; AVX512F-NEXT:    movq %r13, (%r15)
3829; AVX512F-NEXT:    addq $24, %rsp
3830; AVX512F-NEXT:    popq %rbx
3831; AVX512F-NEXT:    popq %r12
3832; AVX512F-NEXT:    popq %r13
3833; AVX512F-NEXT:    popq %r14
3834; AVX512F-NEXT:    popq %r15
3835; AVX512F-NEXT:    popq %rbp
3836; AVX512F-NEXT:    retq
3837;
3838; AVX512BW-LABEL: smulo_v2i128:
3839; AVX512BW:       # %bb.0:
3840; AVX512BW-NEXT:    pushq %rbp
3841; AVX512BW-NEXT:    pushq %r15
3842; AVX512BW-NEXT:    pushq %r14
3843; AVX512BW-NEXT:    pushq %r13
3844; AVX512BW-NEXT:    pushq %r12
3845; AVX512BW-NEXT:    pushq %rbx
3846; AVX512BW-NEXT:    subq $24, %rsp
3847; AVX512BW-NEXT:    movq %r8, %rax
3848; AVX512BW-NEXT:    movq %rcx, %r14
3849; AVX512BW-NEXT:    movq %rdx, %rbx
3850; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3851; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3852; AVX512BW-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3853; AVX512BW-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3854; AVX512BW-NEXT:    movq %rax, %rdx
3855; AVX512BW-NEXT:    movq %r9, %rcx
3856; AVX512BW-NEXT:    callq __muloti4@PLT
3857; AVX512BW-NEXT:    movq %rax, %r13
3858; AVX512BW-NEXT:    movq %rdx, %rbp
3859; AVX512BW-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3860; AVX512BW-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3861; AVX512BW-NEXT:    movq %rbx, %rdi
3862; AVX512BW-NEXT:    movq %r14, %rsi
3863; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3864; AVX512BW-NEXT:    movq %r12, %rcx
3865; AVX512BW-NEXT:    callq __muloti4@PLT
3866; AVX512BW-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3867; AVX512BW-NEXT:    setne %cl
3868; AVX512BW-NEXT:    kmovd %ecx, %k0
3869; AVX512BW-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3870; AVX512BW-NEXT:    setne %cl
3871; AVX512BW-NEXT:    andl $1, %ecx
3872; AVX512BW-NEXT:    kmovw %ecx, %k1
3873; AVX512BW-NEXT:    kshiftlw $1, %k0, %k0
3874; AVX512BW-NEXT:    korw %k0, %k1, %k1
3875; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3876; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3877; AVX512BW-NEXT:    movq %rdx, 24(%r15)
3878; AVX512BW-NEXT:    movq %rax, 16(%r15)
3879; AVX512BW-NEXT:    movq %rbp, 8(%r15)
3880; AVX512BW-NEXT:    movq %r13, (%r15)
3881; AVX512BW-NEXT:    addq $24, %rsp
3882; AVX512BW-NEXT:    popq %rbx
3883; AVX512BW-NEXT:    popq %r12
3884; AVX512BW-NEXT:    popq %r13
3885; AVX512BW-NEXT:    popq %r14
3886; AVX512BW-NEXT:    popq %r15
3887; AVX512BW-NEXT:    popq %rbp
3888; AVX512BW-NEXT:    retq
3889  %t = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
3890  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
3891  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
3892  %res = sext <2 x i1> %obit to <2 x i32>
3893  store <2 x i128> %val, <2 x i128>* %p2
3894  ret <2 x i32> %res
3895}
3896