1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
8
9declare {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
10declare {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
11declare {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
12declare {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
13declare {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
14declare {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
15declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
16
17declare {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
18declare {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
19declare {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
20declare {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
21declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
22
23declare {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
24declare {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
25declare {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
26
27define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
28; SSE-LABEL: smulo_v1i32:
29; SSE:       # %bb.0:
30; SSE-NEXT:    xorl %eax, %eax
31; SSE-NEXT:    imull %esi, %edi
32; SSE-NEXT:    seto %al
33; SSE-NEXT:    negl %eax
34; SSE-NEXT:    movl %edi, (%rdx)
35; SSE-NEXT:    retq
36;
37; AVX-LABEL: smulo_v1i32:
38; AVX:       # %bb.0:
39; AVX-NEXT:    xorl %eax, %eax
40; AVX-NEXT:    imull %esi, %edi
41; AVX-NEXT:    seto %al
42; AVX-NEXT:    negl %eax
43; AVX-NEXT:    movl %edi, (%rdx)
44; AVX-NEXT:    retq
45  %t = call {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
46  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
47  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
48  %res = sext <1 x i1> %obit to <1 x i32>
49  store <1 x i32> %val, <1 x i32>* %p2
50  ret <1 x i32> %res
51}
52
53define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
54; SSE2-LABEL: smulo_v2i32:
55; SSE2:       # %bb.0:
56; SSE2-NEXT:    pxor %xmm2, %xmm2
57; SSE2-NEXT:    pxor %xmm3, %xmm3
58; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
59; SSE2-NEXT:    pand %xmm0, %xmm3
60; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
61; SSE2-NEXT:    pand %xmm1, %xmm2
62; SSE2-NEXT:    paddd %xmm3, %xmm2
63; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
64; SSE2-NEXT:    pmuludq %xmm1, %xmm0
65; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
66; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
67; SSE2-NEXT:    pmuludq %xmm3, %xmm1
68; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
69; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
70; SSE2-NEXT:    psubd %xmm2, %xmm4
71; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
72; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
73; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
74; SSE2-NEXT:    movq %xmm0, (%rdi)
75; SSE2-NEXT:    movdqa %xmm0, %xmm1
76; SSE2-NEXT:    psrad $31, %xmm1
77; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
78; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
79; SSE2-NEXT:    pxor %xmm1, %xmm0
80; SSE2-NEXT:    retq
81;
82; SSSE3-LABEL: smulo_v2i32:
83; SSSE3:       # %bb.0:
84; SSSE3-NEXT:    pxor %xmm2, %xmm2
85; SSSE3-NEXT:    pxor %xmm3, %xmm3
86; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
87; SSSE3-NEXT:    pand %xmm0, %xmm3
88; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
89; SSSE3-NEXT:    pand %xmm1, %xmm2
90; SSSE3-NEXT:    paddd %xmm3, %xmm2
91; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
92; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
93; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
94; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
95; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
96; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
97; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
98; SSSE3-NEXT:    psubd %xmm2, %xmm4
99; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
100; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
101; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
102; SSSE3-NEXT:    movq %xmm0, (%rdi)
103; SSSE3-NEXT:    movdqa %xmm0, %xmm1
104; SSSE3-NEXT:    psrad $31, %xmm1
105; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
106; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
107; SSSE3-NEXT:    pxor %xmm1, %xmm0
108; SSSE3-NEXT:    retq
109;
110; SSE41-LABEL: smulo_v2i32:
111; SSE41:       # %bb.0:
112; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
113; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
114; SSE41-NEXT:    pmuldq %xmm2, %xmm3
115; SSE41-NEXT:    movdqa %xmm0, %xmm2
116; SSE41-NEXT:    pmuldq %xmm1, %xmm2
117; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
118; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
119; SSE41-NEXT:    pmulld %xmm1, %xmm0
120; SSE41-NEXT:    movq %xmm0, (%rdi)
121; SSE41-NEXT:    psrad $31, %xmm0
122; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
123; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
124; SSE41-NEXT:    pxor %xmm1, %xmm0
125; SSE41-NEXT:    retq
126;
127; AVX1-LABEL: smulo_v2i32:
128; AVX1:       # %bb.0:
129; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
130; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
131; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
132; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
133; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
134; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
135; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
136; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
137; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
138; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
139; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
140; AVX1-NEXT:    vmovq %xmm1, (%rdi)
141; AVX1-NEXT:    retq
142;
143; AVX2-LABEL: smulo_v2i32:
144; AVX2:       # %bb.0:
145; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
146; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
147; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
148; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
149; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
150; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
151; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
152; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
153; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
154; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
155; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
156; AVX2-NEXT:    vmovq %xmm1, (%rdi)
157; AVX2-NEXT:    retq
158;
159; AVX512-LABEL: smulo_v2i32:
160; AVX512:       # %bb.0:
161; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
162; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
163; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
164; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
165; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
166; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
167; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
168; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
169; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
170; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
171; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
172; AVX512-NEXT:    vmovq %xmm1, (%rdi)
173; AVX512-NEXT:    retq
174  %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
175  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
176  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
177  %res = sext <2 x i1> %obit to <2 x i32>
178  store <2 x i32> %val, <2 x i32>* %p2
179  ret <2 x i32> %res
180}
181
182define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
183; SSE2-LABEL: smulo_v3i32:
184; SSE2:       # %bb.0:
185; SSE2-NEXT:    pxor %xmm2, %xmm2
186; SSE2-NEXT:    pxor %xmm3, %xmm3
187; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
188; SSE2-NEXT:    pand %xmm0, %xmm3
189; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
190; SSE2-NEXT:    pand %xmm1, %xmm2
191; SSE2-NEXT:    paddd %xmm3, %xmm2
192; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
193; SSE2-NEXT:    pmuludq %xmm1, %xmm0
194; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
195; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
196; SSE2-NEXT:    pmuludq %xmm3, %xmm1
197; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
198; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
199; SSE2-NEXT:    psubd %xmm2, %xmm4
200; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
201; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
202; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
203; SSE2-NEXT:    movq %xmm2, (%rdi)
204; SSE2-NEXT:    psrad $31, %xmm2
205; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
206; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
207; SSE2-NEXT:    pxor %xmm2, %xmm1
208; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
209; SSE2-NEXT:    movd %xmm0, 8(%rdi)
210; SSE2-NEXT:    movdqa %xmm1, %xmm0
211; SSE2-NEXT:    retq
212;
213; SSSE3-LABEL: smulo_v3i32:
214; SSSE3:       # %bb.0:
215; SSSE3-NEXT:    pxor %xmm2, %xmm2
216; SSSE3-NEXT:    pxor %xmm3, %xmm3
217; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
218; SSSE3-NEXT:    pand %xmm0, %xmm3
219; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
220; SSSE3-NEXT:    pand %xmm1, %xmm2
221; SSSE3-NEXT:    paddd %xmm3, %xmm2
222; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
223; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
224; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
225; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
226; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
227; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
228; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
229; SSSE3-NEXT:    psubd %xmm2, %xmm4
230; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
231; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
232; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
233; SSSE3-NEXT:    movq %xmm2, (%rdi)
234; SSSE3-NEXT:    psrad $31, %xmm2
235; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
236; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
237; SSSE3-NEXT:    pxor %xmm2, %xmm1
238; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
239; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
240; SSSE3-NEXT:    movdqa %xmm1, %xmm0
241; SSSE3-NEXT:    retq
242;
243; SSE41-LABEL: smulo_v3i32:
244; SSE41:       # %bb.0:
245; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
246; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
247; SSE41-NEXT:    pmuldq %xmm2, %xmm3
248; SSE41-NEXT:    movdqa %xmm0, %xmm2
249; SSE41-NEXT:    pmuldq %xmm1, %xmm2
250; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
251; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
252; SSE41-NEXT:    pmulld %xmm1, %xmm0
253; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
254; SSE41-NEXT:    movq %xmm0, (%rdi)
255; SSE41-NEXT:    psrad $31, %xmm0
256; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
257; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
258; SSE41-NEXT:    pxor %xmm1, %xmm0
259; SSE41-NEXT:    retq
260;
261; AVX1-LABEL: smulo_v3i32:
262; AVX1:       # %bb.0:
263; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
264; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
265; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
266; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
267; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
268; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
269; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
270; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
271; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
272; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
273; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
274; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
275; AVX1-NEXT:    vmovq %xmm1, (%rdi)
276; AVX1-NEXT:    retq
277;
278; AVX2-LABEL: smulo_v3i32:
279; AVX2:       # %bb.0:
280; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
281; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
282; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
283; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
284; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
285; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
286; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
287; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
288; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
289; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
290; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
291; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
292; AVX2-NEXT:    vmovq %xmm1, (%rdi)
293; AVX2-NEXT:    retq
294;
295; AVX512-LABEL: smulo_v3i32:
296; AVX512:       # %bb.0:
297; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
298; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
299; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
300; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
301; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
302; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
303; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
304; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
305; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
306; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
307; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
308; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
309; AVX512-NEXT:    vmovq %xmm1, (%rdi)
310; AVX512-NEXT:    retq
311  %t = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
312  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
313  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
314  %res = sext <3 x i1> %obit to <3 x i32>
315  store <3 x i32> %val, <3 x i32>* %p2
316  ret <3 x i32> %res
317}
318
319define <4 x i32> @smulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
320; SSE2-LABEL: smulo_v4i32:
321; SSE2:       # %bb.0:
322; SSE2-NEXT:    pxor %xmm2, %xmm2
323; SSE2-NEXT:    pxor %xmm3, %xmm3
324; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
325; SSE2-NEXT:    pand %xmm0, %xmm3
326; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
327; SSE2-NEXT:    pand %xmm1, %xmm2
328; SSE2-NEXT:    paddd %xmm3, %xmm2
329; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
330; SSE2-NEXT:    pmuludq %xmm1, %xmm0
331; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
332; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
333; SSE2-NEXT:    pmuludq %xmm3, %xmm1
334; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
335; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
336; SSE2-NEXT:    psubd %xmm2, %xmm4
337; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
338; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
339; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
340; SSE2-NEXT:    movdqa %xmm0, (%rdi)
341; SSE2-NEXT:    movdqa %xmm0, %xmm1
342; SSE2-NEXT:    psrad $31, %xmm1
343; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
344; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
345; SSE2-NEXT:    pxor %xmm1, %xmm0
346; SSE2-NEXT:    retq
347;
348; SSSE3-LABEL: smulo_v4i32:
349; SSSE3:       # %bb.0:
350; SSSE3-NEXT:    pxor %xmm2, %xmm2
351; SSSE3-NEXT:    pxor %xmm3, %xmm3
352; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
353; SSSE3-NEXT:    pand %xmm0, %xmm3
354; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
355; SSSE3-NEXT:    pand %xmm1, %xmm2
356; SSSE3-NEXT:    paddd %xmm3, %xmm2
357; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
358; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
359; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
360; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
361; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
362; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
363; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
364; SSSE3-NEXT:    psubd %xmm2, %xmm4
365; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
366; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
367; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
368; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
369; SSSE3-NEXT:    movdqa %xmm0, %xmm1
370; SSSE3-NEXT:    psrad $31, %xmm1
371; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
372; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
373; SSSE3-NEXT:    pxor %xmm1, %xmm0
374; SSSE3-NEXT:    retq
375;
376; SSE41-LABEL: smulo_v4i32:
377; SSE41:       # %bb.0:
378; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
379; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
380; SSE41-NEXT:    pmuldq %xmm2, %xmm3
381; SSE41-NEXT:    movdqa %xmm0, %xmm2
382; SSE41-NEXT:    pmuldq %xmm1, %xmm2
383; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
384; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
385; SSE41-NEXT:    pmulld %xmm1, %xmm0
386; SSE41-NEXT:    movdqa %xmm0, (%rdi)
387; SSE41-NEXT:    psrad $31, %xmm0
388; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
389; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
390; SSE41-NEXT:    pxor %xmm1, %xmm0
391; SSE41-NEXT:    retq
392;
393; AVX1-LABEL: smulo_v4i32:
394; AVX1:       # %bb.0:
395; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
396; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
397; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
398; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
399; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
400; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
401; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
402; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
403; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
404; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
405; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
406; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
407; AVX1-NEXT:    retq
408;
409; AVX2-LABEL: smulo_v4i32:
410; AVX2:       # %bb.0:
411; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
412; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
413; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
414; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
415; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
416; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
417; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
418; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
419; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
420; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
421; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
422; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
423; AVX2-NEXT:    retq
424;
425; AVX512-LABEL: smulo_v4i32:
426; AVX512:       # %bb.0:
427; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
428; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
429; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
430; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
431; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
432; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
433; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
434; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
435; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
436; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
437; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
438; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
439; AVX512-NEXT:    retq
440  %t = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
441  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
442  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
443  %res = sext <4 x i1> %obit to <4 x i32>
444  store <4 x i32> %val, <4 x i32>* %p2
445  ret <4 x i32> %res
446}
447
448define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
449; SSE2-LABEL: smulo_v6i32:
450; SSE2:       # %bb.0:
451; SSE2-NEXT:    movq %rdi, %rax
452; SSE2-NEXT:    movd %r8d, %xmm0
453; SSE2-NEXT:    movd %ecx, %xmm1
454; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
455; SSE2-NEXT:    movd %edx, %xmm0
456; SSE2-NEXT:    movd %esi, %xmm2
457; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
458; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
459; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
460; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
461; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
462; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
463; SSE2-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
464; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
465; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
466; SSE2-NEXT:    movd %r9d, %xmm10
467; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
468; SSE2-NEXT:    movdqa %xmm10, %xmm9
469; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
470; SSE2-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
471; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
472; SSE2-NEXT:    pmuludq %xmm6, %xmm10
473; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
474; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
475; SSE2-NEXT:    pxor %xmm7, %xmm7
476; SSE2-NEXT:    pxor %xmm4, %xmm4
477; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
478; SSE2-NEXT:    pand %xmm2, %xmm4
479; SSE2-NEXT:    pxor %xmm3, %xmm3
480; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
481; SSE2-NEXT:    pand %xmm5, %xmm3
482; SSE2-NEXT:    paddd %xmm4, %xmm3
483; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
484; SSE2-NEXT:    pmuludq %xmm5, %xmm2
485; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
486; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
487; SSE2-NEXT:    pmuludq %xmm4, %xmm5
488; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
489; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
490; SSE2-NEXT:    psubd %xmm3, %xmm0
491; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
492; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
493; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
494; SSE2-NEXT:    movdqa %xmm2, (%rcx)
495; SSE2-NEXT:    psrad $31, %xmm2
496; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
497; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
498; SSE2-NEXT:    pxor %xmm0, %xmm2
499; SSE2-NEXT:    pxor %xmm3, %xmm3
500; SSE2-NEXT:    pcmpgtd %xmm6, %xmm3
501; SSE2-NEXT:    pand %xmm9, %xmm3
502; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
503; SSE2-NEXT:    pand %xmm6, %xmm7
504; SSE2-NEXT:    paddd %xmm3, %xmm7
505; SSE2-NEXT:    pmuludq %xmm8, %xmm1
506; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
507; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
508; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
509; SSE2-NEXT:    psubd %xmm7, %xmm4
510; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
511; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
512; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
513; SSE2-NEXT:    movq %xmm3, 16(%rcx)
514; SSE2-NEXT:    psrad $31, %xmm3
515; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
516; SSE2-NEXT:    pxor %xmm0, %xmm3
517; SSE2-NEXT:    movq %xmm3, 16(%rdi)
518; SSE2-NEXT:    movdqa %xmm2, (%rdi)
519; SSE2-NEXT:    retq
520;
521; SSSE3-LABEL: smulo_v6i32:
522; SSSE3:       # %bb.0:
523; SSSE3-NEXT:    movq %rdi, %rax
524; SSSE3-NEXT:    movd %r8d, %xmm0
525; SSSE3-NEXT:    movd %ecx, %xmm1
526; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
527; SSSE3-NEXT:    movd %edx, %xmm0
528; SSSE3-NEXT:    movd %esi, %xmm2
529; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
530; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
531; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
532; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
533; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
534; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
535; SSSE3-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
536; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
537; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
538; SSSE3-NEXT:    movd %r9d, %xmm10
539; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
540; SSSE3-NEXT:    movdqa %xmm10, %xmm9
541; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
542; SSSE3-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
543; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
544; SSSE3-NEXT:    pmuludq %xmm6, %xmm10
545; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
546; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
547; SSSE3-NEXT:    pxor %xmm7, %xmm7
548; SSSE3-NEXT:    pxor %xmm4, %xmm4
549; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
550; SSSE3-NEXT:    pand %xmm2, %xmm4
551; SSSE3-NEXT:    pxor %xmm3, %xmm3
552; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
553; SSSE3-NEXT:    pand %xmm5, %xmm3
554; SSSE3-NEXT:    paddd %xmm4, %xmm3
555; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
556; SSSE3-NEXT:    pmuludq %xmm5, %xmm2
557; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
558; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
559; SSSE3-NEXT:    pmuludq %xmm4, %xmm5
560; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,3,2,3]
561; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
562; SSSE3-NEXT:    psubd %xmm3, %xmm0
563; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
564; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
565; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
566; SSSE3-NEXT:    movdqa %xmm2, (%rcx)
567; SSSE3-NEXT:    psrad $31, %xmm2
568; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
569; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
570; SSSE3-NEXT:    pxor %xmm0, %xmm2
571; SSSE3-NEXT:    pxor %xmm3, %xmm3
572; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm3
573; SSSE3-NEXT:    pand %xmm9, %xmm3
574; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm7
575; SSSE3-NEXT:    pand %xmm6, %xmm7
576; SSSE3-NEXT:    paddd %xmm3, %xmm7
577; SSSE3-NEXT:    pmuludq %xmm8, %xmm1
578; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
579; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
580; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
581; SSSE3-NEXT:    psubd %xmm7, %xmm4
582; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
583; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
584; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
585; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
586; SSSE3-NEXT:    psrad $31, %xmm3
587; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
588; SSSE3-NEXT:    pxor %xmm0, %xmm3
589; SSSE3-NEXT:    movq %xmm3, 16(%rdi)
590; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
591; SSSE3-NEXT:    retq
592;
593; SSE41-LABEL: smulo_v6i32:
594; SSE41:       # %bb.0:
595; SSE41-NEXT:    movq %rdi, %rax
596; SSE41-NEXT:    movd %esi, %xmm2
597; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
598; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
599; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
600; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
601; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
602; SSE41-NEXT:    movdqa %xmm1, %xmm0
603; SSE41-NEXT:    pmuldq %xmm2, %xmm1
604; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
605; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
606; SSE41-NEXT:    movd %r9d, %xmm4
607; SSE41-NEXT:    movdqa %xmm4, %xmm5
608; SSE41-NEXT:    pmuldq %xmm3, %xmm4
609; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
610; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
611; SSE41-NEXT:    pmulld %xmm3, %xmm5
612; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
613; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
614; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
615; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
616; SSE41-NEXT:    pmuldq %xmm3, %xmm6
617; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
618; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
619; SSE41-NEXT:    pmulld %xmm2, %xmm0
620; SSE41-NEXT:    movdqa %xmm0, (%rcx)
621; SSE41-NEXT:    psrad $31, %xmm0
622; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
623; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
624; SSE41-NEXT:    pxor %xmm1, %xmm0
625; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
626; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
627; SSE41-NEXT:    pmuldq %xmm2, %xmm3
628; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
629; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
630; SSE41-NEXT:    movq %xmm5, 16(%rcx)
631; SSE41-NEXT:    psrad $31, %xmm5
632; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
633; SSE41-NEXT:    pxor %xmm1, %xmm5
634; SSE41-NEXT:    movq %xmm5, 16(%rdi)
635; SSE41-NEXT:    movdqa %xmm0, (%rdi)
636; SSE41-NEXT:    retq
637;
638; AVX1-LABEL: smulo_v6i32:
639; AVX1:       # %bb.0:
640; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
641; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
642; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
643; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
644; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
645; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
646; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
647; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
648; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
649; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
650; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
651; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
652; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
653; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
654; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
655; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
656; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
657; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
658; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
659; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
660; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
661; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
662; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
663; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
664; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
665; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
666; AVX1-NEXT:    retq
667;
668; AVX2-LABEL: smulo_v6i32:
669; AVX2:       # %bb.0:
670; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
671; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
672; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
673; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
674; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
675; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
676; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
677; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
678; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
679; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
680; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
681; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
682; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
683; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
684; AVX2-NEXT:    retq
685;
686; AVX512-LABEL: smulo_v6i32:
687; AVX512:       # %bb.0:
688; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
689; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
690; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
691; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
692; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
693; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
694; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
695; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
696; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
697; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
698; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
699; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
700; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
701; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
702; AVX512-NEXT:    retq
703  %t = call {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
704  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
705  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
706  %res = sext <6 x i1> %obit to <6 x i32>
707  store <6 x i32> %val, <6 x i32>* %p2
708  ret <6 x i32> %res
709}
710
711define <8 x i32> @smulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
712; SSE2-LABEL: smulo_v8i32:
713; SSE2:       # %bb.0:
714; SSE2-NEXT:    pxor %xmm4, %xmm4
715; SSE2-NEXT:    pxor %xmm5, %xmm5
716; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
717; SSE2-NEXT:    pand %xmm0, %xmm5
718; SSE2-NEXT:    pxor %xmm6, %xmm6
719; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
720; SSE2-NEXT:    pand %xmm2, %xmm6
721; SSE2-NEXT:    paddd %xmm5, %xmm6
722; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
723; SSE2-NEXT:    pmuludq %xmm2, %xmm0
724; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
725; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
726; SSE2-NEXT:    pmuludq %xmm7, %xmm2
727; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
728; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
729; SSE2-NEXT:    psubd %xmm6, %xmm5
730; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
731; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
732; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
733; SSE2-NEXT:    movdqa %xmm0, (%rdi)
734; SSE2-NEXT:    psrad $31, %xmm0
735; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
736; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
737; SSE2-NEXT:    pxor %xmm2, %xmm0
738; SSE2-NEXT:    pxor %xmm5, %xmm5
739; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
740; SSE2-NEXT:    pand %xmm1, %xmm5
741; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
742; SSE2-NEXT:    pand %xmm3, %xmm4
743; SSE2-NEXT:    paddd %xmm5, %xmm4
744; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
745; SSE2-NEXT:    pmuludq %xmm3, %xmm1
746; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
747; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
748; SSE2-NEXT:    pmuludq %xmm5, %xmm3
749; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
750; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
751; SSE2-NEXT:    psubd %xmm4, %xmm6
752; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
753; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
754; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
755; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
756; SSE2-NEXT:    psrad $31, %xmm1
757; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
758; SSE2-NEXT:    pxor %xmm2, %xmm1
759; SSE2-NEXT:    retq
760;
761; SSSE3-LABEL: smulo_v8i32:
762; SSSE3:       # %bb.0:
763; SSSE3-NEXT:    pxor %xmm4, %xmm4
764; SSSE3-NEXT:    pxor %xmm5, %xmm5
765; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
766; SSSE3-NEXT:    pand %xmm0, %xmm5
767; SSSE3-NEXT:    pxor %xmm6, %xmm6
768; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
769; SSSE3-NEXT:    pand %xmm2, %xmm6
770; SSSE3-NEXT:    paddd %xmm5, %xmm6
771; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
772; SSSE3-NEXT:    pmuludq %xmm2, %xmm0
773; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
774; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
775; SSSE3-NEXT:    pmuludq %xmm7, %xmm2
776; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
777; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
778; SSSE3-NEXT:    psubd %xmm6, %xmm5
779; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
780; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
781; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
782; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
783; SSSE3-NEXT:    psrad $31, %xmm0
784; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
785; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
786; SSSE3-NEXT:    pxor %xmm2, %xmm0
787; SSSE3-NEXT:    pxor %xmm5, %xmm5
788; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
789; SSSE3-NEXT:    pand %xmm1, %xmm5
790; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
791; SSSE3-NEXT:    pand %xmm3, %xmm4
792; SSSE3-NEXT:    paddd %xmm5, %xmm4
793; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
794; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
795; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
796; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
797; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
798; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
799; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
800; SSSE3-NEXT:    psubd %xmm4, %xmm6
801; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
802; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
803; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
804; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
805; SSSE3-NEXT:    psrad $31, %xmm1
806; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm1
807; SSSE3-NEXT:    pxor %xmm2, %xmm1
808; SSSE3-NEXT:    retq
809;
810; SSE41-LABEL: smulo_v8i32:
811; SSE41:       # %bb.0:
812; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
813; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
814; SSE41-NEXT:    pmuldq %xmm4, %xmm5
815; SSE41-NEXT:    movdqa %xmm0, %xmm4
816; SSE41-NEXT:    pmuldq %xmm2, %xmm4
817; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
818; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
819; SSE41-NEXT:    pmulld %xmm2, %xmm0
820; SSE41-NEXT:    movdqa %xmm0, (%rdi)
821; SSE41-NEXT:    psrad $31, %xmm0
822; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
823; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
824; SSE41-NEXT:    pxor %xmm2, %xmm0
825; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
826; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
827; SSE41-NEXT:    pmuldq %xmm4, %xmm5
828; SSE41-NEXT:    movdqa %xmm1, %xmm4
829; SSE41-NEXT:    pmuldq %xmm3, %xmm4
830; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
831; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
832; SSE41-NEXT:    pmulld %xmm3, %xmm1
833; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
834; SSE41-NEXT:    psrad $31, %xmm1
835; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
836; SSE41-NEXT:    pxor %xmm2, %xmm1
837; SSE41-NEXT:    retq
838;
839; AVX1-LABEL: smulo_v8i32:
840; AVX1:       # %bb.0:
841; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
842; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
843; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
844; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
845; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
846; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
847; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
848; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
849; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
850; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
851; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
852; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
853; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
854; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
855; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
856; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
857; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
858; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
859; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
860; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
861; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
862; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
863; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
864; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
865; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
866; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
867; AVX1-NEXT:    retq
868;
869; AVX2-LABEL: smulo_v8i32:
870; AVX2:       # %bb.0:
871; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
872; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
873; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
874; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
875; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
876; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
877; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
878; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
879; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
880; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
881; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
882; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
883; AVX2-NEXT:    retq
884;
885; AVX512-LABEL: smulo_v8i32:
886; AVX512:       # %bb.0:
887; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
888; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
889; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
890; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
891; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
892; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
893; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
894; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
895; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
896; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
897; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
898; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
899; AVX512-NEXT:    retq
900  %t = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
901  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
902  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
903  %res = sext <8 x i1> %obit to <8 x i32>
904  store <8 x i32> %val, <8 x i32>* %p2
905  ret <8 x i32> %res
906}
907
908define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
909; SSE2-LABEL: smulo_v16i32:
910; SSE2:       # %bb.0:
911; SSE2-NEXT:    pxor %xmm8, %xmm8
912; SSE2-NEXT:    pxor %xmm9, %xmm9
913; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
914; SSE2-NEXT:    pand %xmm0, %xmm9
915; SSE2-NEXT:    pxor %xmm10, %xmm10
916; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
917; SSE2-NEXT:    pand %xmm4, %xmm10
918; SSE2-NEXT:    paddd %xmm9, %xmm10
919; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
920; SSE2-NEXT:    pmuludq %xmm4, %xmm0
921; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
922; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
923; SSE2-NEXT:    pmuludq %xmm11, %xmm12
924; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
925; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
926; SSE2-NEXT:    psubd %xmm10, %xmm9
927; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
928; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
929; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
930; SSE2-NEXT:    movdqa %xmm0, (%rdi)
931; SSE2-NEXT:    psrad $31, %xmm0
932; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
933; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
934; SSE2-NEXT:    pxor %xmm9, %xmm0
935; SSE2-NEXT:    pxor %xmm10, %xmm10
936; SSE2-NEXT:    pcmpgtd %xmm5, %xmm10
937; SSE2-NEXT:    pand %xmm1, %xmm10
938; SSE2-NEXT:    pxor %xmm4, %xmm4
939; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
940; SSE2-NEXT:    pand %xmm5, %xmm4
941; SSE2-NEXT:    paddd %xmm10, %xmm4
942; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
943; SSE2-NEXT:    pmuludq %xmm5, %xmm1
944; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
945; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
946; SSE2-NEXT:    pmuludq %xmm10, %xmm12
947; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
948; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
949; SSE2-NEXT:    psubd %xmm4, %xmm11
950; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
951; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
952; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
953; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
954; SSE2-NEXT:    psrad $31, %xmm1
955; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
956; SSE2-NEXT:    pxor %xmm9, %xmm1
957; SSE2-NEXT:    pxor %xmm4, %xmm4
958; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
959; SSE2-NEXT:    pand %xmm2, %xmm4
960; SSE2-NEXT:    pxor %xmm5, %xmm5
961; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
962; SSE2-NEXT:    pand %xmm6, %xmm5
963; SSE2-NEXT:    paddd %xmm4, %xmm5
964; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
965; SSE2-NEXT:    pmuludq %xmm6, %xmm2
966; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
967; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
968; SSE2-NEXT:    pmuludq %xmm10, %xmm11
969; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
970; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
971; SSE2-NEXT:    psubd %xmm5, %xmm4
972; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
973; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
974; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
975; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
976; SSE2-NEXT:    psrad $31, %xmm2
977; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
978; SSE2-NEXT:    pxor %xmm9, %xmm2
979; SSE2-NEXT:    pxor %xmm4, %xmm4
980; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
981; SSE2-NEXT:    pand %xmm3, %xmm4
982; SSE2-NEXT:    pcmpgtd %xmm3, %xmm8
983; SSE2-NEXT:    pand %xmm7, %xmm8
984; SSE2-NEXT:    paddd %xmm4, %xmm8
985; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
986; SSE2-NEXT:    pmuludq %xmm7, %xmm3
987; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
988; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
989; SSE2-NEXT:    pmuludq %xmm4, %xmm6
990; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
991; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
992; SSE2-NEXT:    psubd %xmm8, %xmm5
993; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
994; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
995; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
996; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
997; SSE2-NEXT:    psrad $31, %xmm3
998; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
999; SSE2-NEXT:    pxor %xmm9, %xmm3
1000; SSE2-NEXT:    retq
1001;
1002; SSSE3-LABEL: smulo_v16i32:
1003; SSSE3:       # %bb.0:
1004; SSSE3-NEXT:    pxor %xmm8, %xmm8
1005; SSSE3-NEXT:    pxor %xmm9, %xmm9
1006; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
1007; SSSE3-NEXT:    pand %xmm0, %xmm9
1008; SSSE3-NEXT:    pxor %xmm10, %xmm10
1009; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm10
1010; SSSE3-NEXT:    pand %xmm4, %xmm10
1011; SSSE3-NEXT:    paddd %xmm9, %xmm10
1012; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
1013; SSSE3-NEXT:    pmuludq %xmm4, %xmm0
1014; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
1015; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
1016; SSSE3-NEXT:    pmuludq %xmm11, %xmm12
1017; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
1018; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
1019; SSSE3-NEXT:    psubd %xmm10, %xmm9
1020; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1021; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1022; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1023; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
1024; SSSE3-NEXT:    psrad $31, %xmm0
1025; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
1026; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
1027; SSSE3-NEXT:    pxor %xmm9, %xmm0
1028; SSSE3-NEXT:    pxor %xmm10, %xmm10
1029; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm10
1030; SSSE3-NEXT:    pand %xmm1, %xmm10
1031; SSSE3-NEXT:    pxor %xmm4, %xmm4
1032; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
1033; SSSE3-NEXT:    pand %xmm5, %xmm4
1034; SSSE3-NEXT:    paddd %xmm10, %xmm4
1035; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1036; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
1037; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
1038; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
1039; SSSE3-NEXT:    pmuludq %xmm10, %xmm12
1040; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
1041; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
1042; SSSE3-NEXT:    psubd %xmm4, %xmm11
1043; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1044; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1045; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1046; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
1047; SSSE3-NEXT:    psrad $31, %xmm1
1048; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
1049; SSSE3-NEXT:    pxor %xmm9, %xmm1
1050; SSSE3-NEXT:    pxor %xmm4, %xmm4
1051; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
1052; SSSE3-NEXT:    pand %xmm2, %xmm4
1053; SSSE3-NEXT:    pxor %xmm5, %xmm5
1054; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
1055; SSSE3-NEXT:    pand %xmm6, %xmm5
1056; SSSE3-NEXT:    paddd %xmm4, %xmm5
1057; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
1058; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
1059; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
1060; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
1061; SSSE3-NEXT:    pmuludq %xmm10, %xmm11
1062; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
1063; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
1064; SSSE3-NEXT:    psubd %xmm5, %xmm4
1065; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
1066; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
1067; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1068; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
1069; SSSE3-NEXT:    psrad $31, %xmm2
1070; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
1071; SSSE3-NEXT:    pxor %xmm9, %xmm2
1072; SSSE3-NEXT:    pxor %xmm4, %xmm4
1073; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
1074; SSSE3-NEXT:    pand %xmm3, %xmm4
1075; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm8
1076; SSSE3-NEXT:    pand %xmm7, %xmm8
1077; SSSE3-NEXT:    paddd %xmm4, %xmm8
1078; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
1079; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
1080; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
1081; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
1082; SSSE3-NEXT:    pmuludq %xmm4, %xmm6
1083; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
1084; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
1085; SSSE3-NEXT:    psubd %xmm8, %xmm5
1086; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
1087; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
1088; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1089; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
1090; SSSE3-NEXT:    psrad $31, %xmm3
1091; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
1092; SSSE3-NEXT:    pxor %xmm9, %xmm3
1093; SSSE3-NEXT:    retq
1094;
1095; SSE41-LABEL: smulo_v16i32:
1096; SSE41:       # %bb.0:
1097; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
1098; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
1099; SSE41-NEXT:    pmuldq %xmm8, %xmm9
1100; SSE41-NEXT:    movdqa %xmm0, %xmm8
1101; SSE41-NEXT:    pmuldq %xmm4, %xmm8
1102; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
1103; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
1104; SSE41-NEXT:    pmulld %xmm4, %xmm0
1105; SSE41-NEXT:    movdqa %xmm0, (%rdi)
1106; SSE41-NEXT:    psrad $31, %xmm0
1107; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
1108; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1109; SSE41-NEXT:    pxor %xmm8, %xmm0
1110; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
1111; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1112; SSE41-NEXT:    pmuldq %xmm9, %xmm10
1113; SSE41-NEXT:    movdqa %xmm1, %xmm4
1114; SSE41-NEXT:    pmuldq %xmm5, %xmm4
1115; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1116; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm10[2,3],xmm4[4,5],xmm10[6,7]
1117; SSE41-NEXT:    pmulld %xmm5, %xmm1
1118; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
1119; SSE41-NEXT:    psrad $31, %xmm1
1120; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
1121; SSE41-NEXT:    pxor %xmm8, %xmm1
1122; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
1123; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
1124; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1125; SSE41-NEXT:    movdqa %xmm2, %xmm4
1126; SSE41-NEXT:    pmuldq %xmm6, %xmm4
1127; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1128; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1129; SSE41-NEXT:    pmulld %xmm6, %xmm2
1130; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
1131; SSE41-NEXT:    psrad $31, %xmm2
1132; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
1133; SSE41-NEXT:    pxor %xmm8, %xmm2
1134; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
1135; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
1136; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1137; SSE41-NEXT:    movdqa %xmm3, %xmm4
1138; SSE41-NEXT:    pmuldq %xmm7, %xmm4
1139; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1140; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1141; SSE41-NEXT:    pmulld %xmm7, %xmm3
1142; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
1143; SSE41-NEXT:    psrad $31, %xmm3
1144; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
1145; SSE41-NEXT:    pxor %xmm8, %xmm3
1146; SSE41-NEXT:    retq
1147;
1148; AVX1-LABEL: smulo_v16i32:
1149; AVX1:       # %bb.0:
1150; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
1151; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
1152; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
1153; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
1154; AVX1-NEXT:    vpmuldq %xmm5, %xmm7, %xmm5
1155; AVX1-NEXT:    vpmuldq %xmm4, %xmm6, %xmm7
1156; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1157; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1158; AVX1-NEXT:    vpmulld %xmm4, %xmm6, %xmm8
1159; AVX1-NEXT:    vpsrad $31, %xmm8, %xmm6
1160; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm6
1161; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
1162; AVX1-NEXT:    vpxor %xmm6, %xmm9, %xmm6
1163; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
1164; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
1165; AVX1-NEXT:    vpmuldq %xmm7, %xmm4, %xmm4
1166; AVX1-NEXT:    vpmuldq %xmm3, %xmm1, %xmm7
1167; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1168; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3],xmm7[4,5],xmm4[6,7]
1169; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1170; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm1
1171; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
1172; AVX1-NEXT:    vpxor %xmm1, %xmm9, %xmm1
1173; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
1174; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1175; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
1176; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1177; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
1178; AVX1-NEXT:    vpmuldq %xmm6, %xmm5, %xmm5
1179; AVX1-NEXT:    vpmuldq %xmm4, %xmm7, %xmm6
1180; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
1181; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
1182; AVX1-NEXT:    vpmulld %xmm4, %xmm7, %xmm4
1183; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm6
1184; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm5
1185; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
1186; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
1187; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
1188; AVX1-NEXT:    vpmuldq %xmm6, %xmm7, %xmm6
1189; AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm7
1190; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1191; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
1192; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1193; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm0
1194; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
1195; AVX1-NEXT:    vpxor %xmm0, %xmm9, %xmm0
1196; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
1197; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
1198; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
1199; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1200; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
1201; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
1202; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1203; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1204; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1205; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1206; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1207; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
1208; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1209; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1210; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1211; AVX1-NEXT:    retq
1212;
1213; AVX2-LABEL: smulo_v16i32:
1214; AVX2:       # %bb.0:
1215; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1216; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1217; AVX2-NEXT:    vpmuldq %ymm4, %ymm5, %ymm4
1218; AVX2-NEXT:    vpmuldq %ymm3, %ymm1, %ymm5
1219; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1220; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1221; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1222; AVX2-NEXT:    vpsrad $31, %ymm3, %ymm1
1223; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm4, %ymm1
1224; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
1225; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
1226; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1227; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
1228; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[1,1,3,3,5,5,7,7]
1229; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm0[1,1,3,3,5,5,7,7]
1230; AVX2-NEXT:    vpmuldq %ymm5, %ymm6, %ymm5
1231; AVX2-NEXT:    vpmuldq %ymm2, %ymm0, %ymm6
1232; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,1,3,3,5,5,7,7]
1233; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
1234; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1235; AVX2-NEXT:    vpsrad $31, %ymm2, %ymm0
1236; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
1237; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
1238; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
1239; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
1240; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
1241; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
1242; AVX2-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
1243; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1244; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1245; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1246; AVX2-NEXT:    retq
1247;
1248; AVX512-LABEL: smulo_v16i32:
1249; AVX512:       # %bb.0:
1250; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm2
1251; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1252; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1253; AVX512-NEXT:    vpmuldq %zmm3, %zmm4, %zmm3
1254; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1255; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1256; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1257; AVX512-NEXT:    vpsrad $31, %zmm1, %zmm0
1258; AVX512-NEXT:    vpcmpneqd %zmm0, %zmm4, %k1
1259; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1260; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1261; AVX512-NEXT:    retq
1262  %t = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1263  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1264  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1265  %res = sext <16 x i1> %obit to <16 x i32>
1266  store <16 x i32> %val, <16 x i32>* %p2
1267  ret <16 x i32> %res
1268}
1269
1270define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1271; SSE2-LABEL: smulo_v16i8:
1272; SSE2:       # %bb.0:
1273; SSE2-NEXT:    movdqa %xmm1, %xmm2
1274; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1275; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1276; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1277; SSE2-NEXT:    movdqa %xmm0, %xmm4
1278; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1279; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1280; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1281; SSE2-NEXT:    pmullw %xmm1, %xmm0
1282; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1283; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1284; SSE2-NEXT:    pmullw %xmm2, %xmm4
1285; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1286; SSE2-NEXT:    pand %xmm1, %xmm0
1287; SSE2-NEXT:    pand %xmm1, %xmm4
1288; SSE2-NEXT:    packuswb %xmm0, %xmm4
1289; SSE2-NEXT:    pxor %xmm0, %xmm0
1290; SSE2-NEXT:    pcmpgtb %xmm4, %xmm0
1291; SSE2-NEXT:    psraw $8, %xmm3
1292; SSE2-NEXT:    psraw $8, %xmm6
1293; SSE2-NEXT:    pmullw %xmm3, %xmm6
1294; SSE2-NEXT:    psrlw $8, %xmm6
1295; SSE2-NEXT:    psraw $8, %xmm5
1296; SSE2-NEXT:    psraw $8, %xmm7
1297; SSE2-NEXT:    pmullw %xmm5, %xmm7
1298; SSE2-NEXT:    psrlw $8, %xmm7
1299; SSE2-NEXT:    packuswb %xmm6, %xmm7
1300; SSE2-NEXT:    pcmpeqb %xmm0, %xmm7
1301; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1302; SSE2-NEXT:    pxor %xmm7, %xmm3
1303; SSE2-NEXT:    movdqa %xmm3, %xmm1
1304; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1305; SSE2-NEXT:    movdqa %xmm1, %xmm0
1306; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1307; SSE2-NEXT:    pslld $31, %xmm0
1308; SSE2-NEXT:    psrad $31, %xmm0
1309; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1310; SSE2-NEXT:    pslld $31, %xmm1
1311; SSE2-NEXT:    psrad $31, %xmm1
1312; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1313; SSE2-NEXT:    movdqa %xmm3, %xmm2
1314; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1315; SSE2-NEXT:    pslld $31, %xmm2
1316; SSE2-NEXT:    psrad $31, %xmm2
1317; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1318; SSE2-NEXT:    pslld $31, %xmm3
1319; SSE2-NEXT:    psrad $31, %xmm3
1320; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1321; SSE2-NEXT:    retq
1322;
1323; SSSE3-LABEL: smulo_v16i8:
1324; SSSE3:       # %bb.0:
1325; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1326; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1327; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1328; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1329; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1330; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1331; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1332; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1333; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1334; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1335; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1336; SSSE3-NEXT:    pmullw %xmm2, %xmm4
1337; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1338; SSSE3-NEXT:    pand %xmm1, %xmm0
1339; SSSE3-NEXT:    pand %xmm1, %xmm4
1340; SSSE3-NEXT:    packuswb %xmm0, %xmm4
1341; SSSE3-NEXT:    pxor %xmm0, %xmm0
1342; SSSE3-NEXT:    pcmpgtb %xmm4, %xmm0
1343; SSSE3-NEXT:    psraw $8, %xmm3
1344; SSSE3-NEXT:    psraw $8, %xmm6
1345; SSSE3-NEXT:    pmullw %xmm3, %xmm6
1346; SSSE3-NEXT:    psrlw $8, %xmm6
1347; SSSE3-NEXT:    psraw $8, %xmm5
1348; SSSE3-NEXT:    psraw $8, %xmm7
1349; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1350; SSSE3-NEXT:    psrlw $8, %xmm7
1351; SSSE3-NEXT:    packuswb %xmm6, %xmm7
1352; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm7
1353; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1354; SSSE3-NEXT:    pxor %xmm7, %xmm3
1355; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1356; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1357; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1358; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1359; SSSE3-NEXT:    pslld $31, %xmm0
1360; SSSE3-NEXT:    psrad $31, %xmm0
1361; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1362; SSSE3-NEXT:    pslld $31, %xmm1
1363; SSSE3-NEXT:    psrad $31, %xmm1
1364; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1365; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1366; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1367; SSSE3-NEXT:    pslld $31, %xmm2
1368; SSSE3-NEXT:    psrad $31, %xmm2
1369; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1370; SSSE3-NEXT:    pslld $31, %xmm3
1371; SSSE3-NEXT:    psrad $31, %xmm3
1372; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1373; SSSE3-NEXT:    retq
1374;
1375; SSE41-LABEL: smulo_v16i8:
1376; SSE41:       # %bb.0:
1377; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1378; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
1379; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
1380; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1381; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1382; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
1383; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
1384; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1385; SSE41-NEXT:    pmullw %xmm1, %xmm0
1386; SSE41-NEXT:    pmullw %xmm2, %xmm4
1387; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1388; SSE41-NEXT:    pand %xmm1, %xmm0
1389; SSE41-NEXT:    pand %xmm1, %xmm4
1390; SSE41-NEXT:    packuswb %xmm0, %xmm4
1391; SSE41-NEXT:    pxor %xmm0, %xmm0
1392; SSE41-NEXT:    pcmpgtb %xmm4, %xmm0
1393; SSE41-NEXT:    pmullw %xmm3, %xmm6
1394; SSE41-NEXT:    psrlw $8, %xmm6
1395; SSE41-NEXT:    pmovsxbw %xmm5, %xmm1
1396; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1397; SSE41-NEXT:    pmullw %xmm1, %xmm2
1398; SSE41-NEXT:    psrlw $8, %xmm2
1399; SSE41-NEXT:    packuswb %xmm2, %xmm6
1400; SSE41-NEXT:    pcmpeqb %xmm0, %xmm6
1401; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1402; SSE41-NEXT:    pxor %xmm6, %xmm3
1403; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1404; SSE41-NEXT:    pslld $31, %xmm0
1405; SSE41-NEXT:    psrad $31, %xmm0
1406; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
1407; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1408; SSE41-NEXT:    pslld $31, %xmm1
1409; SSE41-NEXT:    psrad $31, %xmm1
1410; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1411; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1412; SSE41-NEXT:    pslld $31, %xmm2
1413; SSE41-NEXT:    psrad $31, %xmm2
1414; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1415; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1416; SSE41-NEXT:    pslld $31, %xmm3
1417; SSE41-NEXT:    psrad $31, %xmm3
1418; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1419; SSE41-NEXT:    retq
1420;
1421; AVX1-LABEL: smulo_v16i8:
1422; AVX1:       # %bb.0:
1423; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1424; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1425; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1426; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1427; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1428; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1429; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1430; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1431; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1432; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1433; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1434; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1435; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm4
1436; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
1437; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1438; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1439; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1440; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1441; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1442; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1443; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1444; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1445; AVX1-NEXT:    vpackuswb %xmm0, %xmm4, %xmm0
1446; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1447; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1448; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1449; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1450; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
1451; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1452; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1453; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1454; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1455; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1456; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1457; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1458; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1459; AVX1-NEXT:    retq
1460;
1461; AVX2-LABEL: smulo_v16i8:
1462; AVX2:       # %bb.0:
1463; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm2
1464; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm3
1465; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1466; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1467; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
1468; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
1469; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1470; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1471; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1472; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1473; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1474; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm3
1475; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
1476; AVX2-NEXT:    vpcmpgtb %xmm3, %xmm0, %xmm0
1477; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
1478; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1479; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1480; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1481; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1482; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1483; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
1484; AVX2-NEXT:    retq
1485;
1486; AVX512-LABEL: smulo_v16i8:
1487; AVX512:       # %bb.0:
1488; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1489; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1490; AVX512-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1491; AVX512-NEXT:    vpmovwb %ymm2, %xmm2
1492; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1493; AVX512-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1494; AVX512-NEXT:    vpmovsxbw %xmm1, %ymm1
1495; AVX512-NEXT:    vpmovsxbw %xmm0, %ymm0
1496; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1497; AVX512-NEXT:    vpsrlw $8, %ymm0, %ymm0
1498; AVX512-NEXT:    vpmovwb %ymm0, %xmm0
1499; AVX512-NEXT:    vpcmpneqb %xmm3, %xmm0, %k1
1500; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1501; AVX512-NEXT:    vmovdqa %xmm2, (%rdi)
1502; AVX512-NEXT:    retq
1503  %t = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1504  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1505  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1506  %res = sext <16 x i1> %obit to <16 x i32>
1507  store <16 x i8> %val, <16 x i8>* %p2
1508  ret <16 x i32> %res
1509}
1510
1511define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nounwind {
1512; SSE2-LABEL: smulo_v32i8:
1513; SSE2:       # %bb.0:
1514; SSE2-NEXT:    movq %rdi, %rax
1515; SSE2-NEXT:    movdqa %xmm3, %xmm10
1516; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1517; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
1518; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1519; SSE2-NEXT:    movdqa %xmm1, %xmm9
1520; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1521; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1522; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1523; SSE2-NEXT:    pmullw %xmm3, %xmm1
1524; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1525; SSE2-NEXT:    pand %xmm8, %xmm1
1526; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1527; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1528; SSE2-NEXT:    pmullw %xmm10, %xmm9
1529; SSE2-NEXT:    pand %xmm8, %xmm9
1530; SSE2-NEXT:    packuswb %xmm1, %xmm9
1531; SSE2-NEXT:    pxor %xmm1, %xmm1
1532; SSE2-NEXT:    pcmpgtb %xmm9, %xmm1
1533; SSE2-NEXT:    psraw $8, %xmm7
1534; SSE2-NEXT:    psraw $8, %xmm6
1535; SSE2-NEXT:    pmullw %xmm7, %xmm6
1536; SSE2-NEXT:    psrlw $8, %xmm6
1537; SSE2-NEXT:    psraw $8, %xmm4
1538; SSE2-NEXT:    psraw $8, %xmm5
1539; SSE2-NEXT:    pmullw %xmm4, %xmm5
1540; SSE2-NEXT:    psrlw $8, %xmm5
1541; SSE2-NEXT:    packuswb %xmm6, %xmm5
1542; SSE2-NEXT:    pcmpeqb %xmm1, %xmm5
1543; SSE2-NEXT:    movdqa %xmm2, %xmm10
1544; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
1545; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1546; SSE2-NEXT:    movdqa %xmm2, %xmm7
1547; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1548; SSE2-NEXT:    movdqa %xmm0, %xmm2
1549; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1550; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1551; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1552; SSE2-NEXT:    pmullw %xmm7, %xmm0
1553; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1554; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1555; SSE2-NEXT:    pmullw %xmm10, %xmm2
1556; SSE2-NEXT:    pxor %xmm7, %xmm7
1557; SSE2-NEXT:    pand %xmm8, %xmm0
1558; SSE2-NEXT:    pand %xmm8, %xmm2
1559; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
1560; SSE2-NEXT:    pxor %xmm8, %xmm5
1561; SSE2-NEXT:    packuswb %xmm0, %xmm2
1562; SSE2-NEXT:    pcmpgtb %xmm2, %xmm7
1563; SSE2-NEXT:    psraw $8, %xmm4
1564; SSE2-NEXT:    psraw $8, %xmm3
1565; SSE2-NEXT:    pmullw %xmm4, %xmm3
1566; SSE2-NEXT:    psrlw $8, %xmm3
1567; SSE2-NEXT:    psraw $8, %xmm6
1568; SSE2-NEXT:    psraw $8, %xmm1
1569; SSE2-NEXT:    pmullw %xmm6, %xmm1
1570; SSE2-NEXT:    psrlw $8, %xmm1
1571; SSE2-NEXT:    packuswb %xmm3, %xmm1
1572; SSE2-NEXT:    pcmpeqb %xmm7, %xmm1
1573; SSE2-NEXT:    pxor %xmm8, %xmm1
1574; SSE2-NEXT:    movdqa %xmm1, %xmm0
1575; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1576; SSE2-NEXT:    movdqa %xmm0, %xmm8
1577; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
1578; SSE2-NEXT:    pslld $31, %xmm8
1579; SSE2-NEXT:    psrad $31, %xmm8
1580; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1581; SSE2-NEXT:    pslld $31, %xmm0
1582; SSE2-NEXT:    psrad $31, %xmm0
1583; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1584; SSE2-NEXT:    movdqa %xmm1, %xmm6
1585; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1586; SSE2-NEXT:    pslld $31, %xmm6
1587; SSE2-NEXT:    psrad $31, %xmm6
1588; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1589; SSE2-NEXT:    pslld $31, %xmm1
1590; SSE2-NEXT:    psrad $31, %xmm1
1591; SSE2-NEXT:    movdqa %xmm5, %xmm3
1592; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1593; SSE2-NEXT:    movdqa %xmm3, %xmm7
1594; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1595; SSE2-NEXT:    pslld $31, %xmm7
1596; SSE2-NEXT:    psrad $31, %xmm7
1597; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1598; SSE2-NEXT:    pslld $31, %xmm3
1599; SSE2-NEXT:    psrad $31, %xmm3
1600; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1601; SSE2-NEXT:    movdqa %xmm5, %xmm4
1602; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1603; SSE2-NEXT:    pslld $31, %xmm4
1604; SSE2-NEXT:    psrad $31, %xmm4
1605; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1606; SSE2-NEXT:    pslld $31, %xmm5
1607; SSE2-NEXT:    psrad $31, %xmm5
1608; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
1609; SSE2-NEXT:    movdqa %xmm2, (%rsi)
1610; SSE2-NEXT:    movdqa %xmm5, 112(%rdi)
1611; SSE2-NEXT:    movdqa %xmm4, 96(%rdi)
1612; SSE2-NEXT:    movdqa %xmm3, 80(%rdi)
1613; SSE2-NEXT:    movdqa %xmm7, 64(%rdi)
1614; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
1615; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
1616; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
1617; SSE2-NEXT:    movdqa %xmm8, (%rdi)
1618; SSE2-NEXT:    retq
1619;
1620; SSSE3-LABEL: smulo_v32i8:
1621; SSSE3:       # %bb.0:
1622; SSSE3-NEXT:    movq %rdi, %rax
1623; SSSE3-NEXT:    movdqa %xmm3, %xmm10
1624; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1625; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
1626; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1627; SSSE3-NEXT:    movdqa %xmm1, %xmm9
1628; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
1629; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1630; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1631; SSSE3-NEXT:    pmullw %xmm3, %xmm1
1632; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1633; SSSE3-NEXT:    pand %xmm8, %xmm1
1634; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1635; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1636; SSSE3-NEXT:    pmullw %xmm10, %xmm9
1637; SSSE3-NEXT:    pand %xmm8, %xmm9
1638; SSSE3-NEXT:    packuswb %xmm1, %xmm9
1639; SSSE3-NEXT:    pxor %xmm1, %xmm1
1640; SSSE3-NEXT:    pcmpgtb %xmm9, %xmm1
1641; SSSE3-NEXT:    psraw $8, %xmm7
1642; SSSE3-NEXT:    psraw $8, %xmm6
1643; SSSE3-NEXT:    pmullw %xmm7, %xmm6
1644; SSSE3-NEXT:    psrlw $8, %xmm6
1645; SSSE3-NEXT:    psraw $8, %xmm4
1646; SSSE3-NEXT:    psraw $8, %xmm5
1647; SSSE3-NEXT:    pmullw %xmm4, %xmm5
1648; SSSE3-NEXT:    psrlw $8, %xmm5
1649; SSSE3-NEXT:    packuswb %xmm6, %xmm5
1650; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm5
1651; SSSE3-NEXT:    movdqa %xmm2, %xmm10
1652; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
1653; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
1654; SSSE3-NEXT:    movdqa %xmm2, %xmm7
1655; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1656; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1657; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1658; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1659; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1660; SSSE3-NEXT:    pmullw %xmm7, %xmm0
1661; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1662; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1663; SSSE3-NEXT:    pmullw %xmm10, %xmm2
1664; SSSE3-NEXT:    pxor %xmm7, %xmm7
1665; SSSE3-NEXT:    pand %xmm8, %xmm0
1666; SSSE3-NEXT:    pand %xmm8, %xmm2
1667; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
1668; SSSE3-NEXT:    pxor %xmm8, %xmm5
1669; SSSE3-NEXT:    packuswb %xmm0, %xmm2
1670; SSSE3-NEXT:    pcmpgtb %xmm2, %xmm7
1671; SSSE3-NEXT:    psraw $8, %xmm4
1672; SSSE3-NEXT:    psraw $8, %xmm3
1673; SSSE3-NEXT:    pmullw %xmm4, %xmm3
1674; SSSE3-NEXT:    psrlw $8, %xmm3
1675; SSSE3-NEXT:    psraw $8, %xmm6
1676; SSSE3-NEXT:    psraw $8, %xmm1
1677; SSSE3-NEXT:    pmullw %xmm6, %xmm1
1678; SSSE3-NEXT:    psrlw $8, %xmm1
1679; SSSE3-NEXT:    packuswb %xmm3, %xmm1
1680; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm1
1681; SSSE3-NEXT:    pxor %xmm8, %xmm1
1682; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1683; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1684; SSSE3-NEXT:    movdqa %xmm0, %xmm8
1685; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
1686; SSSE3-NEXT:    pslld $31, %xmm8
1687; SSSE3-NEXT:    psrad $31, %xmm8
1688; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1689; SSSE3-NEXT:    pslld $31, %xmm0
1690; SSSE3-NEXT:    psrad $31, %xmm0
1691; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1692; SSSE3-NEXT:    movdqa %xmm1, %xmm6
1693; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1694; SSSE3-NEXT:    pslld $31, %xmm6
1695; SSSE3-NEXT:    psrad $31, %xmm6
1696; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1697; SSSE3-NEXT:    pslld $31, %xmm1
1698; SSSE3-NEXT:    psrad $31, %xmm1
1699; SSSE3-NEXT:    movdqa %xmm5, %xmm3
1700; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1701; SSSE3-NEXT:    movdqa %xmm3, %xmm7
1702; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1703; SSSE3-NEXT:    pslld $31, %xmm7
1704; SSSE3-NEXT:    psrad $31, %xmm7
1705; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1706; SSSE3-NEXT:    pslld $31, %xmm3
1707; SSSE3-NEXT:    psrad $31, %xmm3
1708; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1709; SSSE3-NEXT:    movdqa %xmm5, %xmm4
1710; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1711; SSSE3-NEXT:    pslld $31, %xmm4
1712; SSSE3-NEXT:    psrad $31, %xmm4
1713; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1714; SSSE3-NEXT:    pslld $31, %xmm5
1715; SSSE3-NEXT:    psrad $31, %xmm5
1716; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
1717; SSSE3-NEXT:    movdqa %xmm2, (%rsi)
1718; SSSE3-NEXT:    movdqa %xmm5, 112(%rdi)
1719; SSSE3-NEXT:    movdqa %xmm4, 96(%rdi)
1720; SSSE3-NEXT:    movdqa %xmm3, 80(%rdi)
1721; SSSE3-NEXT:    movdqa %xmm7, 64(%rdi)
1722; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
1723; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
1724; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
1725; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
1726; SSSE3-NEXT:    retq
1727;
1728; SSE41-LABEL: smulo_v32i8:
1729; SSE41:       # %bb.0:
1730; SSE41-NEXT:    movq %rdi, %rax
1731; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1732; SSE41-NEXT:    pmovsxbw %xmm3, %xmm7
1733; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
1734; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1735; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1736; SSE41-NEXT:    pmovsxbw %xmm1, %xmm4
1737; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
1738; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1739; SSE41-NEXT:    pmullw %xmm3, %xmm1
1740; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1741; SSE41-NEXT:    pand %xmm8, %xmm1
1742; SSE41-NEXT:    pmullw %xmm10, %xmm9
1743; SSE41-NEXT:    pand %xmm8, %xmm9
1744; SSE41-NEXT:    packuswb %xmm1, %xmm9
1745; SSE41-NEXT:    pxor %xmm1, %xmm1
1746; SSE41-NEXT:    pcmpgtb %xmm9, %xmm1
1747; SSE41-NEXT:    pmullw %xmm7, %xmm4
1748; SSE41-NEXT:    psrlw $8, %xmm4
1749; SSE41-NEXT:    pmovsxbw %xmm5, %xmm3
1750; SSE41-NEXT:    pmovsxbw %xmm6, %xmm5
1751; SSE41-NEXT:    pmullw %xmm3, %xmm5
1752; SSE41-NEXT:    psrlw $8, %xmm5
1753; SSE41-NEXT:    packuswb %xmm5, %xmm4
1754; SSE41-NEXT:    pcmpeqb %xmm1, %xmm4
1755; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1756; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
1757; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
1758; SSE41-NEXT:    movdqa %xmm2, %xmm7
1759; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1760; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1761; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
1762; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
1763; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1764; SSE41-NEXT:    pmullw %xmm7, %xmm0
1765; SSE41-NEXT:    pmullw %xmm10, %xmm1
1766; SSE41-NEXT:    pxor %xmm7, %xmm7
1767; SSE41-NEXT:    pand %xmm8, %xmm0
1768; SSE41-NEXT:    pand %xmm8, %xmm1
1769; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1770; SSE41-NEXT:    pxor %xmm8, %xmm4
1771; SSE41-NEXT:    packuswb %xmm0, %xmm1
1772; SSE41-NEXT:    pcmpgtb %xmm1, %xmm7
1773; SSE41-NEXT:    pmullw %xmm5, %xmm2
1774; SSE41-NEXT:    psrlw $8, %xmm2
1775; SSE41-NEXT:    pmovsxbw %xmm6, %xmm0
1776; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
1777; SSE41-NEXT:    pmullw %xmm0, %xmm3
1778; SSE41-NEXT:    psrlw $8, %xmm3
1779; SSE41-NEXT:    packuswb %xmm3, %xmm2
1780; SSE41-NEXT:    pcmpeqb %xmm7, %xmm2
1781; SSE41-NEXT:    pxor %xmm8, %xmm2
1782; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
1783; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1784; SSE41-NEXT:    pslld $31, %xmm8
1785; SSE41-NEXT:    psrad $31, %xmm8
1786; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
1787; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1788; SSE41-NEXT:    pslld $31, %xmm5
1789; SSE41-NEXT:    psrad $31, %xmm5
1790; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
1791; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1792; SSE41-NEXT:    pslld $31, %xmm6
1793; SSE41-NEXT:    psrad $31, %xmm6
1794; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
1795; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1796; SSE41-NEXT:    pslld $31, %xmm7
1797; SSE41-NEXT:    psrad $31, %xmm7
1798; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
1799; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1800; SSE41-NEXT:    pslld $31, %xmm3
1801; SSE41-NEXT:    psrad $31, %xmm3
1802; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
1803; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1804; SSE41-NEXT:    pslld $31, %xmm0
1805; SSE41-NEXT:    psrad $31, %xmm0
1806; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1807; SSE41-NEXT:    pslld $31, %xmm2
1808; SSE41-NEXT:    psrad $31, %xmm2
1809; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
1810; SSE41-NEXT:    pslld $31, %xmm4
1811; SSE41-NEXT:    psrad $31, %xmm4
1812; SSE41-NEXT:    movdqa %xmm9, 16(%rsi)
1813; SSE41-NEXT:    movdqa %xmm1, (%rsi)
1814; SSE41-NEXT:    movdqa %xmm4, 64(%rdi)
1815; SSE41-NEXT:    movdqa %xmm2, (%rdi)
1816; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
1817; SSE41-NEXT:    movdqa %xmm3, 96(%rdi)
1818; SSE41-NEXT:    movdqa %xmm7, 80(%rdi)
1819; SSE41-NEXT:    movdqa %xmm6, 48(%rdi)
1820; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)
1821; SSE41-NEXT:    movdqa %xmm8, 16(%rdi)
1822; SSE41-NEXT:    retq
1823;
1824; AVX1-LABEL: smulo_v32i8:
1825; AVX1:       # %bb.0:
1826; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1827; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1828; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
1829; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1830; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
1831; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
1832; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
1833; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1834; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
1835; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
1836; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
1837; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm9
1838; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
1839; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm8, %xmm3
1840; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm7
1841; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
1842; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
1843; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1844; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
1845; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
1846; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
1847; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
1848; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1849; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1850; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
1851; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
1852; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1853; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm3
1854; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1855; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1856; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
1857; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
1858; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1859; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1860; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
1861; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
1862; AVX1-NEXT:    vpackuswb %xmm2, %xmm5, %xmm5
1863; AVX1-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
1864; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm6
1865; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm7
1866; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
1867; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
1868; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1869; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1870; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1871; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1872; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1873; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1874; AVX1-NEXT:    vpackuswb %xmm0, %xmm6, %xmm0
1875; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
1876; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
1877; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1878; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1879; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
1880; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1881; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm2
1882; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
1883; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1884; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
1885; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
1886; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1887; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1888; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1889; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
1890; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
1891; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1892; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1893; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1894; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1895; AVX1-NEXT:    vmovdqa %xmm9, 16(%rdi)
1896; AVX1-NEXT:    vmovdqa %xmm5, (%rdi)
1897; AVX1-NEXT:    retq
1898;
1899; AVX2-LABEL: smulo_v32i8:
1900; AVX2:       # %bb.0:
1901; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1902; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1903; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1904; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1905; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
1906; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1907; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1908; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1909; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
1910; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
1911; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1912; AVX2-NEXT:    vpcmpgtb %ymm4, %ymm2, %ymm2
1913; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1914; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
1915; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1916; AVX2-NEXT:    vpsraw $8, %ymm5, %ymm5
1917; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
1918; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
1919; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1920; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
1921; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1922; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
1923; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1924; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1925; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
1926; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
1927; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
1928; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
1929; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1930; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1931; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
1932; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1933; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1934; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1935; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
1936; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
1937; AVX2-NEXT:    retq
1938;
1939; AVX512-LABEL: smulo_v32i8:
1940; AVX512:       # %bb.0:
1941; AVX512-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
1942; AVX512-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1943; AVX512-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
1944; AVX512-NEXT:    vpmovwb %zmm2, %ymm2
1945; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1946; AVX512-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm3
1947; AVX512-NEXT:    vpmovsxbw %ymm1, %zmm1
1948; AVX512-NEXT:    vpmovsxbw %ymm0, %zmm0
1949; AVX512-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1950; AVX512-NEXT:    vpsrlw $8, %zmm0, %zmm0
1951; AVX512-NEXT:    vpmovwb %zmm0, %ymm0
1952; AVX512-NEXT:    vpcmpneqb %ymm3, %ymm0, %k1
1953; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1954; AVX512-NEXT:    kshiftrd $16, %k1, %k1
1955; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1956; AVX512-NEXT:    vmovdqa %ymm2, (%rdi)
1957; AVX512-NEXT:    retq
1958  %t = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)
1959  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 0
1960  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 1
1961  %res = sext <32 x i1> %obit to <32 x i32>
1962  store <32 x i8> %val, <32 x i8>* %p2
1963  ret <32 x i32> %res
1964}
1965
1966define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nounwind {
1967; SSE2-LABEL: smulo_v64i8:
1968; SSE2:       # %bb.0:
1969; SSE2-NEXT:    movq %rdi, %rax
1970; SSE2-NEXT:    movdqa %xmm7, %xmm9
1971; SSE2-NEXT:    movdqa %xmm7, %xmm10
1972; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1973; SSE2-NEXT:    movdqa %xmm3, %xmm8
1974; SSE2-NEXT:    movdqa %xmm3, %xmm11
1975; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1976; SSE2-NEXT:    pmullw %xmm10, %xmm11
1977; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
1978; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1979; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1980; SSE2-NEXT:    pmullw %xmm9, %xmm8
1981; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
1982; SSE2-NEXT:    pand %xmm9, %xmm11
1983; SSE2-NEXT:    pand %xmm9, %xmm8
1984; SSE2-NEXT:    packuswb %xmm11, %xmm8
1985; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
1986; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1987; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
1988; SSE2-NEXT:    psraw $8, %xmm10
1989; SSE2-NEXT:    psraw $8, %xmm11
1990; SSE2-NEXT:    pmullw %xmm10, %xmm11
1991; SSE2-NEXT:    pxor %xmm3, %xmm3
1992; SSE2-NEXT:    pcmpgtb %xmm8, %xmm3
1993; SSE2-NEXT:    psrlw $8, %xmm11
1994; SSE2-NEXT:    psraw $8, %xmm7
1995; SSE2-NEXT:    psraw $8, %xmm12
1996; SSE2-NEXT:    pmullw %xmm7, %xmm12
1997; SSE2-NEXT:    psrlw $8, %xmm12
1998; SSE2-NEXT:    packuswb %xmm11, %xmm12
1999; SSE2-NEXT:    pcmpeqb %xmm3, %xmm12
2000; SSE2-NEXT:    movdqa %xmm6, %xmm10
2001; SSE2-NEXT:    movdqa %xmm6, %xmm7
2002; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2003; SSE2-NEXT:    movdqa %xmm2, %xmm3
2004; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2005; SSE2-NEXT:    pmullw %xmm7, %xmm3
2006; SSE2-NEXT:    movdqa %xmm2, %xmm11
2007; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2008; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2009; SSE2-NEXT:    pmullw %xmm10, %xmm11
2010; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2011; SSE2-NEXT:    pand %xmm9, %xmm3
2012; SSE2-NEXT:    pand %xmm9, %xmm11
2013; SSE2-NEXT:    packuswb %xmm3, %xmm11
2014; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2015; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2016; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2017; SSE2-NEXT:    psraw $8, %xmm7
2018; SSE2-NEXT:    psraw $8, %xmm3
2019; SSE2-NEXT:    pmullw %xmm7, %xmm3
2020; SSE2-NEXT:    pxor %xmm7, %xmm7
2021; SSE2-NEXT:    pcmpgtb %xmm11, %xmm7
2022; SSE2-NEXT:    psrlw $8, %xmm3
2023; SSE2-NEXT:    psraw $8, %xmm6
2024; SSE2-NEXT:    psraw $8, %xmm2
2025; SSE2-NEXT:    pmullw %xmm6, %xmm2
2026; SSE2-NEXT:    psrlw $8, %xmm2
2027; SSE2-NEXT:    packuswb %xmm3, %xmm2
2028; SSE2-NEXT:    pcmpeqb %xmm7, %xmm2
2029; SSE2-NEXT:    movdqa %xmm5, %xmm3
2030; SSE2-NEXT:    movdqa %xmm5, %xmm6
2031; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2032; SSE2-NEXT:    movdqa %xmm1, %xmm7
2033; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2034; SSE2-NEXT:    pmullw %xmm6, %xmm7
2035; SSE2-NEXT:    movdqa %xmm1, %xmm6
2036; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2037; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2038; SSE2-NEXT:    pmullw %xmm3, %xmm6
2039; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2040; SSE2-NEXT:    pand %xmm9, %xmm7
2041; SSE2-NEXT:    pand %xmm9, %xmm6
2042; SSE2-NEXT:    packuswb %xmm7, %xmm6
2043; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2044; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2045; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2046; SSE2-NEXT:    psraw $8, %xmm3
2047; SSE2-NEXT:    psraw $8, %xmm7
2048; SSE2-NEXT:    pmullw %xmm3, %xmm7
2049; SSE2-NEXT:    pxor %xmm3, %xmm3
2050; SSE2-NEXT:    pcmpgtb %xmm6, %xmm3
2051; SSE2-NEXT:    psrlw $8, %xmm7
2052; SSE2-NEXT:    psraw $8, %xmm5
2053; SSE2-NEXT:    psraw $8, %xmm1
2054; SSE2-NEXT:    pmullw %xmm5, %xmm1
2055; SSE2-NEXT:    psrlw $8, %xmm1
2056; SSE2-NEXT:    packuswb %xmm7, %xmm1
2057; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
2058; SSE2-NEXT:    movdqa %xmm4, %xmm3
2059; SSE2-NEXT:    movdqa %xmm4, %xmm5
2060; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2061; SSE2-NEXT:    movdqa %xmm0, %xmm7
2062; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2063; SSE2-NEXT:    pmullw %xmm5, %xmm7
2064; SSE2-NEXT:    movdqa %xmm0, %xmm5
2065; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2066; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2067; SSE2-NEXT:    pmullw %xmm3, %xmm5
2068; SSE2-NEXT:    pand %xmm9, %xmm7
2069; SSE2-NEXT:    pand %xmm9, %xmm5
2070; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2071; SSE2-NEXT:    packuswb %xmm7, %xmm5
2072; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2073; SSE2-NEXT:    psraw $8, %xmm3
2074; SSE2-NEXT:    psraw $8, %xmm7
2075; SSE2-NEXT:    pmullw %xmm3, %xmm7
2076; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2077; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2078; SSE2-NEXT:    psraw $8, %xmm3
2079; SSE2-NEXT:    psraw $8, %xmm0
2080; SSE2-NEXT:    pmullw %xmm3, %xmm0
2081; SSE2-NEXT:    psrlw $8, %xmm7
2082; SSE2-NEXT:    psrlw $8, %xmm0
2083; SSE2-NEXT:    packuswb %xmm7, %xmm0
2084; SSE2-NEXT:    pxor %xmm3, %xmm3
2085; SSE2-NEXT:    pcmpgtb %xmm5, %xmm3
2086; SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
2087; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
2088; SSE2-NEXT:    pxor %xmm3, %xmm12
2089; SSE2-NEXT:    pxor %xmm3, %xmm2
2090; SSE2-NEXT:    pxor %xmm3, %xmm1
2091; SSE2-NEXT:    pxor %xmm3, %xmm0
2092; SSE2-NEXT:    movdqa %xmm8, 48(%rsi)
2093; SSE2-NEXT:    movdqa %xmm11, 32(%rsi)
2094; SSE2-NEXT:    movdqa %xmm6, 16(%rsi)
2095; SSE2-NEXT:    movdqa %xmm12, %xmm3
2096; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2097; SSE2-NEXT:    movdqa %xmm5, (%rsi)
2098; SSE2-NEXT:    movdqa %xmm12, %xmm4
2099; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
2100; SSE2-NEXT:    pslld $31, %xmm12
2101; SSE2-NEXT:    psrad $31, %xmm12
2102; SSE2-NEXT:    movdqa %xmm12, 224(%rdi)
2103; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2104; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2105; SSE2-NEXT:    pslld $31, %xmm4
2106; SSE2-NEXT:    psrad $31, %xmm4
2107; SSE2-NEXT:    movdqa %xmm4, 240(%rdi)
2108; SSE2-NEXT:    movdqa %xmm3, %xmm4
2109; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2110; SSE2-NEXT:    pslld $31, %xmm3
2111; SSE2-NEXT:    psrad $31, %xmm3
2112; SSE2-NEXT:    movdqa %xmm3, 192(%rdi)
2113; SSE2-NEXT:    movdqa %xmm2, %xmm3
2114; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2115; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2116; SSE2-NEXT:    pslld $31, %xmm4
2117; SSE2-NEXT:    psrad $31, %xmm4
2118; SSE2-NEXT:    movdqa %xmm4, 208(%rdi)
2119; SSE2-NEXT:    movdqa %xmm2, %xmm4
2120; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2121; SSE2-NEXT:    pslld $31, %xmm2
2122; SSE2-NEXT:    psrad $31, %xmm2
2123; SSE2-NEXT:    movdqa %xmm2, 160(%rdi)
2124; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2125; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2126; SSE2-NEXT:    pslld $31, %xmm4
2127; SSE2-NEXT:    psrad $31, %xmm4
2128; SSE2-NEXT:    movdqa %xmm4, 176(%rdi)
2129; SSE2-NEXT:    movdqa %xmm3, %xmm2
2130; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2131; SSE2-NEXT:    pslld $31, %xmm3
2132; SSE2-NEXT:    psrad $31, %xmm3
2133; SSE2-NEXT:    movdqa %xmm3, 128(%rdi)
2134; SSE2-NEXT:    movdqa %xmm1, %xmm3
2135; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2136; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2137; SSE2-NEXT:    pslld $31, %xmm2
2138; SSE2-NEXT:    psrad $31, %xmm2
2139; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
2140; SSE2-NEXT:    movdqa %xmm1, %xmm2
2141; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2142; SSE2-NEXT:    pslld $31, %xmm1
2143; SSE2-NEXT:    psrad $31, %xmm1
2144; SSE2-NEXT:    movdqa %xmm1, 96(%rdi)
2145; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2146; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2147; SSE2-NEXT:    pslld $31, %xmm2
2148; SSE2-NEXT:    psrad $31, %xmm2
2149; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
2150; SSE2-NEXT:    movdqa %xmm3, %xmm1
2151; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2152; SSE2-NEXT:    pslld $31, %xmm3
2153; SSE2-NEXT:    psrad $31, %xmm3
2154; SSE2-NEXT:    movdqa %xmm3, 64(%rdi)
2155; SSE2-NEXT:    movdqa %xmm0, %xmm2
2156; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2157; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2158; SSE2-NEXT:    pslld $31, %xmm1
2159; SSE2-NEXT:    psrad $31, %xmm1
2160; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
2161; SSE2-NEXT:    movdqa %xmm0, %xmm1
2162; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2163; SSE2-NEXT:    pslld $31, %xmm0
2164; SSE2-NEXT:    psrad $31, %xmm0
2165; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
2166; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2167; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2168; SSE2-NEXT:    pslld $31, %xmm1
2169; SSE2-NEXT:    psrad $31, %xmm1
2170; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
2171; SSE2-NEXT:    movdqa %xmm2, %xmm0
2172; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2173; SSE2-NEXT:    pslld $31, %xmm2
2174; SSE2-NEXT:    psrad $31, %xmm2
2175; SSE2-NEXT:    movdqa %xmm2, (%rdi)
2176; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2177; SSE2-NEXT:    pslld $31, %xmm0
2178; SSE2-NEXT:    psrad $31, %xmm0
2179; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
2180; SSE2-NEXT:    retq
2181;
2182; SSSE3-LABEL: smulo_v64i8:
2183; SSSE3:       # %bb.0:
2184; SSSE3-NEXT:    movq %rdi, %rax
2185; SSSE3-NEXT:    movdqa %xmm7, %xmm9
2186; SSSE3-NEXT:    movdqa %xmm7, %xmm10
2187; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2188; SSSE3-NEXT:    movdqa %xmm3, %xmm8
2189; SSSE3-NEXT:    movdqa %xmm3, %xmm11
2190; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2191; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2192; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2193; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2194; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2195; SSSE3-NEXT:    pmullw %xmm9, %xmm8
2196; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
2197; SSSE3-NEXT:    pand %xmm9, %xmm11
2198; SSSE3-NEXT:    pand %xmm9, %xmm8
2199; SSSE3-NEXT:    packuswb %xmm11, %xmm8
2200; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
2201; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2202; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
2203; SSSE3-NEXT:    psraw $8, %xmm10
2204; SSSE3-NEXT:    psraw $8, %xmm11
2205; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2206; SSSE3-NEXT:    pxor %xmm3, %xmm3
2207; SSSE3-NEXT:    pcmpgtb %xmm8, %xmm3
2208; SSSE3-NEXT:    psrlw $8, %xmm11
2209; SSSE3-NEXT:    psraw $8, %xmm7
2210; SSSE3-NEXT:    psraw $8, %xmm12
2211; SSSE3-NEXT:    pmullw %xmm7, %xmm12
2212; SSSE3-NEXT:    psrlw $8, %xmm12
2213; SSSE3-NEXT:    packuswb %xmm11, %xmm12
2214; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm12
2215; SSSE3-NEXT:    movdqa %xmm6, %xmm10
2216; SSSE3-NEXT:    movdqa %xmm6, %xmm7
2217; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2218; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2219; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2220; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2221; SSSE3-NEXT:    movdqa %xmm2, %xmm11
2222; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2223; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2224; SSSE3-NEXT:    pmullw %xmm10, %xmm11
2225; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
2226; SSSE3-NEXT:    pand %xmm9, %xmm3
2227; SSSE3-NEXT:    pand %xmm9, %xmm11
2228; SSSE3-NEXT:    packuswb %xmm3, %xmm11
2229; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2230; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2231; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2232; SSSE3-NEXT:    psraw $8, %xmm7
2233; SSSE3-NEXT:    psraw $8, %xmm3
2234; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2235; SSSE3-NEXT:    pxor %xmm7, %xmm7
2236; SSSE3-NEXT:    pcmpgtb %xmm11, %xmm7
2237; SSSE3-NEXT:    psrlw $8, %xmm3
2238; SSSE3-NEXT:    psraw $8, %xmm6
2239; SSSE3-NEXT:    psraw $8, %xmm2
2240; SSSE3-NEXT:    pmullw %xmm6, %xmm2
2241; SSSE3-NEXT:    psrlw $8, %xmm2
2242; SSSE3-NEXT:    packuswb %xmm3, %xmm2
2243; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm2
2244; SSSE3-NEXT:    movdqa %xmm5, %xmm3
2245; SSSE3-NEXT:    movdqa %xmm5, %xmm6
2246; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2247; SSSE3-NEXT:    movdqa %xmm1, %xmm7
2248; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2249; SSSE3-NEXT:    pmullw %xmm6, %xmm7
2250; SSSE3-NEXT:    movdqa %xmm1, %xmm6
2251; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2252; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2253; SSSE3-NEXT:    pmullw %xmm3, %xmm6
2254; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
2255; SSSE3-NEXT:    pand %xmm9, %xmm7
2256; SSSE3-NEXT:    pand %xmm9, %xmm6
2257; SSSE3-NEXT:    packuswb %xmm7, %xmm6
2258; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
2259; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2260; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2261; SSSE3-NEXT:    psraw $8, %xmm3
2262; SSSE3-NEXT:    psraw $8, %xmm7
2263; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2264; SSSE3-NEXT:    pxor %xmm3, %xmm3
2265; SSSE3-NEXT:    pcmpgtb %xmm6, %xmm3
2266; SSSE3-NEXT:    psrlw $8, %xmm7
2267; SSSE3-NEXT:    psraw $8, %xmm5
2268; SSSE3-NEXT:    psraw $8, %xmm1
2269; SSSE3-NEXT:    pmullw %xmm5, %xmm1
2270; SSSE3-NEXT:    psrlw $8, %xmm1
2271; SSSE3-NEXT:    packuswb %xmm7, %xmm1
2272; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm1
2273; SSSE3-NEXT:    movdqa %xmm4, %xmm3
2274; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2275; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2276; SSSE3-NEXT:    movdqa %xmm0, %xmm7
2277; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2278; SSSE3-NEXT:    pmullw %xmm5, %xmm7
2279; SSSE3-NEXT:    movdqa %xmm0, %xmm5
2280; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2281; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2282; SSSE3-NEXT:    pmullw %xmm3, %xmm5
2283; SSSE3-NEXT:    pand %xmm9, %xmm7
2284; SSSE3-NEXT:    pand %xmm9, %xmm5
2285; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
2286; SSSE3-NEXT:    packuswb %xmm7, %xmm5
2287; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2288; SSSE3-NEXT:    psraw $8, %xmm3
2289; SSSE3-NEXT:    psraw $8, %xmm7
2290; SSSE3-NEXT:    pmullw %xmm3, %xmm7
2291; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2292; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2293; SSSE3-NEXT:    psraw $8, %xmm3
2294; SSSE3-NEXT:    psraw $8, %xmm0
2295; SSSE3-NEXT:    pmullw %xmm3, %xmm0
2296; SSSE3-NEXT:    psrlw $8, %xmm7
2297; SSSE3-NEXT:    psrlw $8, %xmm0
2298; SSSE3-NEXT:    packuswb %xmm7, %xmm0
2299; SSSE3-NEXT:    pxor %xmm3, %xmm3
2300; SSSE3-NEXT:    pcmpgtb %xmm5, %xmm3
2301; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm0
2302; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
2303; SSSE3-NEXT:    pxor %xmm3, %xmm12
2304; SSSE3-NEXT:    pxor %xmm3, %xmm2
2305; SSSE3-NEXT:    pxor %xmm3, %xmm1
2306; SSSE3-NEXT:    pxor %xmm3, %xmm0
2307; SSSE3-NEXT:    movdqa %xmm8, 48(%rsi)
2308; SSSE3-NEXT:    movdqa %xmm11, 32(%rsi)
2309; SSSE3-NEXT:    movdqa %xmm6, 16(%rsi)
2310; SSSE3-NEXT:    movdqa %xmm12, %xmm3
2311; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2312; SSSE3-NEXT:    movdqa %xmm5, (%rsi)
2313; SSSE3-NEXT:    movdqa %xmm12, %xmm4
2314; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
2315; SSSE3-NEXT:    pslld $31, %xmm12
2316; SSSE3-NEXT:    psrad $31, %xmm12
2317; SSSE3-NEXT:    movdqa %xmm12, 224(%rdi)
2318; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2319; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2320; SSSE3-NEXT:    pslld $31, %xmm4
2321; SSSE3-NEXT:    psrad $31, %xmm4
2322; SSSE3-NEXT:    movdqa %xmm4, 240(%rdi)
2323; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2324; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2325; SSSE3-NEXT:    pslld $31, %xmm3
2326; SSSE3-NEXT:    psrad $31, %xmm3
2327; SSSE3-NEXT:    movdqa %xmm3, 192(%rdi)
2328; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2329; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2330; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2331; SSSE3-NEXT:    pslld $31, %xmm4
2332; SSSE3-NEXT:    psrad $31, %xmm4
2333; SSSE3-NEXT:    movdqa %xmm4, 208(%rdi)
2334; SSSE3-NEXT:    movdqa %xmm2, %xmm4
2335; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2336; SSSE3-NEXT:    pslld $31, %xmm2
2337; SSSE3-NEXT:    psrad $31, %xmm2
2338; SSSE3-NEXT:    movdqa %xmm2, 160(%rdi)
2339; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2340; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
2341; SSSE3-NEXT:    pslld $31, %xmm4
2342; SSSE3-NEXT:    psrad $31, %xmm4
2343; SSSE3-NEXT:    movdqa %xmm4, 176(%rdi)
2344; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2345; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2346; SSSE3-NEXT:    pslld $31, %xmm3
2347; SSSE3-NEXT:    psrad $31, %xmm3
2348; SSSE3-NEXT:    movdqa %xmm3, 128(%rdi)
2349; SSSE3-NEXT:    movdqa %xmm1, %xmm3
2350; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2351; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2352; SSSE3-NEXT:    pslld $31, %xmm2
2353; SSSE3-NEXT:    psrad $31, %xmm2
2354; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
2355; SSSE3-NEXT:    movdqa %xmm1, %xmm2
2356; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2357; SSSE3-NEXT:    pslld $31, %xmm1
2358; SSSE3-NEXT:    psrad $31, %xmm1
2359; SSSE3-NEXT:    movdqa %xmm1, 96(%rdi)
2360; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2361; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2362; SSSE3-NEXT:    pslld $31, %xmm2
2363; SSSE3-NEXT:    psrad $31, %xmm2
2364; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
2365; SSSE3-NEXT:    movdqa %xmm3, %xmm1
2366; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2367; SSSE3-NEXT:    pslld $31, %xmm3
2368; SSSE3-NEXT:    psrad $31, %xmm3
2369; SSSE3-NEXT:    movdqa %xmm3, 64(%rdi)
2370; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2371; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2372; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2373; SSSE3-NEXT:    pslld $31, %xmm1
2374; SSSE3-NEXT:    psrad $31, %xmm1
2375; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
2376; SSSE3-NEXT:    movdqa %xmm0, %xmm1
2377; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2378; SSSE3-NEXT:    pslld $31, %xmm0
2379; SSSE3-NEXT:    psrad $31, %xmm0
2380; SSSE3-NEXT:    movdqa %xmm0, 32(%rdi)
2381; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2382; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2383; SSSE3-NEXT:    pslld $31, %xmm1
2384; SSSE3-NEXT:    psrad $31, %xmm1
2385; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
2386; SSSE3-NEXT:    movdqa %xmm2, %xmm0
2387; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2388; SSSE3-NEXT:    pslld $31, %xmm2
2389; SSSE3-NEXT:    psrad $31, %xmm2
2390; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
2391; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2392; SSSE3-NEXT:    pslld $31, %xmm0
2393; SSSE3-NEXT:    psrad $31, %xmm0
2394; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
2395; SSSE3-NEXT:    retq
2396;
2397; SSE41-LABEL: smulo_v64i8:
2398; SSE41:       # %bb.0:
2399; SSE41-NEXT:    movq %rdi, %rax
2400; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2401; SSE41-NEXT:    movdqa %xmm7, %xmm10
2402; SSE41-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2403; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2404; SSE41-NEXT:    movdqa %xmm3, %xmm11
2405; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2406; SSE41-NEXT:    pmullw %xmm10, %xmm11
2407; SSE41-NEXT:    pmovsxbw %xmm7, %xmm12
2408; SSE41-NEXT:    pmullw %xmm8, %xmm9
2409; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2410; SSE41-NEXT:    pand %xmm10, %xmm11
2411; SSE41-NEXT:    pand %xmm10, %xmm9
2412; SSE41-NEXT:    packuswb %xmm11, %xmm9
2413; SSE41-NEXT:    pmovsxbw %xmm3, %xmm8
2414; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,2,3]
2415; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,2,3]
2416; SSE41-NEXT:    pmullw %xmm12, %xmm8
2417; SSE41-NEXT:    pxor %xmm7, %xmm7
2418; SSE41-NEXT:    pcmpgtb %xmm9, %xmm7
2419; SSE41-NEXT:    psrlw $8, %xmm8
2420; SSE41-NEXT:    pmovsxbw %xmm11, %xmm11
2421; SSE41-NEXT:    pmovsxbw %xmm13, %xmm3
2422; SSE41-NEXT:    pmullw %xmm11, %xmm3
2423; SSE41-NEXT:    psrlw $8, %xmm3
2424; SSE41-NEXT:    packuswb %xmm3, %xmm8
2425; SSE41-NEXT:    pcmpeqb %xmm7, %xmm8
2426; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2427; SSE41-NEXT:    movdqa %xmm6, %xmm7
2428; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2429; SSE41-NEXT:    movdqa %xmm2, %xmm3
2430; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2431; SSE41-NEXT:    pmullw %xmm7, %xmm3
2432; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2433; SSE41-NEXT:    pmullw %xmm11, %xmm12
2434; SSE41-NEXT:    pmovsxbw %xmm6, %xmm7
2435; SSE41-NEXT:    pand %xmm10, %xmm3
2436; SSE41-NEXT:    pand %xmm10, %xmm12
2437; SSE41-NEXT:    packuswb %xmm3, %xmm12
2438; SSE41-NEXT:    pmovsxbw %xmm2, %xmm3
2439; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2440; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2441; SSE41-NEXT:    pmullw %xmm7, %xmm3
2442; SSE41-NEXT:    pxor %xmm7, %xmm7
2443; SSE41-NEXT:    pcmpgtb %xmm12, %xmm7
2444; SSE41-NEXT:    psrlw $8, %xmm3
2445; SSE41-NEXT:    pmovsxbw %xmm6, %xmm6
2446; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
2447; SSE41-NEXT:    pmullw %xmm6, %xmm2
2448; SSE41-NEXT:    psrlw $8, %xmm2
2449; SSE41-NEXT:    packuswb %xmm2, %xmm3
2450; SSE41-NEXT:    pcmpeqb %xmm7, %xmm3
2451; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2452; SSE41-NEXT:    movdqa %xmm5, %xmm6
2453; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2454; SSE41-NEXT:    movdqa %xmm1, %xmm7
2455; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2456; SSE41-NEXT:    pmullw %xmm6, %xmm7
2457; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2458; SSE41-NEXT:    pmullw %xmm2, %xmm11
2459; SSE41-NEXT:    pmovsxbw %xmm5, %xmm6
2460; SSE41-NEXT:    pand %xmm10, %xmm7
2461; SSE41-NEXT:    pand %xmm10, %xmm11
2462; SSE41-NEXT:    packuswb %xmm7, %xmm11
2463; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
2464; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2465; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2466; SSE41-NEXT:    pmullw %xmm6, %xmm2
2467; SSE41-NEXT:    pxor %xmm6, %xmm6
2468; SSE41-NEXT:    pcmpgtb %xmm11, %xmm6
2469; SSE41-NEXT:    psrlw $8, %xmm2
2470; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
2471; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
2472; SSE41-NEXT:    pmullw %xmm5, %xmm1
2473; SSE41-NEXT:    psrlw $8, %xmm1
2474; SSE41-NEXT:    packuswb %xmm1, %xmm2
2475; SSE41-NEXT:    pcmpeqb %xmm6, %xmm2
2476; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2477; SSE41-NEXT:    movdqa %xmm4, %xmm5
2478; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2479; SSE41-NEXT:    movdqa %xmm0, %xmm6
2480; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2481; SSE41-NEXT:    pmullw %xmm5, %xmm6
2482; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2483; SSE41-NEXT:    pmullw %xmm1, %xmm5
2484; SSE41-NEXT:    pand %xmm10, %xmm6
2485; SSE41-NEXT:    pand %xmm10, %xmm5
2486; SSE41-NEXT:    pmovsxbw %xmm4, %xmm7
2487; SSE41-NEXT:    packuswb %xmm6, %xmm5
2488; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
2489; SSE41-NEXT:    pmullw %xmm7, %xmm1
2490; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2491; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2492; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
2493; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
2494; SSE41-NEXT:    pmullw %xmm4, %xmm0
2495; SSE41-NEXT:    psrlw $8, %xmm1
2496; SSE41-NEXT:    psrlw $8, %xmm0
2497; SSE41-NEXT:    packuswb %xmm0, %xmm1
2498; SSE41-NEXT:    pxor %xmm0, %xmm0
2499; SSE41-NEXT:    pcmpgtb %xmm5, %xmm0
2500; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
2501; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2502; SSE41-NEXT:    pxor %xmm0, %xmm8
2503; SSE41-NEXT:    pxor %xmm0, %xmm3
2504; SSE41-NEXT:    pxor %xmm0, %xmm2
2505; SSE41-NEXT:    pxor %xmm0, %xmm1
2506; SSE41-NEXT:    movdqa %xmm9, 48(%rsi)
2507; SSE41-NEXT:    movdqa %xmm12, 32(%rsi)
2508; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)
2509; SSE41-NEXT:    movdqa %xmm5, (%rsi)
2510; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
2511; SSE41-NEXT:    pslld $31, %xmm0
2512; SSE41-NEXT:    psrad $31, %xmm0
2513; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
2514; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
2515; SSE41-NEXT:    pslld $31, %xmm0
2516; SSE41-NEXT:    psrad $31, %xmm0
2517; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
2518; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
2519; SSE41-NEXT:    pslld $31, %xmm0
2520; SSE41-NEXT:    psrad $31, %xmm0
2521; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
2522; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
2523; SSE41-NEXT:    pslld $31, %xmm0
2524; SSE41-NEXT:    psrad $31, %xmm0
2525; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2526; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
2527; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2528; SSE41-NEXT:    pslld $31, %xmm0
2529; SSE41-NEXT:    psrad $31, %xmm0
2530; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
2531; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
2532; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2533; SSE41-NEXT:    pslld $31, %xmm0
2534; SSE41-NEXT:    psrad $31, %xmm0
2535; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
2536; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
2537; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2538; SSE41-NEXT:    pslld $31, %xmm0
2539; SSE41-NEXT:    psrad $31, %xmm0
2540; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
2541; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
2542; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2543; SSE41-NEXT:    pslld $31, %xmm0
2544; SSE41-NEXT:    psrad $31, %xmm0
2545; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
2546; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
2547; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2548; SSE41-NEXT:    pslld $31, %xmm0
2549; SSE41-NEXT:    psrad $31, %xmm0
2550; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
2551; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
2552; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2553; SSE41-NEXT:    pslld $31, %xmm0
2554; SSE41-NEXT:    psrad $31, %xmm0
2555; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
2556; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
2557; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2558; SSE41-NEXT:    pslld $31, %xmm0
2559; SSE41-NEXT:    psrad $31, %xmm0
2560; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
2561; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
2562; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2563; SSE41-NEXT:    pslld $31, %xmm0
2564; SSE41-NEXT:    psrad $31, %xmm0
2565; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
2566; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
2567; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2568; SSE41-NEXT:    pslld $31, %xmm0
2569; SSE41-NEXT:    psrad $31, %xmm0
2570; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
2571; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
2572; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2573; SSE41-NEXT:    pslld $31, %xmm0
2574; SSE41-NEXT:    psrad $31, %xmm0
2575; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
2576; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
2577; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2578; SSE41-NEXT:    pslld $31, %xmm0
2579; SSE41-NEXT:    psrad $31, %xmm0
2580; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
2581; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
2582; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2583; SSE41-NEXT:    pslld $31, %xmm0
2584; SSE41-NEXT:    psrad $31, %xmm0
2585; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
2586; SSE41-NEXT:    retq
2587;
2588; AVX1-LABEL: smulo_v64i8:
2589; AVX1:       # %bb.0:
2590; AVX1-NEXT:    movq %rdi, %rax
2591; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
2592; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2593; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
2594; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2595; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2596; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2597; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm8
2598; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2599; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2600; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
2601; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2602; AVX1-NEXT:    vpackuswb %xmm8, %xmm4, %xmm9
2603; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
2604; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm11, %xmm8
2605; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
2606; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2607; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
2608; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2609; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
2610; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
2611; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2612; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2613; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2614; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2615; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
2616; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm8, %xmm8
2617; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2618; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2619; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2620; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
2621; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2622; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2623; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2624; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2625; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm12
2626; AVX1-NEXT:    vpcmpgtb %xmm12, %xmm11, %xmm4
2627; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm6
2628; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm7
2629; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2630; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
2631; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
2632; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
2633; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2634; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
2635; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2636; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2637; AVX1-NEXT:    vpackuswb %xmm1, %xmm6, %xmm1
2638; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm3
2639; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
2640; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2641; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
2642; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2643; AVX1-NEXT:    vpmullw %xmm1, %xmm7, %xmm1
2644; AVX1-NEXT:    vpand %xmm1, %xmm10, %xmm1
2645; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2646; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2647; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
2648; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2649; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
2650; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm5
2651; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
2652; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
2653; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm11, %xmm7
2654; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
2655; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
2656; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
2657; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
2658; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
2659; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
2660; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2661; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
2662; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm4
2663; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2664; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2665; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2666; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2667; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2668; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
2669; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
2670; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
2671; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm7
2672; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm5
2673; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm6
2674; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2675; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2676; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
2677; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2678; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
2679; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
2680; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm2
2681; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
2682; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
2683; AVX1-NEXT:    vpcmpgtb %xmm7, %xmm11, %xmm2
2684; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2685; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
2686; AVX1-NEXT:    vpxor %xmm5, %xmm8, %xmm6
2687; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
2688; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
2689; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
2690; AVX1-NEXT:    vmovdqa %xmm9, 48(%rsi)
2691; AVX1-NEXT:    vmovdqa %xmm12, 32(%rsi)
2692; AVX1-NEXT:    vmovdqa %xmm1, 16(%rsi)
2693; AVX1-NEXT:    vmovdqa %xmm7, (%rsi)
2694; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
2695; AVX1-NEXT:    vmovdqa %xmm1, 192(%rdi)
2696; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm1
2697; AVX1-NEXT:    vmovdqa %xmm1, 128(%rdi)
2698; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm1
2699; AVX1-NEXT:    vmovdqa %xmm1, 64(%rdi)
2700; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
2701; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
2702; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
2703; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2704; AVX1-NEXT:    vmovdqa %xmm1, 224(%rdi)
2705; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
2706; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2707; AVX1-NEXT:    vmovdqa %xmm1, 240(%rdi)
2708; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
2709; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2710; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
2711; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
2712; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2713; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
2714; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
2715; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2716; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
2717; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
2718; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2719; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
2720; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
2721; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2722; AVX1-NEXT:    vmovdqa %xmm1, 96(%rdi)
2723; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
2724; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2725; AVX1-NEXT:    vmovdqa %xmm1, 112(%rdi)
2726; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
2727; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2728; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
2729; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
2730; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2731; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
2732; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2733; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2734; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
2735; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
2736; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
2737; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
2738; AVX1-NEXT:    vzeroupper
2739; AVX1-NEXT:    retq
2740;
2741; AVX2-LABEL: smulo_v64i8:
2742; AVX2:       # %bb.0:
2743; AVX2-NEXT:    movq %rdi, %rax
2744; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2745; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2746; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2747; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2748; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
2749; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2750; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2751; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2752; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2753; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm10
2754; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
2755; AVX2-NEXT:    vpcmpgtb %ymm10, %ymm6, %ymm7
2756; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2757; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2758; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2759; AVX2-NEXT:    vpsraw $8, %ymm9, %ymm9
2760; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2761; AVX2-NEXT:    vpsrlw $8, %ymm8, %ymm8
2762; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2763; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
2764; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2765; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
2766; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2767; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
2768; AVX2-NEXT:    vpackuswb %ymm8, %ymm1, %ymm1
2769; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm1
2770; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm7, %ymm7
2771; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm1
2772; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2773; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2774; AVX2-NEXT:    vpmullw %ymm3, %ymm8, %ymm3
2775; AVX2-NEXT:    vpand %ymm5, %ymm3, %ymm3
2776; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2777; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2778; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
2779; AVX2-NEXT:    vpand %ymm5, %ymm8, %ymm5
2780; AVX2-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
2781; AVX2-NEXT:    vpcmpgtb %ymm3, %ymm6, %ymm5
2782; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2783; AVX2-NEXT:    vpsraw $8, %ymm6, %ymm6
2784; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2785; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
2786; AVX2-NEXT:    vpmullw %ymm6, %ymm8, %ymm6
2787; AVX2-NEXT:    vpsrlw $8, %ymm6, %ymm6
2788; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2789; AVX2-NEXT:    vpsraw $8, %ymm2, %ymm2
2790; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2791; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
2792; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
2793; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
2794; AVX2-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
2795; AVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
2796; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm0
2797; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2798; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm8
2799; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
2800; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
2801; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
2802; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
2803; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
2804; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
2805; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
2806; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
2807; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
2808; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
2809; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
2810; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
2811; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
2812; AVX2-NEXT:    vmovdqa %ymm3, (%rsi)
2813; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
2814; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
2815; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)
2816; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
2817; AVX2-NEXT:    vmovdqa %ymm2, 224(%rdi)
2818; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
2819; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
2820; AVX2-NEXT:    vmovdqa %ymm8, 32(%rdi)
2821; AVX2-NEXT:    vzeroupper
2822; AVX2-NEXT:    retq
2823;
2824; AVX512-LABEL: smulo_v64i8:
2825; AVX512:       # %bb.0:
2826; AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2827; AVX512-NEXT:    vpsraw $8, %zmm2, %zmm2
2828; AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2829; AVX512-NEXT:    vpsraw $8, %zmm3, %zmm3
2830; AVX512-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2831; AVX512-NEXT:    vpsrlw $8, %zmm2, %zmm2
2832; AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2833; AVX512-NEXT:    vpsraw $8, %zmm3, %zmm3
2834; AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2835; AVX512-NEXT:    vpsraw $8, %zmm4, %zmm4
2836; AVX512-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2837; AVX512-NEXT:    vpsrlw $8, %zmm3, %zmm3
2838; AVX512-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
2839; AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2840; AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2841; AVX512-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
2842; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2843; AVX512-NEXT:    vpandq %zmm4, %zmm3, %zmm3
2844; AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2845; AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2846; AVX512-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2847; AVX512-NEXT:    vpandq %zmm4, %zmm0, %zmm0
2848; AVX512-NEXT:    vpackuswb %zmm3, %zmm0, %zmm4
2849; AVX512-NEXT:    vpmovb2m %zmm4, %k0
2850; AVX512-NEXT:    vpmovm2b %k0, %zmm0
2851; AVX512-NEXT:    vpcmpneqb %zmm0, %zmm2, %k1
2852; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2853; AVX512-NEXT:    kshiftrd $16, %k1, %k2
2854; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2855; AVX512-NEXT:    kshiftrq $32, %k1, %k1
2856; AVX512-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
2857; AVX512-NEXT:    kshiftrd $16, %k1, %k1
2858; AVX512-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2859; AVX512-NEXT:    vmovdqa64 %zmm4, (%rdi)
2860; AVX512-NEXT:    retq
2861  %t = call {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)
2862  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 0
2863  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 1
2864  %res = sext <64 x i1> %obit to <64 x i32>
2865  store <64 x i8> %val, <64 x i8>* %p2
2866  ret <64 x i32> %res
2867}
2868
2869define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
2870; SSE2-LABEL: smulo_v8i16:
2871; SSE2:       # %bb.0:
2872; SSE2-NEXT:    movdqa %xmm0, %xmm2
2873; SSE2-NEXT:    pmulhw %xmm1, %xmm2
2874; SSE2-NEXT:    pmullw %xmm1, %xmm0
2875; SSE2-NEXT:    movdqa %xmm0, (%rdi)
2876; SSE2-NEXT:    psraw $15, %xmm0
2877; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
2878; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
2879; SSE2-NEXT:    pxor %xmm0, %xmm1
2880; SSE2-NEXT:    movdqa %xmm1, %xmm0
2881; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2882; SSE2-NEXT:    pslld $31, %xmm0
2883; SSE2-NEXT:    psrad $31, %xmm0
2884; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2885; SSE2-NEXT:    pslld $31, %xmm1
2886; SSE2-NEXT:    psrad $31, %xmm1
2887; SSE2-NEXT:    retq
2888;
2889; SSSE3-LABEL: smulo_v8i16:
2890; SSSE3:       # %bb.0:
2891; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2892; SSSE3-NEXT:    pmulhw %xmm1, %xmm2
2893; SSSE3-NEXT:    pmullw %xmm1, %xmm0
2894; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
2895; SSSE3-NEXT:    psraw $15, %xmm0
2896; SSSE3-NEXT:    pcmpeqw %xmm2, %xmm0
2897; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
2898; SSSE3-NEXT:    pxor %xmm0, %xmm1
2899; SSSE3-NEXT:    movdqa %xmm1, %xmm0
2900; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2901; SSSE3-NEXT:    pslld $31, %xmm0
2902; SSSE3-NEXT:    psrad $31, %xmm0
2903; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2904; SSSE3-NEXT:    pslld $31, %xmm1
2905; SSSE3-NEXT:    psrad $31, %xmm1
2906; SSSE3-NEXT:    retq
2907;
2908; SSE41-LABEL: smulo_v8i16:
2909; SSE41:       # %bb.0:
2910; SSE41-NEXT:    movdqa %xmm0, %xmm2
2911; SSE41-NEXT:    pmulhw %xmm1, %xmm2
2912; SSE41-NEXT:    pmullw %xmm1, %xmm0
2913; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2914; SSE41-NEXT:    psraw $15, %xmm0
2915; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
2916; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
2917; SSE41-NEXT:    pxor %xmm0, %xmm1
2918; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2919; SSE41-NEXT:    pslld $31, %xmm0
2920; SSE41-NEXT:    psrad $31, %xmm0
2921; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2922; SSE41-NEXT:    pslld $31, %xmm1
2923; SSE41-NEXT:    psrad $31, %xmm1
2924; SSE41-NEXT:    retq
2925;
2926; AVX1-LABEL: smulo_v8i16:
2927; AVX1:       # %bb.0:
2928; AVX1-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
2929; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
2930; AVX1-NEXT:    vpsraw $15, %xmm1, %xmm0
2931; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
2932; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
2933; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
2934; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
2935; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2936; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
2937; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
2938; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
2939; AVX1-NEXT:    retq
2940;
2941; AVX2-LABEL: smulo_v8i16:
2942; AVX2:       # %bb.0:
2943; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
2944; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
2945; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm0
2946; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
2947; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
2948; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
2949; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
2950; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
2951; AVX2-NEXT:    retq
2952;
2953; AVX512-LABEL: smulo_v8i16:
2954; AVX512:       # %bb.0:
2955; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
2956; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
2957; AVX512-NEXT:    vpsraw $15, %xmm1, %xmm0
2958; AVX512-NEXT:    vpcmpneqw %xmm0, %xmm2, %k1
2959; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
2960; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
2961; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
2962; AVX512-NEXT:    retq
2963  %t = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
2964  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
2965  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
2966  %res = sext <8 x i1> %obit to <8 x i32>
2967  store <8 x i16> %val, <8 x i16>* %p2
2968  ret <8 x i32> %res
2969}
2970
2971define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
2972; SSE2-LABEL: smulo_v2i64:
2973; SSE2:       # %bb.0:
2974; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2975; SSE2-NEXT:    movq %xmm2, %r8
2976; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2977; SSE2-NEXT:    movq %xmm2, %rcx
2978; SSE2-NEXT:    movq %xmm1, %rdx
2979; SSE2-NEXT:    movq %xmm0, %rsi
2980; SSE2-NEXT:    xorl %eax, %eax
2981; SSE2-NEXT:    imulq %rdx, %rsi
2982; SSE2-NEXT:    movq $-1, %r9
2983; SSE2-NEXT:    movl $0, %edx
2984; SSE2-NEXT:    cmovoq %r9, %rdx
2985; SSE2-NEXT:    movq %rsi, %xmm1
2986; SSE2-NEXT:    imulq %r8, %rcx
2987; SSE2-NEXT:    movq %rcx, %xmm0
2988; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2989; SSE2-NEXT:    movq %rdx, %xmm0
2990; SSE2-NEXT:    cmovoq %r9, %rax
2991; SSE2-NEXT:    movq %rax, %xmm2
2992; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2993; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2994; SSE2-NEXT:    movdqa %xmm1, (%rdi)
2995; SSE2-NEXT:    retq
2996;
2997; SSSE3-LABEL: smulo_v2i64:
2998; SSSE3:       # %bb.0:
2999; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
3000; SSSE3-NEXT:    movq %xmm2, %r8
3001; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
3002; SSSE3-NEXT:    movq %xmm2, %rcx
3003; SSSE3-NEXT:    movq %xmm1, %rdx
3004; SSSE3-NEXT:    movq %xmm0, %rsi
3005; SSSE3-NEXT:    xorl %eax, %eax
3006; SSSE3-NEXT:    imulq %rdx, %rsi
3007; SSSE3-NEXT:    movq $-1, %r9
3008; SSSE3-NEXT:    movl $0, %edx
3009; SSSE3-NEXT:    cmovoq %r9, %rdx
3010; SSSE3-NEXT:    movq %rsi, %xmm1
3011; SSSE3-NEXT:    imulq %r8, %rcx
3012; SSSE3-NEXT:    movq %rcx, %xmm0
3013; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3014; SSSE3-NEXT:    movq %rdx, %xmm0
3015; SSSE3-NEXT:    cmovoq %r9, %rax
3016; SSSE3-NEXT:    movq %rax, %xmm2
3017; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3018; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3019; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
3020; SSSE3-NEXT:    retq
3021;
3022; SSE41-LABEL: smulo_v2i64:
3023; SSE41:       # %bb.0:
3024; SSE41-NEXT:    movq %xmm1, %r8
3025; SSE41-NEXT:    movq %xmm0, %rcx
3026; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
3027; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
3028; SSE41-NEXT:    xorl %eax, %eax
3029; SSE41-NEXT:    imulq %rdx, %rsi
3030; SSE41-NEXT:    movq $-1, %r9
3031; SSE41-NEXT:    movl $0, %edx
3032; SSE41-NEXT:    cmovoq %r9, %rdx
3033; SSE41-NEXT:    movq %rsi, %xmm0
3034; SSE41-NEXT:    imulq %r8, %rcx
3035; SSE41-NEXT:    movq %rcx, %xmm1
3036; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3037; SSE41-NEXT:    movq %rdx, %xmm0
3038; SSE41-NEXT:    cmovoq %r9, %rax
3039; SSE41-NEXT:    movq %rax, %xmm2
3040; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
3041; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
3042; SSE41-NEXT:    movdqa %xmm1, (%rdi)
3043; SSE41-NEXT:    retq
3044;
3045; AVX1-LABEL: smulo_v2i64:
3046; AVX1:       # %bb.0:
3047; AVX1-NEXT:    vmovq %xmm1, %r8
3048; AVX1-NEXT:    vmovq %xmm0, %rcx
3049; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
3050; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
3051; AVX1-NEXT:    xorl %eax, %eax
3052; AVX1-NEXT:    imulq %rdx, %rsi
3053; AVX1-NEXT:    movq $-1, %r9
3054; AVX1-NEXT:    movl $0, %edx
3055; AVX1-NEXT:    cmovoq %r9, %rdx
3056; AVX1-NEXT:    vmovq %rsi, %xmm0
3057; AVX1-NEXT:    imulq %r8, %rcx
3058; AVX1-NEXT:    vmovq %rcx, %xmm1
3059; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3060; AVX1-NEXT:    vmovq %rdx, %xmm0
3061; AVX1-NEXT:    cmovoq %r9, %rax
3062; AVX1-NEXT:    vmovq %rax, %xmm2
3063; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
3064; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3065; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
3066; AVX1-NEXT:    retq
3067;
3068; AVX2-LABEL: smulo_v2i64:
3069; AVX2:       # %bb.0:
3070; AVX2-NEXT:    vmovq %xmm1, %r8
3071; AVX2-NEXT:    vmovq %xmm0, %rcx
3072; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
3073; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
3074; AVX2-NEXT:    xorl %eax, %eax
3075; AVX2-NEXT:    imulq %rdx, %rsi
3076; AVX2-NEXT:    movq $-1, %r9
3077; AVX2-NEXT:    movl $0, %edx
3078; AVX2-NEXT:    cmovoq %r9, %rdx
3079; AVX2-NEXT:    vmovq %rsi, %xmm0
3080; AVX2-NEXT:    imulq %r8, %rcx
3081; AVX2-NEXT:    vmovq %rcx, %xmm1
3082; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3083; AVX2-NEXT:    vmovq %rdx, %xmm0
3084; AVX2-NEXT:    cmovoq %r9, %rax
3085; AVX2-NEXT:    vmovq %rax, %xmm2
3086; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
3087; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3088; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
3089; AVX2-NEXT:    retq
3090;
3091; AVX512-LABEL: smulo_v2i64:
3092; AVX512:       # %bb.0:
3093; AVX512-NEXT:    vmovq %xmm1, %rax
3094; AVX512-NEXT:    vmovq %xmm0, %rcx
3095; AVX512-NEXT:    vpextrq $1, %xmm1, %rdx
3096; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
3097; AVX512-NEXT:    imulq %rdx, %rsi
3098; AVX512-NEXT:    seto %dl
3099; AVX512-NEXT:    vmovq %rsi, %xmm0
3100; AVX512-NEXT:    imulq %rax, %rcx
3101; AVX512-NEXT:    vmovq %rcx, %xmm1
3102; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
3103; AVX512-NEXT:    seto %al
3104; AVX512-NEXT:    movw $-3, %cx
3105; AVX512-NEXT:    kmovd %ecx, %k0
3106; AVX512-NEXT:    kmovd %eax, %k1
3107; AVX512-NEXT:    kandw %k0, %k1, %k0
3108; AVX512-NEXT:    kmovd %edx, %k1
3109; AVX512-NEXT:    kshiftlw $15, %k1, %k1
3110; AVX512-NEXT:    kshiftrw $14, %k1, %k1
3111; AVX512-NEXT:    korw %k1, %k0, %k1
3112; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3113; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3114; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
3115; AVX512-NEXT:    retq
3116  %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
3117  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
3118  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
3119  %res = sext <2 x i1> %obit to <2 x i32>
3120  store <2 x i64> %val, <2 x i64>* %p2
3121  ret <2 x i32> %res
3122}
3123
3124define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
3125; SSE2-LABEL: smulo_v4i24:
3126; SSE2:       # %bb.0:
3127; SSE2-NEXT:    pslld $8, %xmm0
3128; SSE2-NEXT:    psrad $8, %xmm0
3129; SSE2-NEXT:    pslld $8, %xmm1
3130; SSE2-NEXT:    psrad $8, %xmm1
3131; SSE2-NEXT:    pxor %xmm3, %xmm3
3132; SSE2-NEXT:    pxor %xmm2, %xmm2
3133; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3134; SSE2-NEXT:    pand %xmm0, %xmm2
3135; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3136; SSE2-NEXT:    pand %xmm1, %xmm3
3137; SSE2-NEXT:    paddd %xmm2, %xmm3
3138; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3139; SSE2-NEXT:    pmuludq %xmm1, %xmm0
3140; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3141; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3142; SSE2-NEXT:    pmuludq %xmm4, %xmm2
3143; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3144; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3145; SSE2-NEXT:    psubd %xmm3, %xmm5
3146; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3147; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3148; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3149; SSE2-NEXT:    movdqa %xmm3, %xmm1
3150; SSE2-NEXT:    pslld $8, %xmm1
3151; SSE2-NEXT:    psrad $8, %xmm1
3152; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
3153; SSE2-NEXT:    psrad $31, %xmm3
3154; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
3155; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
3156; SSE2-NEXT:    pxor %xmm4, %xmm3
3157; SSE2-NEXT:    pxor %xmm4, %xmm1
3158; SSE2-NEXT:    por %xmm3, %xmm1
3159; SSE2-NEXT:    movd %xmm0, %eax
3160; SSE2-NEXT:    movw %ax, (%rdi)
3161; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3162; SSE2-NEXT:    movd %xmm0, %ecx
3163; SSE2-NEXT:    movw %cx, 6(%rdi)
3164; SSE2-NEXT:    movd %xmm2, %edx
3165; SSE2-NEXT:    movw %dx, 3(%rdi)
3166; SSE2-NEXT:    shrl $16, %eax
3167; SSE2-NEXT:    movb %al, 2(%rdi)
3168; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3169; SSE2-NEXT:    movd %xmm0, %eax
3170; SSE2-NEXT:    movw %ax, 9(%rdi)
3171; SSE2-NEXT:    shrl $16, %ecx
3172; SSE2-NEXT:    movb %cl, 8(%rdi)
3173; SSE2-NEXT:    shrl $16, %edx
3174; SSE2-NEXT:    movb %dl, 5(%rdi)
3175; SSE2-NEXT:    shrl $16, %eax
3176; SSE2-NEXT:    movb %al, 11(%rdi)
3177; SSE2-NEXT:    movdqa %xmm1, %xmm0
3178; SSE2-NEXT:    retq
3179;
3180; SSSE3-LABEL: smulo_v4i24:
3181; SSSE3:       # %bb.0:
3182; SSSE3-NEXT:    pslld $8, %xmm0
3183; SSSE3-NEXT:    psrad $8, %xmm0
3184; SSSE3-NEXT:    pslld $8, %xmm1
3185; SSSE3-NEXT:    psrad $8, %xmm1
3186; SSSE3-NEXT:    pxor %xmm3, %xmm3
3187; SSSE3-NEXT:    pxor %xmm2, %xmm2
3188; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
3189; SSSE3-NEXT:    pand %xmm0, %xmm2
3190; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
3191; SSSE3-NEXT:    pand %xmm1, %xmm3
3192; SSSE3-NEXT:    paddd %xmm2, %xmm3
3193; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3194; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
3195; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
3196; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3197; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
3198; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
3199; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3200; SSSE3-NEXT:    psubd %xmm3, %xmm5
3201; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
3202; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
3203; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3204; SSSE3-NEXT:    movdqa %xmm3, %xmm1
3205; SSSE3-NEXT:    pslld $8, %xmm1
3206; SSSE3-NEXT:    psrad $8, %xmm1
3207; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm1
3208; SSSE3-NEXT:    psrad $31, %xmm3
3209; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
3210; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
3211; SSSE3-NEXT:    pxor %xmm4, %xmm3
3212; SSSE3-NEXT:    pxor %xmm4, %xmm1
3213; SSSE3-NEXT:    por %xmm3, %xmm1
3214; SSSE3-NEXT:    movd %xmm0, %eax
3215; SSSE3-NEXT:    movw %ax, (%rdi)
3216; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
3217; SSSE3-NEXT:    movd %xmm0, %ecx
3218; SSSE3-NEXT:    movw %cx, 6(%rdi)
3219; SSSE3-NEXT:    movd %xmm2, %edx
3220; SSSE3-NEXT:    movw %dx, 3(%rdi)
3221; SSSE3-NEXT:    shrl $16, %eax
3222; SSSE3-NEXT:    movb %al, 2(%rdi)
3223; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
3224; SSSE3-NEXT:    movd %xmm0, %eax
3225; SSSE3-NEXT:    movw %ax, 9(%rdi)
3226; SSSE3-NEXT:    shrl $16, %ecx
3227; SSSE3-NEXT:    movb %cl, 8(%rdi)
3228; SSSE3-NEXT:    shrl $16, %edx
3229; SSSE3-NEXT:    movb %dl, 5(%rdi)
3230; SSSE3-NEXT:    shrl $16, %eax
3231; SSSE3-NEXT:    movb %al, 11(%rdi)
3232; SSSE3-NEXT:    movdqa %xmm1, %xmm0
3233; SSSE3-NEXT:    retq
3234;
3235; SSE41-LABEL: smulo_v4i24:
3236; SSE41:       # %bb.0:
3237; SSE41-NEXT:    pslld $8, %xmm0
3238; SSE41-NEXT:    psrad $8, %xmm0
3239; SSE41-NEXT:    pslld $8, %xmm1
3240; SSE41-NEXT:    psrad $8, %xmm1
3241; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3242; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3243; SSE41-NEXT:    pmuldq %xmm2, %xmm3
3244; SSE41-NEXT:    movdqa %xmm0, %xmm2
3245; SSE41-NEXT:    pmuldq %xmm1, %xmm2
3246; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3247; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
3248; SSE41-NEXT:    pmulld %xmm0, %xmm1
3249; SSE41-NEXT:    movdqa %xmm1, %xmm3
3250; SSE41-NEXT:    pslld $8, %xmm3
3251; SSE41-NEXT:    psrad $8, %xmm3
3252; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
3253; SSE41-NEXT:    pextrd $3, %xmm1, %eax
3254; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
3255; SSE41-NEXT:    pextrd $1, %xmm1, %edx
3256; SSE41-NEXT:    movd %xmm1, %esi
3257; SSE41-NEXT:    psrad $31, %xmm1
3258; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
3259; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
3260; SSE41-NEXT:    pxor %xmm0, %xmm1
3261; SSE41-NEXT:    pxor %xmm3, %xmm0
3262; SSE41-NEXT:    por %xmm1, %xmm0
3263; SSE41-NEXT:    movw %ax, 9(%rdi)
3264; SSE41-NEXT:    movw %cx, 6(%rdi)
3265; SSE41-NEXT:    movw %dx, 3(%rdi)
3266; SSE41-NEXT:    movw %si, (%rdi)
3267; SSE41-NEXT:    shrl $16, %eax
3268; SSE41-NEXT:    movb %al, 11(%rdi)
3269; SSE41-NEXT:    shrl $16, %ecx
3270; SSE41-NEXT:    movb %cl, 8(%rdi)
3271; SSE41-NEXT:    shrl $16, %edx
3272; SSE41-NEXT:    movb %dl, 5(%rdi)
3273; SSE41-NEXT:    shrl $16, %esi
3274; SSE41-NEXT:    movb %sil, 2(%rdi)
3275; SSE41-NEXT:    retq
3276;
3277; AVX1-LABEL: smulo_v4i24:
3278; AVX1:       # %bb.0:
3279; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
3280; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
3281; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
3282; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
3283; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3284; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3285; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3286; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3287; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3288; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3289; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3290; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
3291; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3292; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3293; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3294; AVX1-NEXT:    vpslld $8, %xmm1, %xmm3
3295; AVX1-NEXT:    vpsrad $8, %xmm3, %xmm3
3296; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3297; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3298; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
3299; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
3300; AVX1-NEXT:    movw %ax, 9(%rdi)
3301; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
3302; AVX1-NEXT:    movw %cx, 6(%rdi)
3303; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
3304; AVX1-NEXT:    movw %dx, 3(%rdi)
3305; AVX1-NEXT:    vmovd %xmm1, %esi
3306; AVX1-NEXT:    movw %si, (%rdi)
3307; AVX1-NEXT:    shrl $16, %eax
3308; AVX1-NEXT:    movb %al, 11(%rdi)
3309; AVX1-NEXT:    shrl $16, %ecx
3310; AVX1-NEXT:    movb %cl, 8(%rdi)
3311; AVX1-NEXT:    shrl $16, %edx
3312; AVX1-NEXT:    movb %dl, 5(%rdi)
3313; AVX1-NEXT:    shrl $16, %esi
3314; AVX1-NEXT:    movb %sil, 2(%rdi)
3315; AVX1-NEXT:    retq
3316;
3317; AVX2-LABEL: smulo_v4i24:
3318; AVX2:       # %bb.0:
3319; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
3320; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
3321; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
3322; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
3323; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3324; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3325; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3326; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3327; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3328; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3329; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3330; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
3331; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
3332; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3333; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3334; AVX2-NEXT:    vpslld $8, %xmm1, %xmm3
3335; AVX2-NEXT:    vpsrad $8, %xmm3, %xmm3
3336; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
3337; AVX2-NEXT:    vpxor %xmm2, %xmm3, %xmm2
3338; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
3339; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
3340; AVX2-NEXT:    movw %ax, 9(%rdi)
3341; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
3342; AVX2-NEXT:    movw %cx, 6(%rdi)
3343; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
3344; AVX2-NEXT:    movw %dx, 3(%rdi)
3345; AVX2-NEXT:    vmovd %xmm1, %esi
3346; AVX2-NEXT:    movw %si, (%rdi)
3347; AVX2-NEXT:    shrl $16, %eax
3348; AVX2-NEXT:    movb %al, 11(%rdi)
3349; AVX2-NEXT:    shrl $16, %ecx
3350; AVX2-NEXT:    movb %cl, 8(%rdi)
3351; AVX2-NEXT:    shrl $16, %edx
3352; AVX2-NEXT:    movb %dl, 5(%rdi)
3353; AVX2-NEXT:    shrl $16, %esi
3354; AVX2-NEXT:    movb %sil, 2(%rdi)
3355; AVX2-NEXT:    retq
3356;
3357; AVX512-LABEL: smulo_v4i24:
3358; AVX512:       # %bb.0:
3359; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
3360; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
3361; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
3362; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3363; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
3364; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
3365; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3366; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
3367; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
3368; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
3369; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3370; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
3371; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k0
3372; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
3373; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
3374; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k1
3375; AVX512-NEXT:    korw %k0, %k1, %k1
3376; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3377; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3378; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
3379; AVX512-NEXT:    movw %ax, 9(%rdi)
3380; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
3381; AVX512-NEXT:    movw %cx, 6(%rdi)
3382; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
3383; AVX512-NEXT:    movw %dx, 3(%rdi)
3384; AVX512-NEXT:    vmovd %xmm1, %esi
3385; AVX512-NEXT:    movw %si, (%rdi)
3386; AVX512-NEXT:    shrl $16, %eax
3387; AVX512-NEXT:    movb %al, 11(%rdi)
3388; AVX512-NEXT:    shrl $16, %ecx
3389; AVX512-NEXT:    movb %cl, 8(%rdi)
3390; AVX512-NEXT:    shrl $16, %edx
3391; AVX512-NEXT:    movb %dl, 5(%rdi)
3392; AVX512-NEXT:    shrl $16, %esi
3393; AVX512-NEXT:    movb %sil, 2(%rdi)
3394; AVX512-NEXT:    retq
3395  %t = call {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
3396  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
3397  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
3398  %res = sext <4 x i1> %obit to <4 x i32>
3399  store <4 x i24> %val, <4 x i24>* %p2
3400  ret <4 x i32> %res
3401}
3402
3403define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
3404; SSE2-LABEL: smulo_v4i1:
3405; SSE2:       # %bb.0:
3406; SSE2-NEXT:    pslld $31, %xmm0
3407; SSE2-NEXT:    psrad $31, %xmm0
3408; SSE2-NEXT:    pslld $31, %xmm1
3409; SSE2-NEXT:    psrad $31, %xmm1
3410; SSE2-NEXT:    pxor %xmm2, %xmm2
3411; SSE2-NEXT:    pxor %xmm3, %xmm3
3412; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
3413; SSE2-NEXT:    pand %xmm0, %xmm3
3414; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
3415; SSE2-NEXT:    pand %xmm1, %xmm2
3416; SSE2-NEXT:    paddd %xmm3, %xmm2
3417; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3418; SSE2-NEXT:    pmuludq %xmm1, %xmm0
3419; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
3420; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3421; SSE2-NEXT:    pmuludq %xmm3, %xmm1
3422; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
3423; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
3424; SSE2-NEXT:    psubd %xmm2, %xmm4
3425; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
3426; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
3427; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
3428; SSE2-NEXT:    movdqa %xmm2, %xmm0
3429; SSE2-NEXT:    pslld $31, %xmm0
3430; SSE2-NEXT:    movmskps %xmm0, %eax
3431; SSE2-NEXT:    psrad $31, %xmm0
3432; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
3433; SSE2-NEXT:    psrad $31, %xmm2
3434; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
3435; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
3436; SSE2-NEXT:    pxor %xmm1, %xmm2
3437; SSE2-NEXT:    pxor %xmm1, %xmm0
3438; SSE2-NEXT:    por %xmm2, %xmm0
3439; SSE2-NEXT:    movb %al, (%rdi)
3440; SSE2-NEXT:    retq
3441;
3442; SSSE3-LABEL: smulo_v4i1:
3443; SSSE3:       # %bb.0:
3444; SSSE3-NEXT:    pslld $31, %xmm0
3445; SSSE3-NEXT:    psrad $31, %xmm0
3446; SSSE3-NEXT:    pslld $31, %xmm1
3447; SSSE3-NEXT:    psrad $31, %xmm1
3448; SSSE3-NEXT:    pxor %xmm2, %xmm2
3449; SSSE3-NEXT:    pxor %xmm3, %xmm3
3450; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
3451; SSSE3-NEXT:    pand %xmm0, %xmm3
3452; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
3453; SSSE3-NEXT:    pand %xmm1, %xmm2
3454; SSSE3-NEXT:    paddd %xmm3, %xmm2
3455; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3456; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
3457; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
3458; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3459; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
3460; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
3461; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
3462; SSSE3-NEXT:    psubd %xmm2, %xmm4
3463; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
3464; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
3465; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
3466; SSSE3-NEXT:    movdqa %xmm2, %xmm0
3467; SSSE3-NEXT:    pslld $31, %xmm0
3468; SSSE3-NEXT:    movmskps %xmm0, %eax
3469; SSSE3-NEXT:    psrad $31, %xmm0
3470; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
3471; SSSE3-NEXT:    psrad $31, %xmm2
3472; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
3473; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
3474; SSSE3-NEXT:    pxor %xmm1, %xmm2
3475; SSSE3-NEXT:    pxor %xmm1, %xmm0
3476; SSSE3-NEXT:    por %xmm2, %xmm0
3477; SSSE3-NEXT:    movb %al, (%rdi)
3478; SSSE3-NEXT:    retq
3479;
3480; SSE41-LABEL: smulo_v4i1:
3481; SSE41:       # %bb.0:
3482; SSE41-NEXT:    pslld $31, %xmm0
3483; SSE41-NEXT:    psrad $31, %xmm0
3484; SSE41-NEXT:    pslld $31, %xmm1
3485; SSE41-NEXT:    psrad $31, %xmm1
3486; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3487; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3488; SSE41-NEXT:    pmuldq %xmm2, %xmm3
3489; SSE41-NEXT:    movdqa %xmm0, %xmm2
3490; SSE41-NEXT:    pmuldq %xmm1, %xmm2
3491; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3492; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
3493; SSE41-NEXT:    pmulld %xmm0, %xmm1
3494; SSE41-NEXT:    movdqa %xmm1, %xmm3
3495; SSE41-NEXT:    pslld $31, %xmm3
3496; SSE41-NEXT:    movmskps %xmm3, %eax
3497; SSE41-NEXT:    psrad $31, %xmm3
3498; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
3499; SSE41-NEXT:    psrad $31, %xmm1
3500; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
3501; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
3502; SSE41-NEXT:    pxor %xmm0, %xmm1
3503; SSE41-NEXT:    pxor %xmm3, %xmm0
3504; SSE41-NEXT:    por %xmm1, %xmm0
3505; SSE41-NEXT:    movb %al, (%rdi)
3506; SSE41-NEXT:    retq
3507;
3508; AVX1-LABEL: smulo_v4i1:
3509; AVX1:       # %bb.0:
3510; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
3511; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
3512; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
3513; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
3514; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3515; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3516; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3517; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3518; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3519; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3520; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
3521; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
3522; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
3523; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3524; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
3525; AVX1-NEXT:    vpslld $31, %xmm0, %xmm3
3526; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm4
3527; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
3528; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3529; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
3530; AVX1-NEXT:    vmovmskps %xmm3, %eax
3531; AVX1-NEXT:    movb %al, (%rdi)
3532; AVX1-NEXT:    retq
3533;
3534; AVX2-LABEL: smulo_v4i1:
3535; AVX2:       # %bb.0:
3536; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
3537; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
3538; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
3539; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
3540; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3541; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3542; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
3543; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
3544; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3545; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3546; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
3547; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
3548; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
3549; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3550; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
3551; AVX2-NEXT:    vpslld $31, %xmm0, %xmm3
3552; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
3553; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
3554; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
3555; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
3556; AVX2-NEXT:    vmovmskps %xmm3, %eax
3557; AVX2-NEXT:    movb %al, (%rdi)
3558; AVX2-NEXT:    retq
3559;
3560; AVX512-LABEL: smulo_v4i1:
3561; AVX512:       # %bb.0:
3562; AVX512-NEXT:    pushq %rbp
3563; AVX512-NEXT:    pushq %rbx
3564; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
3565; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k0
3566; AVX512-NEXT:    kshiftrw $3, %k0, %k1
3567; AVX512-NEXT:    kmovd %k1, %r10d
3568; AVX512-NEXT:    andb $1, %r10b
3569; AVX512-NEXT:    negb %r10b
3570; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
3571; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
3572; AVX512-NEXT:    kshiftrw $3, %k1, %k2
3573; AVX512-NEXT:    kmovd %k2, %r9d
3574; AVX512-NEXT:    andb $1, %r9b
3575; AVX512-NEXT:    negb %r9b
3576; AVX512-NEXT:    kshiftrw $2, %k1, %k2
3577; AVX512-NEXT:    kmovd %k2, %ebp
3578; AVX512-NEXT:    andb $1, %bpl
3579; AVX512-NEXT:    negb %bpl
3580; AVX512-NEXT:    kshiftrw $2, %k0, %k2
3581; AVX512-NEXT:    kmovd %k2, %edx
3582; AVX512-NEXT:    andb $1, %dl
3583; AVX512-NEXT:    negb %dl
3584; AVX512-NEXT:    kshiftrw $1, %k0, %k2
3585; AVX512-NEXT:    kmovd %k2, %esi
3586; AVX512-NEXT:    andb $1, %sil
3587; AVX512-NEXT:    negb %sil
3588; AVX512-NEXT:    kshiftrw $1, %k1, %k2
3589; AVX512-NEXT:    kmovd %k2, %ecx
3590; AVX512-NEXT:    andb $1, %cl
3591; AVX512-NEXT:    negb %cl
3592; AVX512-NEXT:    kmovd %k1, %eax
3593; AVX512-NEXT:    andb $1, %al
3594; AVX512-NEXT:    negb %al
3595; AVX512-NEXT:    kmovd %k0, %ebx
3596; AVX512-NEXT:    andb $1, %bl
3597; AVX512-NEXT:    negb %bl
3598; AVX512-NEXT:    # kill: def $al killed $al killed $eax
3599; AVX512-NEXT:    imulb %bl
3600; AVX512-NEXT:    movl %eax, %r11d
3601; AVX512-NEXT:    seto %al
3602; AVX512-NEXT:    movl %r11d, %ebx
3603; AVX512-NEXT:    andb $1, %bl
3604; AVX512-NEXT:    negb %bl
3605; AVX512-NEXT:    cmpb %r11b, %bl
3606; AVX512-NEXT:    setne %bl
3607; AVX512-NEXT:    orb %al, %bl
3608; AVX512-NEXT:    setne %al
3609; AVX512-NEXT:    kmovd %eax, %k1
3610; AVX512-NEXT:    movw $-3, %ax
3611; AVX512-NEXT:    kmovd %eax, %k0
3612; AVX512-NEXT:    kandw %k0, %k1, %k1
3613; AVX512-NEXT:    movl %ecx, %eax
3614; AVX512-NEXT:    imulb %sil
3615; AVX512-NEXT:    movl %eax, %r8d
3616; AVX512-NEXT:    seto %al
3617; AVX512-NEXT:    movl %r8d, %ecx
3618; AVX512-NEXT:    andb $1, %cl
3619; AVX512-NEXT:    negb %cl
3620; AVX512-NEXT:    cmpb %r8b, %cl
3621; AVX512-NEXT:    setne %cl
3622; AVX512-NEXT:    orb %al, %cl
3623; AVX512-NEXT:    setne %al
3624; AVX512-NEXT:    kmovd %eax, %k2
3625; AVX512-NEXT:    kshiftlw $15, %k2, %k2
3626; AVX512-NEXT:    kshiftrw $14, %k2, %k2
3627; AVX512-NEXT:    korw %k2, %k1, %k2
3628; AVX512-NEXT:    movw $-5, %ax
3629; AVX512-NEXT:    kmovd %eax, %k1
3630; AVX512-NEXT:    kandw %k1, %k2, %k2
3631; AVX512-NEXT:    movl %ebp, %eax
3632; AVX512-NEXT:    imulb %dl
3633; AVX512-NEXT:    movl %eax, %esi
3634; AVX512-NEXT:    seto %al
3635; AVX512-NEXT:    movl %esi, %ecx
3636; AVX512-NEXT:    andb $1, %cl
3637; AVX512-NEXT:    negb %cl
3638; AVX512-NEXT:    cmpb %sil, %cl
3639; AVX512-NEXT:    setne %cl
3640; AVX512-NEXT:    orb %al, %cl
3641; AVX512-NEXT:    setne %al
3642; AVX512-NEXT:    kmovd %eax, %k3
3643; AVX512-NEXT:    kshiftlw $2, %k3, %k3
3644; AVX512-NEXT:    korw %k3, %k2, %k2
3645; AVX512-NEXT:    kshiftlw $13, %k2, %k2
3646; AVX512-NEXT:    kshiftrw $13, %k2, %k2
3647; AVX512-NEXT:    movl %r9d, %eax
3648; AVX512-NEXT:    imulb %r10b
3649; AVX512-NEXT:    # kill: def $al killed $al def $eax
3650; AVX512-NEXT:    seto %cl
3651; AVX512-NEXT:    movl %eax, %edx
3652; AVX512-NEXT:    andb $1, %dl
3653; AVX512-NEXT:    negb %dl
3654; AVX512-NEXT:    cmpb %al, %dl
3655; AVX512-NEXT:    setne %dl
3656; AVX512-NEXT:    orb %cl, %dl
3657; AVX512-NEXT:    setne %cl
3658; AVX512-NEXT:    kmovd %ecx, %k3
3659; AVX512-NEXT:    kshiftlw $3, %k3, %k3
3660; AVX512-NEXT:    korw %k3, %k2, %k2
3661; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3662; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
3663; AVX512-NEXT:    andl $1, %r11d
3664; AVX512-NEXT:    kmovw %r11d, %k2
3665; AVX512-NEXT:    kandw %k0, %k2, %k0
3666; AVX512-NEXT:    kmovd %r8d, %k2
3667; AVX512-NEXT:    kshiftlw $15, %k2, %k2
3668; AVX512-NEXT:    kshiftrw $14, %k2, %k2
3669; AVX512-NEXT:    korw %k2, %k0, %k0
3670; AVX512-NEXT:    kandw %k1, %k0, %k0
3671; AVX512-NEXT:    kmovd %esi, %k1
3672; AVX512-NEXT:    kshiftlw $15, %k1, %k1
3673; AVX512-NEXT:    kshiftrw $13, %k1, %k1
3674; AVX512-NEXT:    korw %k1, %k0, %k0
3675; AVX512-NEXT:    movw $-9, %cx
3676; AVX512-NEXT:    kmovd %ecx, %k1
3677; AVX512-NEXT:    kandw %k1, %k0, %k0
3678; AVX512-NEXT:    kmovd %eax, %k1
3679; AVX512-NEXT:    kshiftlw $15, %k1, %k1
3680; AVX512-NEXT:    kshiftrw $12, %k1, %k1
3681; AVX512-NEXT:    korw %k1, %k0, %k0
3682; AVX512-NEXT:    kmovd %k0, %eax
3683; AVX512-NEXT:    movb %al, (%rdi)
3684; AVX512-NEXT:    popq %rbx
3685; AVX512-NEXT:    popq %rbp
3686; AVX512-NEXT:    retq
3687  %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
3688  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
3689  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
3690  %res = sext <4 x i1> %obit to <4 x i32>
3691  store <4 x i1> %val, <4 x i1>* %p2
3692  ret <4 x i32> %res
3693}
3694
3695define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
3696; SSE2-LABEL: smulo_v2i128:
3697; SSE2:       # %bb.0:
3698; SSE2-NEXT:    pushq %rbp
3699; SSE2-NEXT:    pushq %r15
3700; SSE2-NEXT:    pushq %r14
3701; SSE2-NEXT:    pushq %r13
3702; SSE2-NEXT:    pushq %r12
3703; SSE2-NEXT:    pushq %rbx
3704; SSE2-NEXT:    subq $24, %rsp
3705; SSE2-NEXT:    movq %r8, %rax
3706; SSE2-NEXT:    movq %rcx, %r14
3707; SSE2-NEXT:    movq %rdx, %rbx
3708; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3709; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3710; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3711; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3712; SSE2-NEXT:    movq %rax, %rdx
3713; SSE2-NEXT:    movq %r9, %rcx
3714; SSE2-NEXT:    callq __muloti4
3715; SSE2-NEXT:    movq %rax, %r13
3716; SSE2-NEXT:    movq %rdx, %rbp
3717; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3718; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3719; SSE2-NEXT:    movq %rbx, %rdi
3720; SSE2-NEXT:    movq %r14, %rsi
3721; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3722; SSE2-NEXT:    movq %r12, %rcx
3723; SSE2-NEXT:    callq __muloti4
3724; SSE2-NEXT:    xorl %ecx, %ecx
3725; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3726; SSE2-NEXT:    sbbl %esi, %esi
3727; SSE2-NEXT:    movd %esi, %xmm1
3728; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3729; SSE2-NEXT:    sbbl %ecx, %ecx
3730; SSE2-NEXT:    movd %ecx, %xmm0
3731; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3732; SSE2-NEXT:    movq %rdx, 24(%r15)
3733; SSE2-NEXT:    movq %rax, 16(%r15)
3734; SSE2-NEXT:    movq %rbp, 8(%r15)
3735; SSE2-NEXT:    movq %r13, (%r15)
3736; SSE2-NEXT:    addq $24, %rsp
3737; SSE2-NEXT:    popq %rbx
3738; SSE2-NEXT:    popq %r12
3739; SSE2-NEXT:    popq %r13
3740; SSE2-NEXT:    popq %r14
3741; SSE2-NEXT:    popq %r15
3742; SSE2-NEXT:    popq %rbp
3743; SSE2-NEXT:    retq
3744;
3745; SSSE3-LABEL: smulo_v2i128:
3746; SSSE3:       # %bb.0:
3747; SSSE3-NEXT:    pushq %rbp
3748; SSSE3-NEXT:    pushq %r15
3749; SSSE3-NEXT:    pushq %r14
3750; SSSE3-NEXT:    pushq %r13
3751; SSSE3-NEXT:    pushq %r12
3752; SSSE3-NEXT:    pushq %rbx
3753; SSSE3-NEXT:    subq $24, %rsp
3754; SSSE3-NEXT:    movq %r8, %rax
3755; SSSE3-NEXT:    movq %rcx, %r14
3756; SSSE3-NEXT:    movq %rdx, %rbx
3757; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3758; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3759; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3760; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3761; SSSE3-NEXT:    movq %rax, %rdx
3762; SSSE3-NEXT:    movq %r9, %rcx
3763; SSSE3-NEXT:    callq __muloti4
3764; SSSE3-NEXT:    movq %rax, %r13
3765; SSSE3-NEXT:    movq %rdx, %rbp
3766; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3767; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3768; SSSE3-NEXT:    movq %rbx, %rdi
3769; SSSE3-NEXT:    movq %r14, %rsi
3770; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3771; SSSE3-NEXT:    movq %r12, %rcx
3772; SSSE3-NEXT:    callq __muloti4
3773; SSSE3-NEXT:    xorl %ecx, %ecx
3774; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3775; SSSE3-NEXT:    sbbl %esi, %esi
3776; SSSE3-NEXT:    movd %esi, %xmm1
3777; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3778; SSSE3-NEXT:    sbbl %ecx, %ecx
3779; SSSE3-NEXT:    movd %ecx, %xmm0
3780; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3781; SSSE3-NEXT:    movq %rdx, 24(%r15)
3782; SSSE3-NEXT:    movq %rax, 16(%r15)
3783; SSSE3-NEXT:    movq %rbp, 8(%r15)
3784; SSSE3-NEXT:    movq %r13, (%r15)
3785; SSSE3-NEXT:    addq $24, %rsp
3786; SSSE3-NEXT:    popq %rbx
3787; SSSE3-NEXT:    popq %r12
3788; SSSE3-NEXT:    popq %r13
3789; SSSE3-NEXT:    popq %r14
3790; SSSE3-NEXT:    popq %r15
3791; SSSE3-NEXT:    popq %rbp
3792; SSSE3-NEXT:    retq
3793;
3794; SSE41-LABEL: smulo_v2i128:
3795; SSE41:       # %bb.0:
3796; SSE41-NEXT:    pushq %rbp
3797; SSE41-NEXT:    pushq %r15
3798; SSE41-NEXT:    pushq %r14
3799; SSE41-NEXT:    pushq %r13
3800; SSE41-NEXT:    pushq %r12
3801; SSE41-NEXT:    pushq %rbx
3802; SSE41-NEXT:    subq $24, %rsp
3803; SSE41-NEXT:    movq %r8, %rax
3804; SSE41-NEXT:    movq %rcx, %r14
3805; SSE41-NEXT:    movq %rdx, %rbx
3806; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3807; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3808; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3809; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3810; SSE41-NEXT:    movq %rax, %rdx
3811; SSE41-NEXT:    movq %r9, %rcx
3812; SSE41-NEXT:    callq __muloti4
3813; SSE41-NEXT:    movq %rax, %r13
3814; SSE41-NEXT:    movq %rdx, %rbp
3815; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3816; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3817; SSE41-NEXT:    movq %rbx, %rdi
3818; SSE41-NEXT:    movq %r14, %rsi
3819; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3820; SSE41-NEXT:    movq %r12, %rcx
3821; SSE41-NEXT:    callq __muloti4
3822; SSE41-NEXT:    xorl %ecx, %ecx
3823; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3824; SSE41-NEXT:    sbbl %esi, %esi
3825; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3826; SSE41-NEXT:    sbbl %ecx, %ecx
3827; SSE41-NEXT:    movd %ecx, %xmm0
3828; SSE41-NEXT:    pinsrd $1, %esi, %xmm0
3829; SSE41-NEXT:    movq %rdx, 24(%r15)
3830; SSE41-NEXT:    movq %rax, 16(%r15)
3831; SSE41-NEXT:    movq %rbp, 8(%r15)
3832; SSE41-NEXT:    movq %r13, (%r15)
3833; SSE41-NEXT:    addq $24, %rsp
3834; SSE41-NEXT:    popq %rbx
3835; SSE41-NEXT:    popq %r12
3836; SSE41-NEXT:    popq %r13
3837; SSE41-NEXT:    popq %r14
3838; SSE41-NEXT:    popq %r15
3839; SSE41-NEXT:    popq %rbp
3840; SSE41-NEXT:    retq
3841;
3842; AVX1-LABEL: smulo_v2i128:
3843; AVX1:       # %bb.0:
3844; AVX1-NEXT:    pushq %rbp
3845; AVX1-NEXT:    pushq %r15
3846; AVX1-NEXT:    pushq %r14
3847; AVX1-NEXT:    pushq %r13
3848; AVX1-NEXT:    pushq %r12
3849; AVX1-NEXT:    pushq %rbx
3850; AVX1-NEXT:    subq $24, %rsp
3851; AVX1-NEXT:    movq %r8, %rax
3852; AVX1-NEXT:    movq %rcx, %r14
3853; AVX1-NEXT:    movq %rdx, %rbx
3854; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3855; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3856; AVX1-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3857; AVX1-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3858; AVX1-NEXT:    movq %rax, %rdx
3859; AVX1-NEXT:    movq %r9, %rcx
3860; AVX1-NEXT:    callq __muloti4
3861; AVX1-NEXT:    movq %rax, %r13
3862; AVX1-NEXT:    movq %rdx, %rbp
3863; AVX1-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3864; AVX1-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3865; AVX1-NEXT:    movq %rbx, %rdi
3866; AVX1-NEXT:    movq %r14, %rsi
3867; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3868; AVX1-NEXT:    movq %r12, %rcx
3869; AVX1-NEXT:    callq __muloti4
3870; AVX1-NEXT:    xorl %ecx, %ecx
3871; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3872; AVX1-NEXT:    sbbl %esi, %esi
3873; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3874; AVX1-NEXT:    sbbl %ecx, %ecx
3875; AVX1-NEXT:    vmovd %ecx, %xmm0
3876; AVX1-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
3877; AVX1-NEXT:    movq %rdx, 24(%r15)
3878; AVX1-NEXT:    movq %rax, 16(%r15)
3879; AVX1-NEXT:    movq %rbp, 8(%r15)
3880; AVX1-NEXT:    movq %r13, (%r15)
3881; AVX1-NEXT:    addq $24, %rsp
3882; AVX1-NEXT:    popq %rbx
3883; AVX1-NEXT:    popq %r12
3884; AVX1-NEXT:    popq %r13
3885; AVX1-NEXT:    popq %r14
3886; AVX1-NEXT:    popq %r15
3887; AVX1-NEXT:    popq %rbp
3888; AVX1-NEXT:    retq
3889;
3890; AVX2-LABEL: smulo_v2i128:
3891; AVX2:       # %bb.0:
3892; AVX2-NEXT:    pushq %rbp
3893; AVX2-NEXT:    pushq %r15
3894; AVX2-NEXT:    pushq %r14
3895; AVX2-NEXT:    pushq %r13
3896; AVX2-NEXT:    pushq %r12
3897; AVX2-NEXT:    pushq %rbx
3898; AVX2-NEXT:    subq $24, %rsp
3899; AVX2-NEXT:    movq %r8, %rax
3900; AVX2-NEXT:    movq %rcx, %r14
3901; AVX2-NEXT:    movq %rdx, %rbx
3902; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3903; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3904; AVX2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3905; AVX2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3906; AVX2-NEXT:    movq %rax, %rdx
3907; AVX2-NEXT:    movq %r9, %rcx
3908; AVX2-NEXT:    callq __muloti4
3909; AVX2-NEXT:    movq %rax, %r13
3910; AVX2-NEXT:    movq %rdx, %rbp
3911; AVX2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3912; AVX2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3913; AVX2-NEXT:    movq %rbx, %rdi
3914; AVX2-NEXT:    movq %r14, %rsi
3915; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3916; AVX2-NEXT:    movq %r12, %rcx
3917; AVX2-NEXT:    callq __muloti4
3918; AVX2-NEXT:    xorl %ecx, %ecx
3919; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3920; AVX2-NEXT:    sbbl %esi, %esi
3921; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
3922; AVX2-NEXT:    sbbl %ecx, %ecx
3923; AVX2-NEXT:    vmovd %ecx, %xmm0
3924; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
3925; AVX2-NEXT:    movq %rdx, 24(%r15)
3926; AVX2-NEXT:    movq %rax, 16(%r15)
3927; AVX2-NEXT:    movq %rbp, 8(%r15)
3928; AVX2-NEXT:    movq %r13, (%r15)
3929; AVX2-NEXT:    addq $24, %rsp
3930; AVX2-NEXT:    popq %rbx
3931; AVX2-NEXT:    popq %r12
3932; AVX2-NEXT:    popq %r13
3933; AVX2-NEXT:    popq %r14
3934; AVX2-NEXT:    popq %r15
3935; AVX2-NEXT:    popq %rbp
3936; AVX2-NEXT:    retq
3937;
3938; AVX512-LABEL: smulo_v2i128:
3939; AVX512:       # %bb.0:
3940; AVX512-NEXT:    pushq %rbp
3941; AVX512-NEXT:    pushq %r15
3942; AVX512-NEXT:    pushq %r14
3943; AVX512-NEXT:    pushq %r13
3944; AVX512-NEXT:    pushq %r12
3945; AVX512-NEXT:    pushq %rbx
3946; AVX512-NEXT:    subq $24, %rsp
3947; AVX512-NEXT:    movq %r8, %rax
3948; AVX512-NEXT:    movq %rcx, %r14
3949; AVX512-NEXT:    movq %rdx, %rbx
3950; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3951; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r12
3952; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3953; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3954; AVX512-NEXT:    movq %rax, %rdx
3955; AVX512-NEXT:    movq %r9, %rcx
3956; AVX512-NEXT:    callq __muloti4
3957; AVX512-NEXT:    movq %rax, %r13
3958; AVX512-NEXT:    movq %rdx, %rbp
3959; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
3960; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
3961; AVX512-NEXT:    movq %rbx, %rdi
3962; AVX512-NEXT:    movq %r14, %rsi
3963; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
3964; AVX512-NEXT:    movq %r12, %rcx
3965; AVX512-NEXT:    callq __muloti4
3966; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3967; AVX512-NEXT:    setne %cl
3968; AVX512-NEXT:    kmovd %ecx, %k0
3969; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
3970; AVX512-NEXT:    setne %cl
3971; AVX512-NEXT:    andl $1, %ecx
3972; AVX512-NEXT:    kmovw %ecx, %k1
3973; AVX512-NEXT:    kshiftlw $1, %k0, %k0
3974; AVX512-NEXT:    korw %k0, %k1, %k1
3975; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3976; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3977; AVX512-NEXT:    movq %rdx, 24(%r15)
3978; AVX512-NEXT:    movq %rax, 16(%r15)
3979; AVX512-NEXT:    movq %rbp, 8(%r15)
3980; AVX512-NEXT:    movq %r13, (%r15)
3981; AVX512-NEXT:    addq $24, %rsp
3982; AVX512-NEXT:    popq %rbx
3983; AVX512-NEXT:    popq %r12
3984; AVX512-NEXT:    popq %r13
3985; AVX512-NEXT:    popq %r14
3986; AVX512-NEXT:    popq %r15
3987; AVX512-NEXT:    popq %rbp
3988; AVX512-NEXT:    retq
3989  %t = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
3990  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
3991  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
3992  %res = sext <2 x i1> %obit to <2 x i32>
3993  store <2 x i128> %val, <2 x i128>* %p2
3994  ret <2 x i32> %res
3995}
3996