1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
8
9declare {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
10declare {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
11declare {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
12declare {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
13declare {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
14declare {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
15declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
16
17declare {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
18declare {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
19declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
20
21declare {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
22declare {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
23declare {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
24
25define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
26; SSE-LABEL: smulo_v1i32:
27; SSE:       # %bb.0:
28; SSE-NEXT:    xorl %eax, %eax
29; SSE-NEXT:    imull %esi, %edi
30; SSE-NEXT:    seto %al
31; SSE-NEXT:    negl %eax
32; SSE-NEXT:    movl %edi, (%rdx)
33; SSE-NEXT:    retq
34;
35; AVX-LABEL: smulo_v1i32:
36; AVX:       # %bb.0:
37; AVX-NEXT:    xorl %eax, %eax
38; AVX-NEXT:    imull %esi, %edi
39; AVX-NEXT:    seto %al
40; AVX-NEXT:    negl %eax
41; AVX-NEXT:    movl %edi, (%rdx)
42; AVX-NEXT:    retq
43  %t = call {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
44  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
45  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
46  %res = sext <1 x i1> %obit to <1 x i32>
47  store <1 x i32> %val, <1 x i32>* %p2
48  ret <1 x i32> %res
49}
50
51define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
52; SSE2-LABEL: smulo_v2i32:
53; SSE2:       # %bb.0:
54; SSE2-NEXT:    pxor %xmm2, %xmm2
55; SSE2-NEXT:    pxor %xmm3, %xmm3
56; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
57; SSE2-NEXT:    pand %xmm0, %xmm3
58; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
59; SSE2-NEXT:    pand %xmm1, %xmm2
60; SSE2-NEXT:    paddd %xmm3, %xmm2
61; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
62; SSE2-NEXT:    pmuludq %xmm1, %xmm0
63; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
64; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
65; SSE2-NEXT:    pmuludq %xmm3, %xmm1
66; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
67; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
68; SSE2-NEXT:    psubd %xmm2, %xmm4
69; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
70; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
71; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
72; SSE2-NEXT:    movq %xmm0, (%rdi)
73; SSE2-NEXT:    movdqa %xmm0, %xmm1
74; SSE2-NEXT:    psrad $31, %xmm1
75; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
76; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
77; SSE2-NEXT:    pxor %xmm1, %xmm0
78; SSE2-NEXT:    retq
79;
80; SSSE3-LABEL: smulo_v2i32:
81; SSSE3:       # %bb.0:
82; SSSE3-NEXT:    pxor %xmm2, %xmm2
83; SSSE3-NEXT:    pxor %xmm3, %xmm3
84; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
85; SSSE3-NEXT:    pand %xmm0, %xmm3
86; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
87; SSSE3-NEXT:    pand %xmm1, %xmm2
88; SSSE3-NEXT:    paddd %xmm3, %xmm2
89; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
90; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
91; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
92; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
93; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
94; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
95; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
96; SSSE3-NEXT:    psubd %xmm2, %xmm4
97; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
98; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
99; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
100; SSSE3-NEXT:    movq %xmm0, (%rdi)
101; SSSE3-NEXT:    movdqa %xmm0, %xmm1
102; SSSE3-NEXT:    psrad $31, %xmm1
103; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
104; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
105; SSSE3-NEXT:    pxor %xmm1, %xmm0
106; SSSE3-NEXT:    retq
107;
108; SSE41-LABEL: smulo_v2i32:
109; SSE41:       # %bb.0:
110; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
111; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
112; SSE41-NEXT:    pmuldq %xmm2, %xmm3
113; SSE41-NEXT:    movdqa %xmm0, %xmm2
114; SSE41-NEXT:    pmuldq %xmm1, %xmm2
115; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
116; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
117; SSE41-NEXT:    pmulld %xmm1, %xmm0
118; SSE41-NEXT:    movq %xmm0, (%rdi)
119; SSE41-NEXT:    psrad $31, %xmm0
120; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
121; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
122; SSE41-NEXT:    pxor %xmm1, %xmm0
123; SSE41-NEXT:    retq
124;
125; AVX1-LABEL: smulo_v2i32:
126; AVX1:       # %bb.0:
127; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
128; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
129; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
130; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
131; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
132; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
133; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
134; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
135; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
136; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
137; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
138; AVX1-NEXT:    vmovq %xmm1, (%rdi)
139; AVX1-NEXT:    retq
140;
141; AVX2-LABEL: smulo_v2i32:
142; AVX2:       # %bb.0:
143; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
144; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
145; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
146; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
147; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
148; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
149; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
150; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
151; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
152; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
153; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
154; AVX2-NEXT:    vmovq %xmm1, (%rdi)
155; AVX2-NEXT:    retq
156;
157; AVX512-LABEL: smulo_v2i32:
158; AVX512:       # %bb.0:
159; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
160; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
161; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
162; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
163; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
164; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
165; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
166; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
167; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
168; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
169; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
170; AVX512-NEXT:    vmovq %xmm1, (%rdi)
171; AVX512-NEXT:    retq
172  %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
173  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
174  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
175  %res = sext <2 x i1> %obit to <2 x i32>
176  store <2 x i32> %val, <2 x i32>* %p2
177  ret <2 x i32> %res
178}
179
180define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
181; SSE2-LABEL: smulo_v3i32:
182; SSE2:       # %bb.0:
183; SSE2-NEXT:    pxor %xmm2, %xmm2
184; SSE2-NEXT:    pxor %xmm3, %xmm3
185; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
186; SSE2-NEXT:    pand %xmm0, %xmm3
187; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
188; SSE2-NEXT:    pand %xmm1, %xmm2
189; SSE2-NEXT:    paddd %xmm3, %xmm2
190; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
191; SSE2-NEXT:    pmuludq %xmm1, %xmm0
192; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
193; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
194; SSE2-NEXT:    pmuludq %xmm3, %xmm1
195; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
196; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
197; SSE2-NEXT:    psubd %xmm2, %xmm4
198; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
199; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
200; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
201; SSE2-NEXT:    movq %xmm0, (%rdi)
202; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
203; SSE2-NEXT:    movdqa %xmm0, %xmm2
204; SSE2-NEXT:    psrad $31, %xmm2
205; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
206; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
207; SSE2-NEXT:    pxor %xmm2, %xmm0
208; SSE2-NEXT:    movd %xmm1, 8(%rdi)
209; SSE2-NEXT:    retq
210;
211; SSSE3-LABEL: smulo_v3i32:
212; SSSE3:       # %bb.0:
213; SSSE3-NEXT:    pxor %xmm2, %xmm2
214; SSSE3-NEXT:    pxor %xmm3, %xmm3
215; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
216; SSSE3-NEXT:    pand %xmm0, %xmm3
217; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
218; SSSE3-NEXT:    pand %xmm1, %xmm2
219; SSSE3-NEXT:    paddd %xmm3, %xmm2
220; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
221; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
222; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
223; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
224; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
225; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
226; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
227; SSSE3-NEXT:    psubd %xmm2, %xmm4
228; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
229; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
230; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
231; SSSE3-NEXT:    movq %xmm0, (%rdi)
232; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
233; SSSE3-NEXT:    movdqa %xmm0, %xmm2
234; SSSE3-NEXT:    psrad $31, %xmm2
235; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
236; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
237; SSSE3-NEXT:    pxor %xmm2, %xmm0
238; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
239; SSSE3-NEXT:    retq
240;
241; SSE41-LABEL: smulo_v3i32:
242; SSE41:       # %bb.0:
243; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
244; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
245; SSE41-NEXT:    pmuldq %xmm2, %xmm3
246; SSE41-NEXT:    movdqa %xmm0, %xmm2
247; SSE41-NEXT:    pmuldq %xmm1, %xmm2
248; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
249; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
250; SSE41-NEXT:    pmulld %xmm1, %xmm0
251; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
252; SSE41-NEXT:    movq %xmm0, (%rdi)
253; SSE41-NEXT:    psrad $31, %xmm0
254; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
255; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
256; SSE41-NEXT:    pxor %xmm1, %xmm0
257; SSE41-NEXT:    retq
258;
259; AVX1-LABEL: smulo_v3i32:
260; AVX1:       # %bb.0:
261; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
262; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
263; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
264; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
265; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
266; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
267; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
268; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
269; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
270; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
271; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
272; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
273; AVX1-NEXT:    vmovq %xmm1, (%rdi)
274; AVX1-NEXT:    retq
275;
276; AVX2-LABEL: smulo_v3i32:
277; AVX2:       # %bb.0:
278; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
279; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
280; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
281; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
282; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
283; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
284; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
285; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
286; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
287; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
288; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
289; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
290; AVX2-NEXT:    vmovq %xmm1, (%rdi)
291; AVX2-NEXT:    retq
292;
293; AVX512-LABEL: smulo_v3i32:
294; AVX512:       # %bb.0:
295; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
296; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
297; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
298; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
299; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
300; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
301; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
302; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
303; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
304; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
305; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
306; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
307; AVX512-NEXT:    vmovq %xmm1, (%rdi)
308; AVX512-NEXT:    retq
309  %t = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
310  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
311  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
312  %res = sext <3 x i1> %obit to <3 x i32>
313  store <3 x i32> %val, <3 x i32>* %p2
314  ret <3 x i32> %res
315}
316
317define <4 x i32> @smulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
318; SSE2-LABEL: smulo_v4i32:
319; SSE2:       # %bb.0:
320; SSE2-NEXT:    pxor %xmm2, %xmm2
321; SSE2-NEXT:    pxor %xmm3, %xmm3
322; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
323; SSE2-NEXT:    pand %xmm0, %xmm3
324; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
325; SSE2-NEXT:    pand %xmm1, %xmm2
326; SSE2-NEXT:    paddd %xmm3, %xmm2
327; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
328; SSE2-NEXT:    pmuludq %xmm1, %xmm0
329; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
330; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
331; SSE2-NEXT:    pmuludq %xmm3, %xmm1
332; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
333; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
334; SSE2-NEXT:    psubd %xmm2, %xmm4
335; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
336; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
337; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
338; SSE2-NEXT:    movdqa %xmm0, (%rdi)
339; SSE2-NEXT:    movdqa %xmm0, %xmm1
340; SSE2-NEXT:    psrad $31, %xmm1
341; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
342; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
343; SSE2-NEXT:    pxor %xmm1, %xmm0
344; SSE2-NEXT:    retq
345;
346; SSSE3-LABEL: smulo_v4i32:
347; SSSE3:       # %bb.0:
348; SSSE3-NEXT:    pxor %xmm2, %xmm2
349; SSSE3-NEXT:    pxor %xmm3, %xmm3
350; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
351; SSSE3-NEXT:    pand %xmm0, %xmm3
352; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
353; SSSE3-NEXT:    pand %xmm1, %xmm2
354; SSSE3-NEXT:    paddd %xmm3, %xmm2
355; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
356; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
357; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
358; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
359; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
360; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
361; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
362; SSSE3-NEXT:    psubd %xmm2, %xmm4
363; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
364; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
365; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
366; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
367; SSSE3-NEXT:    movdqa %xmm0, %xmm1
368; SSSE3-NEXT:    psrad $31, %xmm1
369; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
370; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
371; SSSE3-NEXT:    pxor %xmm1, %xmm0
372; SSSE3-NEXT:    retq
373;
374; SSE41-LABEL: smulo_v4i32:
375; SSE41:       # %bb.0:
376; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
377; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
378; SSE41-NEXT:    pmuldq %xmm2, %xmm3
379; SSE41-NEXT:    movdqa %xmm0, %xmm2
380; SSE41-NEXT:    pmuldq %xmm1, %xmm2
381; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
382; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
383; SSE41-NEXT:    pmulld %xmm1, %xmm0
384; SSE41-NEXT:    movdqa %xmm0, (%rdi)
385; SSE41-NEXT:    psrad $31, %xmm0
386; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
387; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
388; SSE41-NEXT:    pxor %xmm1, %xmm0
389; SSE41-NEXT:    retq
390;
391; AVX1-LABEL: smulo_v4i32:
392; AVX1:       # %bb.0:
393; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
394; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
395; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
396; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
397; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
398; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
399; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
400; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
401; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
402; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
403; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
404; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
405; AVX1-NEXT:    retq
406;
407; AVX2-LABEL: smulo_v4i32:
408; AVX2:       # %bb.0:
409; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
410; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
411; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
412; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
413; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
414; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
415; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
416; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
417; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
418; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
419; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
420; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
421; AVX2-NEXT:    retq
422;
423; AVX512-LABEL: smulo_v4i32:
424; AVX512:       # %bb.0:
425; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
426; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
427; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
428; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
429; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
430; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
431; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
432; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
433; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
434; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
435; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
436; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
437; AVX512-NEXT:    retq
438  %t = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
439  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
440  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
441  %res = sext <4 x i1> %obit to <4 x i32>
442  store <4 x i32> %val, <4 x i32>* %p2
443  ret <4 x i32> %res
444}
445
446define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
447; SSE2-LABEL: smulo_v6i32:
448; SSE2:       # %bb.0:
449; SSE2-NEXT:    movq %rdi, %rax
450; SSE2-NEXT:    movd %r8d, %xmm8
451; SSE2-NEXT:    movd %ecx, %xmm0
452; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
453; SSE2-NEXT:    movd %edx, %xmm3
454; SSE2-NEXT:    movd %esi, %xmm6
455; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
456; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
457; SSE2-NEXT:    movd {{.*#+}} xmm10 = mem[0],zero,zero,zero
458; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
459; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1]
460; SSE2-NEXT:    movd {{.*#+}} xmm12 = mem[0],zero,zero,zero
461; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
462; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
463; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
464; SSE2-NEXT:    movd %r9d, %xmm13
465; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
466; SSE2-NEXT:    movdqa %xmm13, %xmm11
467; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
468; SSE2-NEXT:    movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
469; SSE2-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
470; SSE2-NEXT:    pmuludq %xmm7, %xmm13
471; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
472; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
473; SSE2-NEXT:    pxor %xmm4, %xmm4
474; SSE2-NEXT:    pxor %xmm5, %xmm5
475; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
476; SSE2-NEXT:    pand %xmm6, %xmm5
477; SSE2-NEXT:    pxor %xmm0, %xmm0
478; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
479; SSE2-NEXT:    pand %xmm1, %xmm0
480; SSE2-NEXT:    paddd %xmm5, %xmm0
481; SSE2-NEXT:    pmuludq %xmm6, %xmm1
482; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,3,2,3]
483; SSE2-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,0],xmm10[0,0]
484; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm8[0,0]
485; SSE2-NEXT:    pmuludq %xmm12, %xmm3
486; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
487; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
488; SSE2-NEXT:    psubd %xmm0, %xmm5
489; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
490; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
491; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
492; SSE2-NEXT:    movdqa %xmm0, (%rcx)
493; SSE2-NEXT:    psrad $31, %xmm0
494; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
495; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
496; SSE2-NEXT:    pxor %xmm1, %xmm0
497; SSE2-NEXT:    pxor %xmm3, %xmm3
498; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
499; SSE2-NEXT:    pand %xmm11, %xmm3
500; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
501; SSE2-NEXT:    pand %xmm7, %xmm4
502; SSE2-NEXT:    paddd %xmm3, %xmm4
503; SSE2-NEXT:    pmuludq %xmm9, %xmm2
504; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
505; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,3,2,3]
506; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
507; SSE2-NEXT:    psubd %xmm4, %xmm5
508; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
509; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3]
510; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
511; SSE2-NEXT:    movq %xmm3, 16(%rcx)
512; SSE2-NEXT:    psrad $31, %xmm3
513; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
514; SSE2-NEXT:    pxor %xmm1, %xmm3
515; SSE2-NEXT:    movq %xmm3, 16(%rdi)
516; SSE2-NEXT:    movdqa %xmm0, (%rdi)
517; SSE2-NEXT:    retq
518;
519; SSSE3-LABEL: smulo_v6i32:
520; SSSE3:       # %bb.0:
521; SSSE3-NEXT:    movq %rdi, %rax
522; SSSE3-NEXT:    movd %r8d, %xmm8
523; SSSE3-NEXT:    movd %ecx, %xmm0
524; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
525; SSSE3-NEXT:    movd %edx, %xmm3
526; SSSE3-NEXT:    movd %esi, %xmm6
527; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
528; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
529; SSSE3-NEXT:    movd {{.*#+}} xmm10 = mem[0],zero,zero,zero
530; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
531; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1]
532; SSSE3-NEXT:    movd {{.*#+}} xmm12 = mem[0],zero,zero,zero
533; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
534; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
535; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
536; SSSE3-NEXT:    movd %r9d, %xmm13
537; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
538; SSSE3-NEXT:    movdqa %xmm13, %xmm11
539; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
540; SSSE3-NEXT:    movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
541; SSSE3-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
542; SSSE3-NEXT:    pmuludq %xmm7, %xmm13
543; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
544; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
545; SSSE3-NEXT:    pxor %xmm4, %xmm4
546; SSSE3-NEXT:    pxor %xmm5, %xmm5
547; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
548; SSSE3-NEXT:    pand %xmm6, %xmm5
549; SSSE3-NEXT:    pxor %xmm0, %xmm0
550; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm0
551; SSSE3-NEXT:    pand %xmm1, %xmm0
552; SSSE3-NEXT:    paddd %xmm5, %xmm0
553; SSSE3-NEXT:    pmuludq %xmm6, %xmm1
554; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,3,2,3]
555; SSSE3-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,0],xmm10[0,0]
556; SSSE3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm8[0,0]
557; SSSE3-NEXT:    pmuludq %xmm12, %xmm3
558; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
559; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
560; SSSE3-NEXT:    psubd %xmm0, %xmm5
561; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
562; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
563; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
564; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
565; SSSE3-NEXT:    psrad $31, %xmm0
566; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
567; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
568; SSSE3-NEXT:    pxor %xmm1, %xmm0
569; SSSE3-NEXT:    pxor %xmm3, %xmm3
570; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
571; SSSE3-NEXT:    pand %xmm11, %xmm3
572; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
573; SSSE3-NEXT:    pand %xmm7, %xmm4
574; SSSE3-NEXT:    paddd %xmm3, %xmm4
575; SSSE3-NEXT:    pmuludq %xmm9, %xmm2
576; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
577; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,3,2,3]
578; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
579; SSSE3-NEXT:    psubd %xmm4, %xmm5
580; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
581; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3]
582; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
583; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
584; SSSE3-NEXT:    psrad $31, %xmm3
585; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
586; SSSE3-NEXT:    pxor %xmm1, %xmm3
587; SSSE3-NEXT:    movq %xmm3, 16(%rdi)
588; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
589; SSSE3-NEXT:    retq
590;
591; SSE41-LABEL: smulo_v6i32:
592; SSE41:       # %bb.0:
593; SSE41-NEXT:    movq %rdi, %rax
594; SSE41-NEXT:    movd %esi, %xmm2
595; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
596; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
597; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
598; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
599; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
600; SSE41-NEXT:    movdqa %xmm1, %xmm0
601; SSE41-NEXT:    pmuldq %xmm2, %xmm1
602; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
603; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
604; SSE41-NEXT:    movd %r9d, %xmm4
605; SSE41-NEXT:    movdqa %xmm4, %xmm5
606; SSE41-NEXT:    pmuldq %xmm3, %xmm4
607; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
608; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
609; SSE41-NEXT:    pmulld %xmm3, %xmm5
610; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
611; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
612; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
613; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
614; SSE41-NEXT:    pmuldq %xmm3, %xmm6
615; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
616; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
617; SSE41-NEXT:    pmulld %xmm2, %xmm0
618; SSE41-NEXT:    movdqa %xmm0, (%rcx)
619; SSE41-NEXT:    psrad $31, %xmm0
620; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
621; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
622; SSE41-NEXT:    pxor %xmm1, %xmm0
623; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
624; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
625; SSE41-NEXT:    pmuldq %xmm2, %xmm3
626; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
627; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
628; SSE41-NEXT:    movq %xmm5, 16(%rcx)
629; SSE41-NEXT:    psrad $31, %xmm5
630; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
631; SSE41-NEXT:    pxor %xmm1, %xmm5
632; SSE41-NEXT:    movq %xmm5, 16(%rdi)
633; SSE41-NEXT:    movdqa %xmm0, (%rdi)
634; SSE41-NEXT:    retq
635;
636; AVX1-LABEL: smulo_v6i32:
637; AVX1:       # %bb.0:
638; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
639; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
640; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
641; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
642; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
643; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
644; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
645; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
646; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
647; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
648; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
649; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
650; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
651; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
652; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
653; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
654; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
655; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
656; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
657; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
658; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
659; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
660; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
661; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
662; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
663; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
664; AVX1-NEXT:    retq
665;
666; AVX2-LABEL: smulo_v6i32:
667; AVX2:       # %bb.0:
668; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
669; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
670; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
671; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
672; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
673; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
674; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
675; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
676; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
677; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
678; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
679; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
680; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
681; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
682; AVX2-NEXT:    retq
683;
684; AVX512-LABEL: smulo_v6i32:
685; AVX512:       # %bb.0:
686; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
687; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
688; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
689; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
690; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
691; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
692; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
693; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
694; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
695; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
696; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
697; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
698; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
699; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
700; AVX512-NEXT:    retq
701  %t = call {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
702  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
703  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
704  %res = sext <6 x i1> %obit to <6 x i32>
705  store <6 x i32> %val, <6 x i32>* %p2
706  ret <6 x i32> %res
707}
708
709define <8 x i32> @smulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
710; SSE2-LABEL: smulo_v8i32:
711; SSE2:       # %bb.0:
712; SSE2-NEXT:    pxor %xmm4, %xmm4
713; SSE2-NEXT:    pxor %xmm5, %xmm5
714; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
715; SSE2-NEXT:    pand %xmm0, %xmm5
716; SSE2-NEXT:    pxor %xmm6, %xmm6
717; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
718; SSE2-NEXT:    pand %xmm2, %xmm6
719; SSE2-NEXT:    paddd %xmm5, %xmm6
720; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
721; SSE2-NEXT:    pmuludq %xmm2, %xmm0
722; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
723; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
724; SSE2-NEXT:    pmuludq %xmm7, %xmm2
725; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
726; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
727; SSE2-NEXT:    psubd %xmm6, %xmm5
728; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
729; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
730; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
731; SSE2-NEXT:    movdqa %xmm0, (%rdi)
732; SSE2-NEXT:    psrad $31, %xmm0
733; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
734; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
735; SSE2-NEXT:    pxor %xmm2, %xmm0
736; SSE2-NEXT:    pxor %xmm5, %xmm5
737; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
738; SSE2-NEXT:    pand %xmm1, %xmm5
739; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
740; SSE2-NEXT:    pand %xmm3, %xmm4
741; SSE2-NEXT:    paddd %xmm5, %xmm4
742; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
743; SSE2-NEXT:    pmuludq %xmm3, %xmm1
744; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
745; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
746; SSE2-NEXT:    pmuludq %xmm5, %xmm3
747; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
748; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
749; SSE2-NEXT:    psubd %xmm4, %xmm6
750; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
751; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
752; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
753; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
754; SSE2-NEXT:    psrad $31, %xmm1
755; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
756; SSE2-NEXT:    pxor %xmm2, %xmm1
757; SSE2-NEXT:    retq
758;
759; SSSE3-LABEL: smulo_v8i32:
760; SSSE3:       # %bb.0:
761; SSSE3-NEXT:    pxor %xmm4, %xmm4
762; SSSE3-NEXT:    pxor %xmm5, %xmm5
763; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
764; SSSE3-NEXT:    pand %xmm0, %xmm5
765; SSSE3-NEXT:    pxor %xmm6, %xmm6
766; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
767; SSSE3-NEXT:    pand %xmm2, %xmm6
768; SSSE3-NEXT:    paddd %xmm5, %xmm6
769; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
770; SSSE3-NEXT:    pmuludq %xmm2, %xmm0
771; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
772; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
773; SSSE3-NEXT:    pmuludq %xmm7, %xmm2
774; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]
775; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
776; SSSE3-NEXT:    psubd %xmm6, %xmm5
777; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
778; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
779; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
780; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
781; SSSE3-NEXT:    psrad $31, %xmm0
782; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
783; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
784; SSSE3-NEXT:    pxor %xmm2, %xmm0
785; SSSE3-NEXT:    pxor %xmm5, %xmm5
786; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
787; SSSE3-NEXT:    pand %xmm1, %xmm5
788; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
789; SSSE3-NEXT:    pand %xmm3, %xmm4
790; SSSE3-NEXT:    paddd %xmm5, %xmm4
791; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
792; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
793; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]
794; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
795; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
796; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
797; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
798; SSSE3-NEXT:    psubd %xmm4, %xmm6
799; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
800; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
801; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
802; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
803; SSSE3-NEXT:    psrad $31, %xmm1
804; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm1
805; SSSE3-NEXT:    pxor %xmm2, %xmm1
806; SSSE3-NEXT:    retq
807;
808; SSE41-LABEL: smulo_v8i32:
809; SSE41:       # %bb.0:
810; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
811; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
812; SSE41-NEXT:    pmuldq %xmm4, %xmm5
813; SSE41-NEXT:    movdqa %xmm0, %xmm4
814; SSE41-NEXT:    pmuldq %xmm2, %xmm4
815; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
816; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
817; SSE41-NEXT:    pmulld %xmm2, %xmm0
818; SSE41-NEXT:    movdqa %xmm0, (%rdi)
819; SSE41-NEXT:    psrad $31, %xmm0
820; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
821; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
822; SSE41-NEXT:    pxor %xmm2, %xmm0
823; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
824; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
825; SSE41-NEXT:    pmuldq %xmm4, %xmm5
826; SSE41-NEXT:    movdqa %xmm1, %xmm4
827; SSE41-NEXT:    pmuldq %xmm3, %xmm4
828; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
829; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
830; SSE41-NEXT:    pmulld %xmm3, %xmm1
831; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
832; SSE41-NEXT:    psrad $31, %xmm1
833; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
834; SSE41-NEXT:    pxor %xmm2, %xmm1
835; SSE41-NEXT:    retq
836;
837; AVX1-LABEL: smulo_v8i32:
838; AVX1:       # %bb.0:
839; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
840; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
841; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
842; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
843; AVX1-NEXT:    vpmuldq %xmm3, %xmm5, %xmm3
844; AVX1-NEXT:    vpmuldq %xmm2, %xmm4, %xmm5
845; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
846; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
847; AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
848; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
849; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
850; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
851; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
852; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
853; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
854; AVX1-NEXT:    vpmuldq %xmm5, %xmm6, %xmm5
855; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm6
856; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
857; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
858; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
859; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
860; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
861; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
862; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
863; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
864; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
865; AVX1-NEXT:    retq
866;
867; AVX2-LABEL: smulo_v8i32:
868; AVX2:       # %bb.0:
869; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
870; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
871; AVX2-NEXT:    vpmuldq %ymm2, %ymm3, %ymm2
872; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm3
873; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
874; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
875; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
876; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm0
877; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
878; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
879; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
880; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
881; AVX2-NEXT:    retq
882;
883; AVX512-LABEL: smulo_v8i32:
884; AVX512:       # %bb.0:
885; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm2
886; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
887; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
888; AVX512-NEXT:    vpmuldq %ymm3, %ymm4, %ymm3
889; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
890; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
891; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
892; AVX512-NEXT:    vpsrad $31, %ymm1, %ymm0
893; AVX512-NEXT:    vpcmpneqd %ymm0, %ymm4, %k1
894; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
895; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
896; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
897; AVX512-NEXT:    retq
898  %t = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
899  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
900  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
901  %res = sext <8 x i1> %obit to <8 x i32>
902  store <8 x i32> %val, <8 x i32>* %p2
903  ret <8 x i32> %res
904}
905
906define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
907; SSE2-LABEL: smulo_v16i32:
908; SSE2:       # %bb.0:
909; SSE2-NEXT:    pxor %xmm8, %xmm8
910; SSE2-NEXT:    pxor %xmm9, %xmm9
911; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
912; SSE2-NEXT:    pand %xmm0, %xmm9
913; SSE2-NEXT:    pxor %xmm10, %xmm10
914; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
915; SSE2-NEXT:    pand %xmm4, %xmm10
916; SSE2-NEXT:    paddd %xmm9, %xmm10
917; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
918; SSE2-NEXT:    pmuludq %xmm4, %xmm0
919; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
920; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
921; SSE2-NEXT:    pmuludq %xmm11, %xmm12
922; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
923; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
924; SSE2-NEXT:    psubd %xmm10, %xmm9
925; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
926; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
927; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
928; SSE2-NEXT:    movdqa %xmm0, (%rdi)
929; SSE2-NEXT:    psrad $31, %xmm0
930; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
931; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
932; SSE2-NEXT:    pxor %xmm9, %xmm0
933; SSE2-NEXT:    pxor %xmm10, %xmm10
934; SSE2-NEXT:    pcmpgtd %xmm5, %xmm10
935; SSE2-NEXT:    pand %xmm1, %xmm10
936; SSE2-NEXT:    pxor %xmm4, %xmm4
937; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
938; SSE2-NEXT:    pand %xmm5, %xmm4
939; SSE2-NEXT:    paddd %xmm10, %xmm4
940; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
941; SSE2-NEXT:    pmuludq %xmm5, %xmm1
942; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
943; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
944; SSE2-NEXT:    pmuludq %xmm10, %xmm12
945; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
946; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
947; SSE2-NEXT:    psubd %xmm4, %xmm11
948; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
949; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
950; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
951; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
952; SSE2-NEXT:    psrad $31, %xmm1
953; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
954; SSE2-NEXT:    pxor %xmm9, %xmm1
955; SSE2-NEXT:    pxor %xmm4, %xmm4
956; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
957; SSE2-NEXT:    pand %xmm2, %xmm4
958; SSE2-NEXT:    pxor %xmm5, %xmm5
959; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
960; SSE2-NEXT:    pand %xmm6, %xmm5
961; SSE2-NEXT:    paddd %xmm4, %xmm5
962; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
963; SSE2-NEXT:    pmuludq %xmm6, %xmm2
964; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
965; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
966; SSE2-NEXT:    pmuludq %xmm10, %xmm11
967; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
968; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
969; SSE2-NEXT:    psubd %xmm5, %xmm4
970; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
971; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
972; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
973; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
974; SSE2-NEXT:    psrad $31, %xmm2
975; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
976; SSE2-NEXT:    pxor %xmm9, %xmm2
977; SSE2-NEXT:    pxor %xmm4, %xmm4
978; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
979; SSE2-NEXT:    pand %xmm3, %xmm4
980; SSE2-NEXT:    pcmpgtd %xmm3, %xmm8
981; SSE2-NEXT:    pand %xmm7, %xmm8
982; SSE2-NEXT:    paddd %xmm4, %xmm8
983; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
984; SSE2-NEXT:    pmuludq %xmm7, %xmm3
985; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
986; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
987; SSE2-NEXT:    pmuludq %xmm4, %xmm6
988; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
989; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
990; SSE2-NEXT:    psubd %xmm8, %xmm5
991; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
992; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
993; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
994; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
995; SSE2-NEXT:    psrad $31, %xmm3
996; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
997; SSE2-NEXT:    pxor %xmm9, %xmm3
998; SSE2-NEXT:    retq
999;
1000; SSSE3-LABEL: smulo_v16i32:
1001; SSSE3:       # %bb.0:
1002; SSSE3-NEXT:    pxor %xmm8, %xmm8
1003; SSSE3-NEXT:    pxor %xmm9, %xmm9
1004; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
1005; SSSE3-NEXT:    pand %xmm0, %xmm9
1006; SSSE3-NEXT:    pxor %xmm10, %xmm10
1007; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm10
1008; SSSE3-NEXT:    pand %xmm4, %xmm10
1009; SSSE3-NEXT:    paddd %xmm9, %xmm10
1010; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
1011; SSSE3-NEXT:    pmuludq %xmm4, %xmm0
1012; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]
1013; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
1014; SSSE3-NEXT:    pmuludq %xmm11, %xmm12
1015; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[1,3,2,3]
1016; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
1017; SSSE3-NEXT:    psubd %xmm10, %xmm9
1018; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1019; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1020; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1021; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
1022; SSSE3-NEXT:    psrad $31, %xmm0
1023; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
1024; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
1025; SSSE3-NEXT:    pxor %xmm9, %xmm0
1026; SSSE3-NEXT:    pxor %xmm10, %xmm10
1027; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm10
1028; SSSE3-NEXT:    pand %xmm1, %xmm10
1029; SSSE3-NEXT:    pxor %xmm4, %xmm4
1030; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
1031; SSSE3-NEXT:    pand %xmm5, %xmm4
1032; SSSE3-NEXT:    paddd %xmm10, %xmm4
1033; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1034; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
1035; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]
1036; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
1037; SSSE3-NEXT:    pmuludq %xmm10, %xmm12
1038; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
1039; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1]
1040; SSSE3-NEXT:    psubd %xmm4, %xmm11
1041; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1042; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1043; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1044; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
1045; SSSE3-NEXT:    psrad $31, %xmm1
1046; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
1047; SSSE3-NEXT:    pxor %xmm9, %xmm1
1048; SSSE3-NEXT:    pxor %xmm4, %xmm4
1049; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
1050; SSSE3-NEXT:    pand %xmm2, %xmm4
1051; SSSE3-NEXT:    pxor %xmm5, %xmm5
1052; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
1053; SSSE3-NEXT:    pand %xmm6, %xmm5
1054; SSSE3-NEXT:    paddd %xmm4, %xmm5
1055; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,3,3]
1056; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
1057; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
1058; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
1059; SSSE3-NEXT:    pmuludq %xmm10, %xmm11
1060; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,3,2,3]
1061; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
1062; SSSE3-NEXT:    psubd %xmm5, %xmm4
1063; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
1064; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
1065; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1066; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
1067; SSSE3-NEXT:    psrad $31, %xmm2
1068; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
1069; SSSE3-NEXT:    pxor %xmm9, %xmm2
1070; SSSE3-NEXT:    pxor %xmm4, %xmm4
1071; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
1072; SSSE3-NEXT:    pand %xmm3, %xmm4
1073; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm8
1074; SSSE3-NEXT:    pand %xmm7, %xmm8
1075; SSSE3-NEXT:    paddd %xmm4, %xmm8
1076; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
1077; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
1078; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
1079; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
1080; SSSE3-NEXT:    pmuludq %xmm4, %xmm6
1081; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,3,2,3]
1082; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
1083; SSSE3-NEXT:    psubd %xmm8, %xmm5
1084; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
1085; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
1086; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1087; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
1088; SSSE3-NEXT:    psrad $31, %xmm3
1089; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
1090; SSSE3-NEXT:    pxor %xmm9, %xmm3
1091; SSSE3-NEXT:    retq
1092;
1093; SSE41-LABEL: smulo_v16i32:
1094; SSE41:       # %bb.0:
1095; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
1096; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
1097; SSE41-NEXT:    pmuldq %xmm8, %xmm9
1098; SSE41-NEXT:    movdqa %xmm0, %xmm8
1099; SSE41-NEXT:    pmuldq %xmm4, %xmm8
1100; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
1101; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
1102; SSE41-NEXT:    pmulld %xmm4, %xmm0
1103; SSE41-NEXT:    movdqa %xmm0, (%rdi)
1104; SSE41-NEXT:    psrad $31, %xmm0
1105; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
1106; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
1107; SSE41-NEXT:    pxor %xmm8, %xmm0
1108; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
1109; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1110; SSE41-NEXT:    pmuldq %xmm9, %xmm10
1111; SSE41-NEXT:    movdqa %xmm1, %xmm4
1112; SSE41-NEXT:    pmuldq %xmm5, %xmm4
1113; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1114; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm10[2,3],xmm4[4,5],xmm10[6,7]
1115; SSE41-NEXT:    pmulld %xmm5, %xmm1
1116; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
1117; SSE41-NEXT:    psrad $31, %xmm1
1118; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
1119; SSE41-NEXT:    pxor %xmm8, %xmm1
1120; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
1121; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
1122; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1123; SSE41-NEXT:    movdqa %xmm2, %xmm4
1124; SSE41-NEXT:    pmuldq %xmm6, %xmm4
1125; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1126; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1127; SSE41-NEXT:    pmulld %xmm6, %xmm2
1128; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
1129; SSE41-NEXT:    psrad $31, %xmm2
1130; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
1131; SSE41-NEXT:    pxor %xmm8, %xmm2
1132; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
1133; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
1134; SSE41-NEXT:    pmuldq %xmm4, %xmm5
1135; SSE41-NEXT:    movdqa %xmm3, %xmm4
1136; SSE41-NEXT:    pmuldq %xmm7, %xmm4
1137; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
1138; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
1139; SSE41-NEXT:    pmulld %xmm7, %xmm3
1140; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
1141; SSE41-NEXT:    psrad $31, %xmm3
1142; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
1143; SSE41-NEXT:    pxor %xmm8, %xmm3
1144; SSE41-NEXT:    retq
1145;
1146; AVX1-LABEL: smulo_v16i32:
1147; AVX1:       # %bb.0:
1148; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
1149; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
1150; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
1151; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
1152; AVX1-NEXT:    vpmuldq %xmm5, %xmm7, %xmm5
1153; AVX1-NEXT:    vpmuldq %xmm4, %xmm6, %xmm7
1154; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1155; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1156; AVX1-NEXT:    vpmulld %xmm4, %xmm6, %xmm8
1157; AVX1-NEXT:    vpsrad $31, %xmm8, %xmm6
1158; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm6
1159; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
1160; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
1161; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
1162; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
1163; AVX1-NEXT:    vpmuldq %xmm7, %xmm4, %xmm4
1164; AVX1-NEXT:    vpmuldq %xmm3, %xmm1, %xmm7
1165; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1166; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3],xmm7[4,5],xmm4[6,7]
1167; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1168; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm1
1169; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
1170; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
1171; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm9
1172; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1173; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
1174; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1175; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm7[1,1,3,3]
1176; AVX1-NEXT:    vpmuldq %xmm6, %xmm1, %xmm1
1177; AVX1-NEXT:    vpmuldq %xmm4, %xmm7, %xmm6
1178; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
1179; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3],xmm6[4,5],xmm1[6,7]
1180; AVX1-NEXT:    vpmulld %xmm4, %xmm7, %xmm4
1181; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm6
1182; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm1, %xmm1
1183; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
1184; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
1185; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
1186; AVX1-NEXT:    vpmuldq %xmm6, %xmm7, %xmm6
1187; AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm7
1188; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1189; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
1190; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1191; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm0
1192; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
1193; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
1194; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
1195; AVX1-NEXT:    vpacksswb %xmm9, %xmm0, %xmm1
1196; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1197; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
1198; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
1199; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
1200; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
1201; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
1202; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
1203; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1204; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1205; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
1206; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1207; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1208; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1209; AVX1-NEXT:    retq
1210;
1211; AVX2-LABEL: smulo_v16i32:
1212; AVX2:       # %bb.0:
1213; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1214; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1215; AVX2-NEXT:    vpmuldq %ymm4, %ymm5, %ymm4
1216; AVX2-NEXT:    vpmuldq %ymm3, %ymm1, %ymm5
1217; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1218; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1219; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1220; AVX2-NEXT:    vpsrad $31, %ymm3, %ymm1
1221; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm4, %ymm1
1222; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
1223; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
1224; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1225; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
1226; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm2[1,1,3,3,5,5,7,7]
1227; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm0[1,1,3,3,5,5,7,7]
1228; AVX2-NEXT:    vpmuldq %ymm5, %ymm6, %ymm5
1229; AVX2-NEXT:    vpmuldq %ymm2, %ymm0, %ymm6
1230; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,1,3,3,5,5,7,7]
1231; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
1232; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1233; AVX2-NEXT:    vpsrad $31, %ymm2, %ymm0
1234; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
1235; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
1236; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
1237; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
1238; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
1239; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1240; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1241; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1242; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1243; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1244; AVX2-NEXT:    retq
1245;
1246; AVX512-LABEL: smulo_v16i32:
1247; AVX512:       # %bb.0:
1248; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm2
1249; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1250; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1251; AVX512-NEXT:    vpmuldq %zmm3, %zmm4, %zmm3
1252; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1253; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1254; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1255; AVX512-NEXT:    vpsrad $31, %zmm1, %zmm0
1256; AVX512-NEXT:    vpcmpneqd %zmm0, %zmm4, %k1
1257; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1258; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1259; AVX512-NEXT:    retq
1260  %t = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1261  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1262  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1263  %res = sext <16 x i1> %obit to <16 x i32>
1264  store <16 x i32> %val, <16 x i32>* %p2
1265  ret <16 x i32> %res
1266}
1267
1268define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1269; SSE2-LABEL: smulo_v16i8:
1270; SSE2:       # %bb.0:
1271; SSE2-NEXT:    movdqa %xmm1, %xmm2
1272; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1273; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1274; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1275; SSE2-NEXT:    movdqa %xmm0, %xmm4
1276; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1277; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1278; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1279; SSE2-NEXT:    pmullw %xmm1, %xmm0
1280; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1281; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1282; SSE2-NEXT:    pmullw %xmm2, %xmm4
1283; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1284; SSE2-NEXT:    pand %xmm1, %xmm0
1285; SSE2-NEXT:    pand %xmm1, %xmm4
1286; SSE2-NEXT:    packuswb %xmm0, %xmm4
1287; SSE2-NEXT:    pxor %xmm0, %xmm0
1288; SSE2-NEXT:    pcmpgtb %xmm4, %xmm0
1289; SSE2-NEXT:    psraw $8, %xmm3
1290; SSE2-NEXT:    psraw $8, %xmm6
1291; SSE2-NEXT:    pmullw %xmm3, %xmm6
1292; SSE2-NEXT:    psrlw $8, %xmm6
1293; SSE2-NEXT:    psraw $8, %xmm5
1294; SSE2-NEXT:    psraw $8, %xmm7
1295; SSE2-NEXT:    pmullw %xmm5, %xmm7
1296; SSE2-NEXT:    psrlw $8, %xmm7
1297; SSE2-NEXT:    packuswb %xmm6, %xmm7
1298; SSE2-NEXT:    pcmpeqb %xmm0, %xmm7
1299; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1300; SSE2-NEXT:    pxor %xmm7, %xmm3
1301; SSE2-NEXT:    movdqa %xmm3, %xmm1
1302; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1303; SSE2-NEXT:    movdqa %xmm1, %xmm0
1304; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1305; SSE2-NEXT:    pslld $31, %xmm0
1306; SSE2-NEXT:    psrad $31, %xmm0
1307; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1308; SSE2-NEXT:    pslld $31, %xmm1
1309; SSE2-NEXT:    psrad $31, %xmm1
1310; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1311; SSE2-NEXT:    movdqa %xmm3, %xmm2
1312; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1313; SSE2-NEXT:    pslld $31, %xmm2
1314; SSE2-NEXT:    psrad $31, %xmm2
1315; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1316; SSE2-NEXT:    pslld $31, %xmm3
1317; SSE2-NEXT:    psrad $31, %xmm3
1318; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1319; SSE2-NEXT:    retq
1320;
1321; SSSE3-LABEL: smulo_v16i8:
1322; SSSE3:       # %bb.0:
1323; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1324; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1325; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1326; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1327; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1328; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
1329; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
1330; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1331; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1332; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1333; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1334; SSSE3-NEXT:    pmullw %xmm2, %xmm4
1335; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1336; SSSE3-NEXT:    pand %xmm1, %xmm0
1337; SSSE3-NEXT:    pand %xmm1, %xmm4
1338; SSSE3-NEXT:    packuswb %xmm0, %xmm4
1339; SSSE3-NEXT:    pxor %xmm0, %xmm0
1340; SSSE3-NEXT:    pcmpgtb %xmm4, %xmm0
1341; SSSE3-NEXT:    psraw $8, %xmm3
1342; SSSE3-NEXT:    psraw $8, %xmm6
1343; SSSE3-NEXT:    pmullw %xmm3, %xmm6
1344; SSSE3-NEXT:    psrlw $8, %xmm6
1345; SSSE3-NEXT:    psraw $8, %xmm5
1346; SSSE3-NEXT:    psraw $8, %xmm7
1347; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1348; SSSE3-NEXT:    psrlw $8, %xmm7
1349; SSSE3-NEXT:    packuswb %xmm6, %xmm7
1350; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm7
1351; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1352; SSSE3-NEXT:    pxor %xmm7, %xmm3
1353; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1354; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1355; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1356; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1357; SSSE3-NEXT:    pslld $31, %xmm0
1358; SSSE3-NEXT:    psrad $31, %xmm0
1359; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1360; SSSE3-NEXT:    pslld $31, %xmm1
1361; SSSE3-NEXT:    psrad $31, %xmm1
1362; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1363; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1364; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1365; SSSE3-NEXT:    pslld $31, %xmm2
1366; SSSE3-NEXT:    psrad $31, %xmm2
1367; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1368; SSSE3-NEXT:    pslld $31, %xmm3
1369; SSSE3-NEXT:    psrad $31, %xmm3
1370; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1371; SSSE3-NEXT:    retq
1372;
1373; SSE41-LABEL: smulo_v16i8:
1374; SSE41:       # %bb.0:
1375; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1376; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
1377; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
1378; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1379; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1380; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
1381; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
1382; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1383; SSE41-NEXT:    pmullw %xmm1, %xmm0
1384; SSE41-NEXT:    pmullw %xmm2, %xmm4
1385; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1386; SSE41-NEXT:    pand %xmm1, %xmm0
1387; SSE41-NEXT:    pand %xmm1, %xmm4
1388; SSE41-NEXT:    packuswb %xmm0, %xmm4
1389; SSE41-NEXT:    pxor %xmm0, %xmm0
1390; SSE41-NEXT:    pcmpgtb %xmm4, %xmm0
1391; SSE41-NEXT:    pmullw %xmm3, %xmm6
1392; SSE41-NEXT:    psrlw $8, %xmm6
1393; SSE41-NEXT:    pmovsxbw %xmm5, %xmm1
1394; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1395; SSE41-NEXT:    pmullw %xmm1, %xmm2
1396; SSE41-NEXT:    psrlw $8, %xmm2
1397; SSE41-NEXT:    packuswb %xmm2, %xmm6
1398; SSE41-NEXT:    pcmpeqb %xmm0, %xmm6
1399; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1400; SSE41-NEXT:    pxor %xmm6, %xmm3
1401; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1402; SSE41-NEXT:    pslld $31, %xmm0
1403; SSE41-NEXT:    psrad $31, %xmm0
1404; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
1405; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1406; SSE41-NEXT:    pslld $31, %xmm1
1407; SSE41-NEXT:    psrad $31, %xmm1
1408; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
1409; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1410; SSE41-NEXT:    pslld $31, %xmm2
1411; SSE41-NEXT:    psrad $31, %xmm2
1412; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
1413; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1414; SSE41-NEXT:    pslld $31, %xmm3
1415; SSE41-NEXT:    psrad $31, %xmm3
1416; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1417; SSE41-NEXT:    retq
1418;
1419; AVX1-LABEL: smulo_v16i8:
1420; AVX1:       # %bb.0:
1421; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1422; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1423; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1424; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1425; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1426; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1427; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1428; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1429; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1430; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1431; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1432; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1433; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm4
1434; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
1435; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1436; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
1437; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1438; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
1439; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1440; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
1441; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1442; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1443; AVX1-NEXT:    vpackuswb %xmm0, %xmm4, %xmm0
1444; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1445; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1446; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1447; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1448; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
1449; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1450; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1451; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
1452; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1453; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
1454; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1455; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1456; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1457; AVX1-NEXT:    retq
1458;
1459; AVX2-LABEL: smulo_v16i8:
1460; AVX2:       # %bb.0:
1461; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm2
1462; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm3
1463; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1464; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1465; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
1466; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
1467; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1468; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1469; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1470; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1471; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1472; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm3
1473; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
1474; AVX2-NEXT:    vpcmpgtb %xmm3, %xmm0, %xmm0
1475; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
1476; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1477; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1478; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1479; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1480; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1481; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
1482; AVX2-NEXT:    retq
1483;
1484; AVX512-LABEL: smulo_v16i8:
1485; AVX512:       # %bb.0:
1486; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1487; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1488; AVX512-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1489; AVX512-NEXT:    vpmovwb %ymm2, %xmm2
1490; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1491; AVX512-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
1492; AVX512-NEXT:    vpmovsxbw %xmm1, %ymm1
1493; AVX512-NEXT:    vpmovsxbw %xmm0, %ymm0
1494; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1495; AVX512-NEXT:    vpsrlw $8, %ymm0, %ymm0
1496; AVX512-NEXT:    vpmovwb %ymm0, %xmm0
1497; AVX512-NEXT:    vpcmpneqb %xmm3, %xmm0, %k1
1498; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1499; AVX512-NEXT:    vmovdqa %xmm2, (%rdi)
1500; AVX512-NEXT:    retq
1501  %t = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1502  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1503  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1504  %res = sext <16 x i1> %obit to <16 x i32>
1505  store <16 x i8> %val, <16 x i8>* %p2
1506  ret <16 x i32> %res
1507}
1508
1509define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
1510; SSE2-LABEL: smulo_v8i16:
1511; SSE2:       # %bb.0:
1512; SSE2-NEXT:    movdqa %xmm0, %xmm2
1513; SSE2-NEXT:    pmulhw %xmm1, %xmm2
1514; SSE2-NEXT:    pmullw %xmm1, %xmm0
1515; SSE2-NEXT:    movdqa %xmm0, (%rdi)
1516; SSE2-NEXT:    psraw $15, %xmm0
1517; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
1518; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
1519; SSE2-NEXT:    pxor %xmm0, %xmm1
1520; SSE2-NEXT:    movdqa %xmm1, %xmm0
1521; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1522; SSE2-NEXT:    pslld $31, %xmm0
1523; SSE2-NEXT:    psrad $31, %xmm0
1524; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1525; SSE2-NEXT:    pslld $31, %xmm1
1526; SSE2-NEXT:    psrad $31, %xmm1
1527; SSE2-NEXT:    retq
1528;
1529; SSSE3-LABEL: smulo_v8i16:
1530; SSSE3:       # %bb.0:
1531; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1532; SSSE3-NEXT:    pmulhw %xmm1, %xmm2
1533; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1534; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
1535; SSSE3-NEXT:    psraw $15, %xmm0
1536; SSSE3-NEXT:    pcmpeqw %xmm2, %xmm0
1537; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
1538; SSSE3-NEXT:    pxor %xmm0, %xmm1
1539; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1540; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1541; SSSE3-NEXT:    pslld $31, %xmm0
1542; SSSE3-NEXT:    psrad $31, %xmm0
1543; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1544; SSSE3-NEXT:    pslld $31, %xmm1
1545; SSSE3-NEXT:    psrad $31, %xmm1
1546; SSSE3-NEXT:    retq
1547;
1548; SSE41-LABEL: smulo_v8i16:
1549; SSE41:       # %bb.0:
1550; SSE41-NEXT:    movdqa %xmm0, %xmm2
1551; SSE41-NEXT:    pmulhw %xmm1, %xmm2
1552; SSE41-NEXT:    pmullw %xmm1, %xmm0
1553; SSE41-NEXT:    movdqa %xmm0, (%rdi)
1554; SSE41-NEXT:    psraw $15, %xmm0
1555; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
1556; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
1557; SSE41-NEXT:    pxor %xmm0, %xmm1
1558; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1559; SSE41-NEXT:    pslld $31, %xmm0
1560; SSE41-NEXT:    psrad $31, %xmm0
1561; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1562; SSE41-NEXT:    pslld $31, %xmm1
1563; SSE41-NEXT:    psrad $31, %xmm1
1564; SSE41-NEXT:    retq
1565;
1566; AVX1-LABEL: smulo_v8i16:
1567; AVX1:       # %bb.0:
1568; AVX1-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
1569; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
1570; AVX1-NEXT:    vpsraw $15, %xmm1, %xmm0
1571; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
1572; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
1573; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
1574; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
1575; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1576; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
1577; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
1578; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
1579; AVX1-NEXT:    retq
1580;
1581; AVX2-LABEL: smulo_v8i16:
1582; AVX2:       # %bb.0:
1583; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
1584; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
1585; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm0
1586; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm0
1587; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
1588; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
1589; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
1590; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
1591; AVX2-NEXT:    retq
1592;
1593; AVX512-LABEL: smulo_v8i16:
1594; AVX512:       # %bb.0:
1595; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2
1596; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
1597; AVX512-NEXT:    vpsraw $15, %xmm1, %xmm0
1598; AVX512-NEXT:    vpcmpneqw %xmm0, %xmm2, %k1
1599; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
1600; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
1601; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
1602; AVX512-NEXT:    retq
1603  %t = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
1604  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
1605  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
1606  %res = sext <8 x i1> %obit to <8 x i32>
1607  store <8 x i16> %val, <8 x i16>* %p2
1608  ret <8 x i32> %res
1609}
1610
1611define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
1612; SSE2-LABEL: smulo_v2i64:
1613; SSE2:       # %bb.0:
1614; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1615; SSE2-NEXT:    movq %xmm2, %r8
1616; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1617; SSE2-NEXT:    movq %xmm2, %rcx
1618; SSE2-NEXT:    movq %xmm1, %rdx
1619; SSE2-NEXT:    movq %xmm0, %rsi
1620; SSE2-NEXT:    xorl %eax, %eax
1621; SSE2-NEXT:    imulq %rdx, %rsi
1622; SSE2-NEXT:    movq $-1, %r9
1623; SSE2-NEXT:    movl $0, %edx
1624; SSE2-NEXT:    cmovoq %r9, %rdx
1625; SSE2-NEXT:    movq %rsi, %xmm1
1626; SSE2-NEXT:    imulq %r8, %rcx
1627; SSE2-NEXT:    movq %rcx, %xmm0
1628; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1629; SSE2-NEXT:    movq %rdx, %xmm0
1630; SSE2-NEXT:    cmovoq %r9, %rax
1631; SSE2-NEXT:    movq %rax, %xmm2
1632; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1633; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1634; SSE2-NEXT:    movdqa %xmm1, (%rdi)
1635; SSE2-NEXT:    retq
1636;
1637; SSSE3-LABEL: smulo_v2i64:
1638; SSSE3:       # %bb.0:
1639; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1640; SSSE3-NEXT:    movq %xmm2, %r8
1641; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1642; SSSE3-NEXT:    movq %xmm2, %rcx
1643; SSSE3-NEXT:    movq %xmm1, %rdx
1644; SSSE3-NEXT:    movq %xmm0, %rsi
1645; SSSE3-NEXT:    xorl %eax, %eax
1646; SSSE3-NEXT:    imulq %rdx, %rsi
1647; SSSE3-NEXT:    movq $-1, %r9
1648; SSSE3-NEXT:    movl $0, %edx
1649; SSSE3-NEXT:    cmovoq %r9, %rdx
1650; SSSE3-NEXT:    movq %rsi, %xmm1
1651; SSSE3-NEXT:    imulq %r8, %rcx
1652; SSSE3-NEXT:    movq %rcx, %xmm0
1653; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1654; SSSE3-NEXT:    movq %rdx, %xmm0
1655; SSSE3-NEXT:    cmovoq %r9, %rax
1656; SSSE3-NEXT:    movq %rax, %xmm2
1657; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1658; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1659; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
1660; SSSE3-NEXT:    retq
1661;
1662; SSE41-LABEL: smulo_v2i64:
1663; SSE41:       # %bb.0:
1664; SSE41-NEXT:    movq %xmm1, %r8
1665; SSE41-NEXT:    movq %xmm0, %rcx
1666; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
1667; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
1668; SSE41-NEXT:    xorl %eax, %eax
1669; SSE41-NEXT:    imulq %rdx, %rsi
1670; SSE41-NEXT:    movq $-1, %r9
1671; SSE41-NEXT:    movl $0, %edx
1672; SSE41-NEXT:    cmovoq %r9, %rdx
1673; SSE41-NEXT:    movq %rsi, %xmm0
1674; SSE41-NEXT:    imulq %r8, %rcx
1675; SSE41-NEXT:    movq %rcx, %xmm1
1676; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1677; SSE41-NEXT:    movq %rdx, %xmm0
1678; SSE41-NEXT:    cmovoq %r9, %rax
1679; SSE41-NEXT:    movq %rax, %xmm2
1680; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
1681; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
1682; SSE41-NEXT:    movdqa %xmm1, (%rdi)
1683; SSE41-NEXT:    retq
1684;
1685; AVX1-LABEL: smulo_v2i64:
1686; AVX1:       # %bb.0:
1687; AVX1-NEXT:    vmovq %xmm1, %r8
1688; AVX1-NEXT:    vmovq %xmm0, %rcx
1689; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
1690; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
1691; AVX1-NEXT:    xorl %eax, %eax
1692; AVX1-NEXT:    imulq %rdx, %rsi
1693; AVX1-NEXT:    movq $-1, %r9
1694; AVX1-NEXT:    movl $0, %edx
1695; AVX1-NEXT:    cmovoq %r9, %rdx
1696; AVX1-NEXT:    vmovq %rsi, %xmm0
1697; AVX1-NEXT:    imulq %r8, %rcx
1698; AVX1-NEXT:    vmovq %rcx, %xmm1
1699; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1700; AVX1-NEXT:    vmovq %rdx, %xmm0
1701; AVX1-NEXT:    cmovoq %r9, %rax
1702; AVX1-NEXT:    vmovq %rax, %xmm2
1703; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1704; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1705; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
1706; AVX1-NEXT:    retq
1707;
1708; AVX2-LABEL: smulo_v2i64:
1709; AVX2:       # %bb.0:
1710; AVX2-NEXT:    vmovq %xmm1, %r8
1711; AVX2-NEXT:    vmovq %xmm0, %rcx
1712; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
1713; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
1714; AVX2-NEXT:    xorl %eax, %eax
1715; AVX2-NEXT:    imulq %rdx, %rsi
1716; AVX2-NEXT:    movq $-1, %r9
1717; AVX2-NEXT:    movl $0, %edx
1718; AVX2-NEXT:    cmovoq %r9, %rdx
1719; AVX2-NEXT:    vmovq %rsi, %xmm0
1720; AVX2-NEXT:    imulq %r8, %rcx
1721; AVX2-NEXT:    vmovq %rcx, %xmm1
1722; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1723; AVX2-NEXT:    vmovq %rdx, %xmm0
1724; AVX2-NEXT:    cmovoq %r9, %rax
1725; AVX2-NEXT:    vmovq %rax, %xmm2
1726; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1727; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1728; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
1729; AVX2-NEXT:    retq
1730;
1731; AVX512-LABEL: smulo_v2i64:
1732; AVX512:       # %bb.0:
1733; AVX512-NEXT:    vpextrq $1, %xmm1, %rax
1734; AVX512-NEXT:    vpextrq $1, %xmm0, %rcx
1735; AVX512-NEXT:    vmovq %xmm1, %rdx
1736; AVX512-NEXT:    vmovq %xmm0, %rsi
1737; AVX512-NEXT:    imulq %rdx, %rsi
1738; AVX512-NEXT:    seto %dl
1739; AVX512-NEXT:    imulq %rax, %rcx
1740; AVX512-NEXT:    vmovq %rcx, %xmm0
1741; AVX512-NEXT:    vmovq %rsi, %xmm1
1742; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1743; AVX512-NEXT:    seto %al
1744; AVX512-NEXT:    kmovd %eax, %k0
1745; AVX512-NEXT:    kmovd %edx, %k1
1746; AVX512-NEXT:    kshiftrw $1, %k1, %k2
1747; AVX512-NEXT:    kxorw %k0, %k2, %k0
1748; AVX512-NEXT:    kshiftlw $15, %k0, %k0
1749; AVX512-NEXT:    kshiftrw $14, %k0, %k0
1750; AVX512-NEXT:    kxorw %k0, %k1, %k1
1751; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
1752; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1753; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
1754; AVX512-NEXT:    retq
1755  %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
1756  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
1757  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
1758  %res = sext <2 x i1> %obit to <2 x i32>
1759  store <2 x i64> %val, <2 x i64>* %p2
1760  ret <2 x i32> %res
1761}
1762
1763define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
1764; SSE2-LABEL: smulo_v4i24:
1765; SSE2:       # %bb.0:
1766; SSE2-NEXT:    pslld $8, %xmm0
1767; SSE2-NEXT:    psrad $8, %xmm0
1768; SSE2-NEXT:    pslld $8, %xmm1
1769; SSE2-NEXT:    psrad $8, %xmm1
1770; SSE2-NEXT:    pxor %xmm4, %xmm4
1771; SSE2-NEXT:    pxor %xmm2, %xmm2
1772; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1773; SSE2-NEXT:    pand %xmm0, %xmm2
1774; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
1775; SSE2-NEXT:    pand %xmm1, %xmm4
1776; SSE2-NEXT:    paddd %xmm2, %xmm4
1777; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
1778; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1779; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1780; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1781; SSE2-NEXT:    pmuludq %xmm5, %xmm2
1782; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
1783; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1784; SSE2-NEXT:    psubd %xmm4, %xmm3
1785; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
1786; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
1787; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1788; SSE2-NEXT:    movdqa %xmm4, %xmm1
1789; SSE2-NEXT:    pslld $8, %xmm1
1790; SSE2-NEXT:    psrad $8, %xmm1
1791; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
1792; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]
1793; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
1794; SSE2-NEXT:    psrad $31, %xmm4
1795; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
1796; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1797; SSE2-NEXT:    pxor %xmm3, %xmm4
1798; SSE2-NEXT:    pxor %xmm3, %xmm1
1799; SSE2-NEXT:    por %xmm4, %xmm1
1800; SSE2-NEXT:    movd %xmm0, %eax
1801; SSE2-NEXT:    movw %ax, (%rdi)
1802; SSE2-NEXT:    movd %xmm2, %ecx
1803; SSE2-NEXT:    movw %cx, 3(%rdi)
1804; SSE2-NEXT:    shrl $16, %eax
1805; SSE2-NEXT:    movb %al, 2(%rdi)
1806; SSE2-NEXT:    shrl $16, %ecx
1807; SSE2-NEXT:    movb %cl, 5(%rdi)
1808; SSE2-NEXT:    movd %xmm5, %eax
1809; SSE2-NEXT:    movw %ax, 9(%rdi)
1810; SSE2-NEXT:    movd %xmm6, %ecx
1811; SSE2-NEXT:    movw %cx, 6(%rdi)
1812; SSE2-NEXT:    shrl $16, %eax
1813; SSE2-NEXT:    movb %al, 11(%rdi)
1814; SSE2-NEXT:    shrl $16, %ecx
1815; SSE2-NEXT:    movb %cl, 8(%rdi)
1816; SSE2-NEXT:    movdqa %xmm1, %xmm0
1817; SSE2-NEXT:    retq
1818;
1819; SSSE3-LABEL: smulo_v4i24:
1820; SSSE3:       # %bb.0:
1821; SSSE3-NEXT:    pslld $8, %xmm0
1822; SSSE3-NEXT:    psrad $8, %xmm0
1823; SSSE3-NEXT:    pslld $8, %xmm1
1824; SSSE3-NEXT:    psrad $8, %xmm1
1825; SSSE3-NEXT:    pxor %xmm4, %xmm4
1826; SSSE3-NEXT:    pxor %xmm2, %xmm2
1827; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1828; SSSE3-NEXT:    pand %xmm0, %xmm2
1829; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
1830; SSSE3-NEXT:    pand %xmm1, %xmm4
1831; SSSE3-NEXT:    paddd %xmm2, %xmm4
1832; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
1833; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
1834; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1835; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1836; SSSE3-NEXT:    pmuludq %xmm5, %xmm2
1837; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
1838; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1839; SSSE3-NEXT:    psubd %xmm4, %xmm3
1840; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
1841; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
1842; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1843; SSSE3-NEXT:    movdqa %xmm4, %xmm1
1844; SSSE3-NEXT:    pslld $8, %xmm1
1845; SSSE3-NEXT:    psrad $8, %xmm1
1846; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
1847; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]
1848; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
1849; SSSE3-NEXT:    psrad $31, %xmm4
1850; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
1851; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1852; SSSE3-NEXT:    pxor %xmm3, %xmm4
1853; SSSE3-NEXT:    pxor %xmm3, %xmm1
1854; SSSE3-NEXT:    por %xmm4, %xmm1
1855; SSSE3-NEXT:    movd %xmm0, %eax
1856; SSSE3-NEXT:    movw %ax, (%rdi)
1857; SSSE3-NEXT:    movd %xmm2, %ecx
1858; SSSE3-NEXT:    movw %cx, 3(%rdi)
1859; SSSE3-NEXT:    shrl $16, %eax
1860; SSSE3-NEXT:    movb %al, 2(%rdi)
1861; SSSE3-NEXT:    shrl $16, %ecx
1862; SSSE3-NEXT:    movb %cl, 5(%rdi)
1863; SSSE3-NEXT:    movd %xmm5, %eax
1864; SSSE3-NEXT:    movw %ax, 9(%rdi)
1865; SSSE3-NEXT:    movd %xmm6, %ecx
1866; SSSE3-NEXT:    movw %cx, 6(%rdi)
1867; SSSE3-NEXT:    shrl $16, %eax
1868; SSSE3-NEXT:    movb %al, 11(%rdi)
1869; SSSE3-NEXT:    shrl $16, %ecx
1870; SSSE3-NEXT:    movb %cl, 8(%rdi)
1871; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1872; SSSE3-NEXT:    retq
1873;
1874; SSE41-LABEL: smulo_v4i24:
1875; SSE41:       # %bb.0:
1876; SSE41-NEXT:    pslld $8, %xmm0
1877; SSE41-NEXT:    psrad $8, %xmm0
1878; SSE41-NEXT:    pslld $8, %xmm1
1879; SSE41-NEXT:    psrad $8, %xmm1
1880; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1881; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1882; SSE41-NEXT:    pmuldq %xmm2, %xmm3
1883; SSE41-NEXT:    movdqa %xmm0, %xmm2
1884; SSE41-NEXT:    pmuldq %xmm1, %xmm2
1885; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
1886; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
1887; SSE41-NEXT:    pmulld %xmm0, %xmm1
1888; SSE41-NEXT:    movdqa %xmm1, %xmm3
1889; SSE41-NEXT:    pslld $8, %xmm3
1890; SSE41-NEXT:    psrad $8, %xmm3
1891; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
1892; SSE41-NEXT:    pextrd $3, %xmm1, %eax
1893; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
1894; SSE41-NEXT:    pextrd $1, %xmm1, %edx
1895; SSE41-NEXT:    movd %xmm1, %esi
1896; SSE41-NEXT:    psrad $31, %xmm1
1897; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
1898; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
1899; SSE41-NEXT:    pxor %xmm0, %xmm1
1900; SSE41-NEXT:    pxor %xmm3, %xmm0
1901; SSE41-NEXT:    por %xmm1, %xmm0
1902; SSE41-NEXT:    movw %ax, 9(%rdi)
1903; SSE41-NEXT:    movw %cx, 6(%rdi)
1904; SSE41-NEXT:    movw %dx, 3(%rdi)
1905; SSE41-NEXT:    movw %si, (%rdi)
1906; SSE41-NEXT:    shrl $16, %eax
1907; SSE41-NEXT:    movb %al, 11(%rdi)
1908; SSE41-NEXT:    shrl $16, %ecx
1909; SSE41-NEXT:    movb %cl, 8(%rdi)
1910; SSE41-NEXT:    shrl $16, %edx
1911; SSE41-NEXT:    movb %dl, 5(%rdi)
1912; SSE41-NEXT:    shrl $16, %esi
1913; SSE41-NEXT:    movb %sil, 2(%rdi)
1914; SSE41-NEXT:    retq
1915;
1916; AVX1-LABEL: smulo_v4i24:
1917; AVX1:       # %bb.0:
1918; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
1919; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
1920; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
1921; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
1922; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1923; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1924; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
1925; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
1926; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1927; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1928; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1929; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
1930; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
1931; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
1932; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
1933; AVX1-NEXT:    vpslld $8, %xmm1, %xmm3
1934; AVX1-NEXT:    vpsrad $8, %xmm3, %xmm3
1935; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
1936; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
1937; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
1938; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
1939; AVX1-NEXT:    movw %ax, 9(%rdi)
1940; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
1941; AVX1-NEXT:    movw %cx, 6(%rdi)
1942; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
1943; AVX1-NEXT:    movw %dx, 3(%rdi)
1944; AVX1-NEXT:    vmovd %xmm1, %esi
1945; AVX1-NEXT:    movw %si, (%rdi)
1946; AVX1-NEXT:    shrl $16, %eax
1947; AVX1-NEXT:    movb %al, 11(%rdi)
1948; AVX1-NEXT:    shrl $16, %ecx
1949; AVX1-NEXT:    movb %cl, 8(%rdi)
1950; AVX1-NEXT:    shrl $16, %edx
1951; AVX1-NEXT:    movb %dl, 5(%rdi)
1952; AVX1-NEXT:    shrl $16, %esi
1953; AVX1-NEXT:    movb %sil, 2(%rdi)
1954; AVX1-NEXT:    retq
1955;
1956; AVX2-LABEL: smulo_v4i24:
1957; AVX2:       # %bb.0:
1958; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
1959; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
1960; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
1961; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
1962; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1963; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1964; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
1965; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
1966; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1967; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
1968; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1969; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
1970; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
1971; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
1972; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
1973; AVX2-NEXT:    vpslld $8, %xmm1, %xmm3
1974; AVX2-NEXT:    vpsrad $8, %xmm3, %xmm3
1975; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm3
1976; AVX2-NEXT:    vpxor %xmm2, %xmm3, %xmm2
1977; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
1978; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
1979; AVX2-NEXT:    movw %ax, 9(%rdi)
1980; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
1981; AVX2-NEXT:    movw %cx, 6(%rdi)
1982; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
1983; AVX2-NEXT:    movw %dx, 3(%rdi)
1984; AVX2-NEXT:    vmovd %xmm1, %esi
1985; AVX2-NEXT:    movw %si, (%rdi)
1986; AVX2-NEXT:    shrl $16, %eax
1987; AVX2-NEXT:    movb %al, 11(%rdi)
1988; AVX2-NEXT:    shrl $16, %ecx
1989; AVX2-NEXT:    movb %cl, 8(%rdi)
1990; AVX2-NEXT:    shrl $16, %edx
1991; AVX2-NEXT:    movb %dl, 5(%rdi)
1992; AVX2-NEXT:    shrl $16, %esi
1993; AVX2-NEXT:    movb %sil, 2(%rdi)
1994; AVX2-NEXT:    retq
1995;
1996; AVX512-LABEL: smulo_v4i24:
1997; AVX512:       # %bb.0:
1998; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
1999; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
2000; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
2001; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
2002; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
2003; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
2004; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
2005; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
2006; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
2007; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
2008; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
2009; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
2010; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k0
2011; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
2012; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
2013; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k1
2014; AVX512-NEXT:    korw %k0, %k1, %k1
2015; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2016; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2017; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
2018; AVX512-NEXT:    movw %ax, 9(%rdi)
2019; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
2020; AVX512-NEXT:    movw %cx, 6(%rdi)
2021; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
2022; AVX512-NEXT:    movw %dx, 3(%rdi)
2023; AVX512-NEXT:    vmovd %xmm1, %esi
2024; AVX512-NEXT:    movw %si, (%rdi)
2025; AVX512-NEXT:    shrl $16, %eax
2026; AVX512-NEXT:    movb %al, 11(%rdi)
2027; AVX512-NEXT:    shrl $16, %ecx
2028; AVX512-NEXT:    movb %cl, 8(%rdi)
2029; AVX512-NEXT:    shrl $16, %edx
2030; AVX512-NEXT:    movb %dl, 5(%rdi)
2031; AVX512-NEXT:    shrl $16, %esi
2032; AVX512-NEXT:    movb %sil, 2(%rdi)
2033; AVX512-NEXT:    retq
2034  %t = call {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
2035  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
2036  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
2037  %res = sext <4 x i1> %obit to <4 x i32>
2038  store <4 x i24> %val, <4 x i24>* %p2
2039  ret <4 x i32> %res
2040}
2041
2042define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
2043; SSE2-LABEL: smulo_v4i1:
2044; SSE2:       # %bb.0:
2045; SSE2-NEXT:    pslld $31, %xmm0
2046; SSE2-NEXT:    psrad $31, %xmm0
2047; SSE2-NEXT:    pslld $31, %xmm1
2048; SSE2-NEXT:    psrad $31, %xmm1
2049; SSE2-NEXT:    pxor %xmm2, %xmm2
2050; SSE2-NEXT:    pxor %xmm3, %xmm3
2051; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
2052; SSE2-NEXT:    pand %xmm0, %xmm3
2053; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
2054; SSE2-NEXT:    pand %xmm1, %xmm2
2055; SSE2-NEXT:    paddd %xmm3, %xmm2
2056; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2057; SSE2-NEXT:    pmuludq %xmm1, %xmm0
2058; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
2059; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2060; SSE2-NEXT:    pmuludq %xmm3, %xmm1
2061; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
2062; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2063; SSE2-NEXT:    psubd %xmm2, %xmm4
2064; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2065; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2066; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2067; SSE2-NEXT:    movdqa %xmm0, %xmm1
2068; SSE2-NEXT:    psrad $31, %xmm1
2069; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
2070; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
2071; SSE2-NEXT:    pxor %xmm2, %xmm1
2072; SSE2-NEXT:    movdqa %xmm0, %xmm3
2073; SSE2-NEXT:    pslld $31, %xmm3
2074; SSE2-NEXT:    psrad $31, %xmm3
2075; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
2076; SSE2-NEXT:    pxor %xmm2, %xmm0
2077; SSE2-NEXT:    por %xmm1, %xmm0
2078; SSE2-NEXT:    movmskps %xmm3, %eax
2079; SSE2-NEXT:    movb %al, (%rdi)
2080; SSE2-NEXT:    retq
2081;
2082; SSSE3-LABEL: smulo_v4i1:
2083; SSSE3:       # %bb.0:
2084; SSSE3-NEXT:    pslld $31, %xmm0
2085; SSSE3-NEXT:    psrad $31, %xmm0
2086; SSSE3-NEXT:    pslld $31, %xmm1
2087; SSSE3-NEXT:    psrad $31, %xmm1
2088; SSSE3-NEXT:    pxor %xmm2, %xmm2
2089; SSSE3-NEXT:    pxor %xmm3, %xmm3
2090; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
2091; SSSE3-NEXT:    pand %xmm0, %xmm3
2092; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
2093; SSSE3-NEXT:    pand %xmm1, %xmm2
2094; SSSE3-NEXT:    paddd %xmm3, %xmm2
2095; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2096; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
2097; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
2098; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2099; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
2100; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
2101; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2102; SSSE3-NEXT:    psubd %xmm2, %xmm4
2103; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2104; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2105; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2106; SSSE3-NEXT:    movdqa %xmm0, %xmm1
2107; SSSE3-NEXT:    psrad $31, %xmm1
2108; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
2109; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
2110; SSSE3-NEXT:    pxor %xmm2, %xmm1
2111; SSSE3-NEXT:    movdqa %xmm0, %xmm3
2112; SSSE3-NEXT:    pslld $31, %xmm3
2113; SSSE3-NEXT:    psrad $31, %xmm3
2114; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
2115; SSSE3-NEXT:    pxor %xmm2, %xmm0
2116; SSSE3-NEXT:    por %xmm1, %xmm0
2117; SSSE3-NEXT:    movmskps %xmm3, %eax
2118; SSSE3-NEXT:    movb %al, (%rdi)
2119; SSSE3-NEXT:    retq
2120;
2121; SSE41-LABEL: smulo_v4i1:
2122; SSE41:       # %bb.0:
2123; SSE41-NEXT:    pslld $31, %xmm0
2124; SSE41-NEXT:    psrad $31, %xmm0
2125; SSE41-NEXT:    pslld $31, %xmm1
2126; SSE41-NEXT:    psrad $31, %xmm1
2127; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
2128; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2129; SSE41-NEXT:    pmuldq %xmm2, %xmm3
2130; SSE41-NEXT:    movdqa %xmm0, %xmm2
2131; SSE41-NEXT:    pmuldq %xmm1, %xmm2
2132; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
2133; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
2134; SSE41-NEXT:    pmulld %xmm0, %xmm1
2135; SSE41-NEXT:    movdqa %xmm1, %xmm3
2136; SSE41-NEXT:    psrad $31, %xmm3
2137; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
2138; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2139; SSE41-NEXT:    pxor %xmm0, %xmm3
2140; SSE41-NEXT:    movdqa %xmm1, %xmm2
2141; SSE41-NEXT:    pslld $31, %xmm2
2142; SSE41-NEXT:    psrad $31, %xmm2
2143; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
2144; SSE41-NEXT:    pxor %xmm1, %xmm0
2145; SSE41-NEXT:    por %xmm3, %xmm0
2146; SSE41-NEXT:    movmskps %xmm2, %eax
2147; SSE41-NEXT:    movb %al, (%rdi)
2148; SSE41-NEXT:    retq
2149;
2150; AVX1-LABEL: smulo_v4i1:
2151; AVX1:       # %bb.0:
2152; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
2153; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
2154; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
2155; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
2156; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
2157; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2158; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
2159; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
2160; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
2161; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
2162; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
2163; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
2164; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
2165; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
2166; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
2167; AVX1-NEXT:    vpslld $31, %xmm0, %xmm3
2168; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
2169; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm3, %xmm0
2170; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
2171; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
2172; AVX1-NEXT:    vmovmskps %xmm3, %eax
2173; AVX1-NEXT:    movb %al, (%rdi)
2174; AVX1-NEXT:    retq
2175;
2176; AVX2-LABEL: smulo_v4i1:
2177; AVX2:       # %bb.0:
2178; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
2179; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
2180; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
2181; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
2182; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
2183; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2184; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
2185; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
2186; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
2187; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
2188; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
2189; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
2190; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
2191; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
2192; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
2193; AVX2-NEXT:    vpslld $31, %xmm0, %xmm3
2194; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
2195; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm3, %xmm0
2196; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
2197; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
2198; AVX2-NEXT:    vmovmskps %xmm3, %eax
2199; AVX2-NEXT:    movb %al, (%rdi)
2200; AVX2-NEXT:    retq
2201;
2202; AVX512-LABEL: smulo_v4i1:
2203; AVX512:       # %bb.0:
2204; AVX512-NEXT:    pushq %rbx
2205; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
2206; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k0
2207; AVX512-NEXT:    kshiftrw $3, %k0, %k1
2208; AVX512-NEXT:    kmovd %k1, %r9d
2209; AVX512-NEXT:    andb $1, %r9b
2210; AVX512-NEXT:    negb %r9b
2211; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
2212; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
2213; AVX512-NEXT:    kshiftrw $3, %k1, %k2
2214; AVX512-NEXT:    kmovd %k2, %r10d
2215; AVX512-NEXT:    andb $1, %r10b
2216; AVX512-NEXT:    negb %r10b
2217; AVX512-NEXT:    kshiftrw $2, %k1, %k2
2218; AVX512-NEXT:    kmovd %k2, %r11d
2219; AVX512-NEXT:    andb $1, %r11b
2220; AVX512-NEXT:    negb %r11b
2221; AVX512-NEXT:    kshiftrw $2, %k0, %k2
2222; AVX512-NEXT:    kmovd %k2, %ebx
2223; AVX512-NEXT:    andb $1, %bl
2224; AVX512-NEXT:    negb %bl
2225; AVX512-NEXT:    kshiftrw $1, %k0, %k2
2226; AVX512-NEXT:    kmovd %k2, %esi
2227; AVX512-NEXT:    andb $1, %sil
2228; AVX512-NEXT:    negb %sil
2229; AVX512-NEXT:    kshiftrw $1, %k1, %k2
2230; AVX512-NEXT:    kmovd %k2, %edx
2231; AVX512-NEXT:    andb $1, %dl
2232; AVX512-NEXT:    negb %dl
2233; AVX512-NEXT:    kmovd %k1, %eax
2234; AVX512-NEXT:    andb $1, %al
2235; AVX512-NEXT:    negb %al
2236; AVX512-NEXT:    kmovd %k0, %ecx
2237; AVX512-NEXT:    andb $1, %cl
2238; AVX512-NEXT:    negb %cl
2239; AVX512-NEXT:    # kill: def $al killed $al killed $eax
2240; AVX512-NEXT:    imulb %cl
2241; AVX512-NEXT:    movl %eax, %r8d
2242; AVX512-NEXT:    seto %al
2243; AVX512-NEXT:    movl %r8d, %ecx
2244; AVX512-NEXT:    andb $1, %cl
2245; AVX512-NEXT:    negb %cl
2246; AVX512-NEXT:    cmpb %r8b, %cl
2247; AVX512-NEXT:    setne %cl
2248; AVX512-NEXT:    orb %al, %cl
2249; AVX512-NEXT:    setne %al
2250; AVX512-NEXT:    kmovd %eax, %k0
2251; AVX512-NEXT:    kshiftrw $1, %k0, %k1
2252; AVX512-NEXT:    movl %edx, %eax
2253; AVX512-NEXT:    imulb %sil
2254; AVX512-NEXT:    movl %eax, %edx
2255; AVX512-NEXT:    seto %al
2256; AVX512-NEXT:    movl %edx, %ecx
2257; AVX512-NEXT:    andb $1, %cl
2258; AVX512-NEXT:    negb %cl
2259; AVX512-NEXT:    cmpb %dl, %cl
2260; AVX512-NEXT:    setne %cl
2261; AVX512-NEXT:    orb %al, %cl
2262; AVX512-NEXT:    setne %al
2263; AVX512-NEXT:    kmovd %eax, %k2
2264; AVX512-NEXT:    kxorw %k2, %k1, %k1
2265; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2266; AVX512-NEXT:    kshiftrw $14, %k1, %k1
2267; AVX512-NEXT:    kxorw %k1, %k0, %k0
2268; AVX512-NEXT:    kshiftrw $2, %k0, %k1
2269; AVX512-NEXT:    movl %r11d, %eax
2270; AVX512-NEXT:    imulb %bl
2271; AVX512-NEXT:    movl %eax, %esi
2272; AVX512-NEXT:    seto %al
2273; AVX512-NEXT:    movl %esi, %ecx
2274; AVX512-NEXT:    andb $1, %cl
2275; AVX512-NEXT:    negb %cl
2276; AVX512-NEXT:    cmpb %sil, %cl
2277; AVX512-NEXT:    setne %cl
2278; AVX512-NEXT:    orb %al, %cl
2279; AVX512-NEXT:    setne %al
2280; AVX512-NEXT:    kmovd %eax, %k2
2281; AVX512-NEXT:    kxorw %k2, %k1, %k1
2282; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2283; AVX512-NEXT:    kshiftrw $13, %k1, %k1
2284; AVX512-NEXT:    kxorw %k1, %k0, %k0
2285; AVX512-NEXT:    kshiftlw $13, %k0, %k0
2286; AVX512-NEXT:    kshiftrw $13, %k0, %k0
2287; AVX512-NEXT:    movl %r10d, %eax
2288; AVX512-NEXT:    imulb %r9b
2289; AVX512-NEXT:    # kill: def $al killed $al def $eax
2290; AVX512-NEXT:    seto %cl
2291; AVX512-NEXT:    movl %eax, %ebx
2292; AVX512-NEXT:    andb $1, %bl
2293; AVX512-NEXT:    negb %bl
2294; AVX512-NEXT:    cmpb %al, %bl
2295; AVX512-NEXT:    setne %bl
2296; AVX512-NEXT:    orb %cl, %bl
2297; AVX512-NEXT:    setne %cl
2298; AVX512-NEXT:    kmovd %ecx, %k1
2299; AVX512-NEXT:    kshiftlw $3, %k1, %k1
2300; AVX512-NEXT:    korw %k1, %k0, %k1
2301; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2302; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2303; AVX512-NEXT:    kmovd %r8d, %k0
2304; AVX512-NEXT:    kshiftrw $1, %k0, %k1
2305; AVX512-NEXT:    kmovd %edx, %k2
2306; AVX512-NEXT:    kxorw %k2, %k1, %k1
2307; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2308; AVX512-NEXT:    kshiftrw $14, %k1, %k1
2309; AVX512-NEXT:    kxorw %k1, %k0, %k0
2310; AVX512-NEXT:    kshiftrw $2, %k0, %k1
2311; AVX512-NEXT:    kmovd %esi, %k2
2312; AVX512-NEXT:    kxorw %k2, %k1, %k1
2313; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2314; AVX512-NEXT:    kshiftrw $13, %k1, %k1
2315; AVX512-NEXT:    kxorw %k1, %k0, %k0
2316; AVX512-NEXT:    kshiftrw $3, %k0, %k1
2317; AVX512-NEXT:    kmovd %eax, %k2
2318; AVX512-NEXT:    kxorw %k2, %k1, %k1
2319; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2320; AVX512-NEXT:    kshiftrw $12, %k1, %k1
2321; AVX512-NEXT:    kxorw %k1, %k0, %k0
2322; AVX512-NEXT:    kmovd %k0, %eax
2323; AVX512-NEXT:    movb %al, (%rdi)
2324; AVX512-NEXT:    popq %rbx
2325; AVX512-NEXT:    retq
2326  %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
2327  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
2328  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
2329  %res = sext <4 x i1> %obit to <4 x i32>
2330  store <4 x i1> %val, <4 x i1>* %p2
2331  ret <4 x i32> %res
2332}
2333
2334define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
2335; SSE2-LABEL: smulo_v2i128:
2336; SSE2:       # %bb.0:
2337; SSE2-NEXT:    pushq %rbp
2338; SSE2-NEXT:    pushq %r15
2339; SSE2-NEXT:    pushq %r14
2340; SSE2-NEXT:    pushq %r13
2341; SSE2-NEXT:    pushq %r12
2342; SSE2-NEXT:    pushq %rbx
2343; SSE2-NEXT:    subq $24, %rsp
2344; SSE2-NEXT:    movq %r8, %rax
2345; SSE2-NEXT:    movq %rcx, %r14
2346; SSE2-NEXT:    movq %rdx, %rbx
2347; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2348; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2349; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2350; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2351; SSE2-NEXT:    movq %rax, %rdx
2352; SSE2-NEXT:    movq %r9, %rcx
2353; SSE2-NEXT:    callq __muloti4
2354; SSE2-NEXT:    movq %rax, %r13
2355; SSE2-NEXT:    movq %rdx, %rbp
2356; SSE2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2357; SSE2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2358; SSE2-NEXT:    movq %rbx, %rdi
2359; SSE2-NEXT:    movq %r14, %rsi
2360; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2361; SSE2-NEXT:    movq %r12, %rcx
2362; SSE2-NEXT:    callq __muloti4
2363; SSE2-NEXT:    xorl %ecx, %ecx
2364; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2365; SSE2-NEXT:    sbbl %esi, %esi
2366; SSE2-NEXT:    movd %esi, %xmm1
2367; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2368; SSE2-NEXT:    sbbl %ecx, %ecx
2369; SSE2-NEXT:    movd %ecx, %xmm0
2370; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2371; SSE2-NEXT:    movq %rdx, 24(%r15)
2372; SSE2-NEXT:    movq %rax, 16(%r15)
2373; SSE2-NEXT:    movq %rbp, 8(%r15)
2374; SSE2-NEXT:    movq %r13, (%r15)
2375; SSE2-NEXT:    addq $24, %rsp
2376; SSE2-NEXT:    popq %rbx
2377; SSE2-NEXT:    popq %r12
2378; SSE2-NEXT:    popq %r13
2379; SSE2-NEXT:    popq %r14
2380; SSE2-NEXT:    popq %r15
2381; SSE2-NEXT:    popq %rbp
2382; SSE2-NEXT:    retq
2383;
2384; SSSE3-LABEL: smulo_v2i128:
2385; SSSE3:       # %bb.0:
2386; SSSE3-NEXT:    pushq %rbp
2387; SSSE3-NEXT:    pushq %r15
2388; SSSE3-NEXT:    pushq %r14
2389; SSSE3-NEXT:    pushq %r13
2390; SSSE3-NEXT:    pushq %r12
2391; SSSE3-NEXT:    pushq %rbx
2392; SSSE3-NEXT:    subq $24, %rsp
2393; SSSE3-NEXT:    movq %r8, %rax
2394; SSSE3-NEXT:    movq %rcx, %r14
2395; SSSE3-NEXT:    movq %rdx, %rbx
2396; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2397; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2398; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2399; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2400; SSSE3-NEXT:    movq %rax, %rdx
2401; SSSE3-NEXT:    movq %r9, %rcx
2402; SSSE3-NEXT:    callq __muloti4
2403; SSSE3-NEXT:    movq %rax, %r13
2404; SSSE3-NEXT:    movq %rdx, %rbp
2405; SSSE3-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2406; SSSE3-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2407; SSSE3-NEXT:    movq %rbx, %rdi
2408; SSSE3-NEXT:    movq %r14, %rsi
2409; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2410; SSSE3-NEXT:    movq %r12, %rcx
2411; SSSE3-NEXT:    callq __muloti4
2412; SSSE3-NEXT:    xorl %ecx, %ecx
2413; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2414; SSSE3-NEXT:    sbbl %esi, %esi
2415; SSSE3-NEXT:    movd %esi, %xmm1
2416; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2417; SSSE3-NEXT:    sbbl %ecx, %ecx
2418; SSSE3-NEXT:    movd %ecx, %xmm0
2419; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2420; SSSE3-NEXT:    movq %rdx, 24(%r15)
2421; SSSE3-NEXT:    movq %rax, 16(%r15)
2422; SSSE3-NEXT:    movq %rbp, 8(%r15)
2423; SSSE3-NEXT:    movq %r13, (%r15)
2424; SSSE3-NEXT:    addq $24, %rsp
2425; SSSE3-NEXT:    popq %rbx
2426; SSSE3-NEXT:    popq %r12
2427; SSSE3-NEXT:    popq %r13
2428; SSSE3-NEXT:    popq %r14
2429; SSSE3-NEXT:    popq %r15
2430; SSSE3-NEXT:    popq %rbp
2431; SSSE3-NEXT:    retq
2432;
2433; SSE41-LABEL: smulo_v2i128:
2434; SSE41:       # %bb.0:
2435; SSE41-NEXT:    pushq %rbp
2436; SSE41-NEXT:    pushq %r15
2437; SSE41-NEXT:    pushq %r14
2438; SSE41-NEXT:    pushq %r13
2439; SSE41-NEXT:    pushq %r12
2440; SSE41-NEXT:    pushq %rbx
2441; SSE41-NEXT:    subq $24, %rsp
2442; SSE41-NEXT:    movq %r8, %rax
2443; SSE41-NEXT:    movq %rcx, %r14
2444; SSE41-NEXT:    movq %rdx, %rbx
2445; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2446; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2447; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2448; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2449; SSE41-NEXT:    movq %rax, %rdx
2450; SSE41-NEXT:    movq %r9, %rcx
2451; SSE41-NEXT:    callq __muloti4
2452; SSE41-NEXT:    movq %rax, %r13
2453; SSE41-NEXT:    movq %rdx, %rbp
2454; SSE41-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2455; SSE41-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2456; SSE41-NEXT:    movq %rbx, %rdi
2457; SSE41-NEXT:    movq %r14, %rsi
2458; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2459; SSE41-NEXT:    movq %r12, %rcx
2460; SSE41-NEXT:    callq __muloti4
2461; SSE41-NEXT:    xorl %ecx, %ecx
2462; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2463; SSE41-NEXT:    sbbl %esi, %esi
2464; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2465; SSE41-NEXT:    sbbl %ecx, %ecx
2466; SSE41-NEXT:    movd %ecx, %xmm0
2467; SSE41-NEXT:    pinsrd $1, %esi, %xmm0
2468; SSE41-NEXT:    movq %rdx, 24(%r15)
2469; SSE41-NEXT:    movq %rax, 16(%r15)
2470; SSE41-NEXT:    movq %rbp, 8(%r15)
2471; SSE41-NEXT:    movq %r13, (%r15)
2472; SSE41-NEXT:    addq $24, %rsp
2473; SSE41-NEXT:    popq %rbx
2474; SSE41-NEXT:    popq %r12
2475; SSE41-NEXT:    popq %r13
2476; SSE41-NEXT:    popq %r14
2477; SSE41-NEXT:    popq %r15
2478; SSE41-NEXT:    popq %rbp
2479; SSE41-NEXT:    retq
2480;
2481; AVX1-LABEL: smulo_v2i128:
2482; AVX1:       # %bb.0:
2483; AVX1-NEXT:    pushq %rbp
2484; AVX1-NEXT:    pushq %r15
2485; AVX1-NEXT:    pushq %r14
2486; AVX1-NEXT:    pushq %r13
2487; AVX1-NEXT:    pushq %r12
2488; AVX1-NEXT:    pushq %rbx
2489; AVX1-NEXT:    subq $24, %rsp
2490; AVX1-NEXT:    movq %r8, %rax
2491; AVX1-NEXT:    movq %rcx, %r14
2492; AVX1-NEXT:    movq %rdx, %rbx
2493; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2494; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2495; AVX1-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2496; AVX1-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2497; AVX1-NEXT:    movq %rax, %rdx
2498; AVX1-NEXT:    movq %r9, %rcx
2499; AVX1-NEXT:    callq __muloti4
2500; AVX1-NEXT:    movq %rax, %r13
2501; AVX1-NEXT:    movq %rdx, %rbp
2502; AVX1-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2503; AVX1-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2504; AVX1-NEXT:    movq %rbx, %rdi
2505; AVX1-NEXT:    movq %r14, %rsi
2506; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2507; AVX1-NEXT:    movq %r12, %rcx
2508; AVX1-NEXT:    callq __muloti4
2509; AVX1-NEXT:    xorl %ecx, %ecx
2510; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2511; AVX1-NEXT:    sbbl %esi, %esi
2512; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2513; AVX1-NEXT:    sbbl %ecx, %ecx
2514; AVX1-NEXT:    vmovd %ecx, %xmm0
2515; AVX1-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
2516; AVX1-NEXT:    movq %rdx, 24(%r15)
2517; AVX1-NEXT:    movq %rax, 16(%r15)
2518; AVX1-NEXT:    movq %rbp, 8(%r15)
2519; AVX1-NEXT:    movq %r13, (%r15)
2520; AVX1-NEXT:    addq $24, %rsp
2521; AVX1-NEXT:    popq %rbx
2522; AVX1-NEXT:    popq %r12
2523; AVX1-NEXT:    popq %r13
2524; AVX1-NEXT:    popq %r14
2525; AVX1-NEXT:    popq %r15
2526; AVX1-NEXT:    popq %rbp
2527; AVX1-NEXT:    retq
2528;
2529; AVX2-LABEL: smulo_v2i128:
2530; AVX2:       # %bb.0:
2531; AVX2-NEXT:    pushq %rbp
2532; AVX2-NEXT:    pushq %r15
2533; AVX2-NEXT:    pushq %r14
2534; AVX2-NEXT:    pushq %r13
2535; AVX2-NEXT:    pushq %r12
2536; AVX2-NEXT:    pushq %rbx
2537; AVX2-NEXT:    subq $24, %rsp
2538; AVX2-NEXT:    movq %r8, %rax
2539; AVX2-NEXT:    movq %rcx, %r14
2540; AVX2-NEXT:    movq %rdx, %rbx
2541; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2542; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2543; AVX2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2544; AVX2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2545; AVX2-NEXT:    movq %rax, %rdx
2546; AVX2-NEXT:    movq %r9, %rcx
2547; AVX2-NEXT:    callq __muloti4
2548; AVX2-NEXT:    movq %rax, %r13
2549; AVX2-NEXT:    movq %rdx, %rbp
2550; AVX2-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2551; AVX2-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2552; AVX2-NEXT:    movq %rbx, %rdi
2553; AVX2-NEXT:    movq %r14, %rsi
2554; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2555; AVX2-NEXT:    movq %r12, %rcx
2556; AVX2-NEXT:    callq __muloti4
2557; AVX2-NEXT:    xorl %ecx, %ecx
2558; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2559; AVX2-NEXT:    sbbl %esi, %esi
2560; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
2561; AVX2-NEXT:    sbbl %ecx, %ecx
2562; AVX2-NEXT:    vmovd %ecx, %xmm0
2563; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
2564; AVX2-NEXT:    movq %rdx, 24(%r15)
2565; AVX2-NEXT:    movq %rax, 16(%r15)
2566; AVX2-NEXT:    movq %rbp, 8(%r15)
2567; AVX2-NEXT:    movq %r13, (%r15)
2568; AVX2-NEXT:    addq $24, %rsp
2569; AVX2-NEXT:    popq %rbx
2570; AVX2-NEXT:    popq %r12
2571; AVX2-NEXT:    popq %r13
2572; AVX2-NEXT:    popq %r14
2573; AVX2-NEXT:    popq %r15
2574; AVX2-NEXT:    popq %rbp
2575; AVX2-NEXT:    retq
2576;
2577; AVX512-LABEL: smulo_v2i128:
2578; AVX512:       # %bb.0:
2579; AVX512-NEXT:    pushq %rbp
2580; AVX512-NEXT:    pushq %r15
2581; AVX512-NEXT:    pushq %r14
2582; AVX512-NEXT:    pushq %r13
2583; AVX512-NEXT:    pushq %r12
2584; AVX512-NEXT:    pushq %rbx
2585; AVX512-NEXT:    subq $24, %rsp
2586; AVX512-NEXT:    movq %r8, %rax
2587; AVX512-NEXT:    movq %rcx, %r15
2588; AVX512-NEXT:    movq %rdx, %rbx
2589; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2590; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r13
2591; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2592; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2593; AVX512-NEXT:    movq %rax, %rdx
2594; AVX512-NEXT:    movq %r9, %rcx
2595; AVX512-NEXT:    callq __muloti4
2596; AVX512-NEXT:    movq %rax, %r14
2597; AVX512-NEXT:    movq %rdx, %rbp
2598; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
2599; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
2600; AVX512-NEXT:    movq %rbx, %rdi
2601; AVX512-NEXT:    movq %r15, %rsi
2602; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
2603; AVX512-NEXT:    movq %r13, %rcx
2604; AVX512-NEXT:    callq __muloti4
2605; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
2606; AVX512-NEXT:    setne %cl
2607; AVX512-NEXT:    kmovd %ecx, %k0
2608; AVX512-NEXT:    kshiftlw $1, %k0, %k0
2609; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
2610; AVX512-NEXT:    setne %cl
2611; AVX512-NEXT:    andl $1, %ecx
2612; AVX512-NEXT:    kmovw %ecx, %k1
2613; AVX512-NEXT:    korw %k0, %k1, %k1
2614; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2615; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2616; AVX512-NEXT:    movq %rdx, 24(%r12)
2617; AVX512-NEXT:    movq %rax, 16(%r12)
2618; AVX512-NEXT:    movq %rbp, 8(%r12)
2619; AVX512-NEXT:    movq %r14, (%r12)
2620; AVX512-NEXT:    addq $24, %rsp
2621; AVX512-NEXT:    popq %rbx
2622; AVX512-NEXT:    popq %r12
2623; AVX512-NEXT:    popq %r13
2624; AVX512-NEXT:    popq %r14
2625; AVX512-NEXT:    popq %r15
2626; AVX512-NEXT:    popq %rbp
2627; AVX512-NEXT:    retq
2628  %t = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
2629  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
2630  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
2631  %res = sext <2 x i1> %obit to <2 x i32>
2632  store <2 x i128> %val, <2 x i128>* %p2
2633  ret <2 x i32> %res
2634}
2635