1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
8
9declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
10declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
11declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
12declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
13declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
14declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
15declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
16
17declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
18declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
19declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
20
21declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
22declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
23declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
24
25define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
26; SSE-LABEL: umulo_v1i32:
27; SSE:       # %bb.0:
28; SSE-NEXT:    movq %rdx, %rcx
29; SSE-NEXT:    movl %edi, %eax
30; SSE-NEXT:    xorl %edi, %edi
31; SSE-NEXT:    mull %esi
32; SSE-NEXT:    seto %dil
33; SSE-NEXT:    negl %edi
34; SSE-NEXT:    movl %eax, (%rcx)
35; SSE-NEXT:    movl %edi, %eax
36; SSE-NEXT:    retq
37;
38; AVX-LABEL: umulo_v1i32:
39; AVX:       # %bb.0:
40; AVX-NEXT:    movq %rdx, %rcx
41; AVX-NEXT:    movl %edi, %eax
42; AVX-NEXT:    xorl %edi, %edi
43; AVX-NEXT:    mull %esi
44; AVX-NEXT:    seto %dil
45; AVX-NEXT:    negl %edi
46; AVX-NEXT:    movl %eax, (%rcx)
47; AVX-NEXT:    movl %edi, %eax
48; AVX-NEXT:    retq
49  %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
50  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
51  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
52  %res = sext <1 x i1> %obit to <1 x i32>
53  store <1 x i32> %val, <1 x i32>* %p2
54  ret <1 x i32> %res
55}
56
57define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
58; SSE2-LABEL: umulo_v2i32:
59; SSE2:       # %bb.0:
60; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
61; SSE2-NEXT:    pmuludq %xmm1, %xmm0
62; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
63; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
64; SSE2-NEXT:    pmuludq %xmm2, %xmm4
65; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
66; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
67; SSE2-NEXT:    pxor %xmm2, %xmm2
68; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
69; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
70; SSE2-NEXT:    pxor %xmm2, %xmm1
71; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
72; SSE2-NEXT:    movq %xmm0, (%rdi)
73; SSE2-NEXT:    movdqa %xmm1, %xmm0
74; SSE2-NEXT:    retq
75;
76; SSSE3-LABEL: umulo_v2i32:
77; SSSE3:       # %bb.0:
78; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
79; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
80; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
81; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
82; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
83; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
84; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
85; SSSE3-NEXT:    pxor %xmm2, %xmm2
86; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
87; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
88; SSSE3-NEXT:    pxor %xmm2, %xmm1
89; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
90; SSSE3-NEXT:    movq %xmm0, (%rdi)
91; SSSE3-NEXT:    movdqa %xmm1, %xmm0
92; SSSE3-NEXT:    retq
93;
94; SSE41-LABEL: umulo_v2i32:
95; SSE41:       # %bb.0:
96; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
97; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
98; SSE41-NEXT:    pmuludq %xmm2, %xmm3
99; SSE41-NEXT:    movdqa %xmm0, %xmm2
100; SSE41-NEXT:    pmuludq %xmm1, %xmm2
101; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
102; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
103; SSE41-NEXT:    pxor %xmm3, %xmm3
104; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
105; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
106; SSE41-NEXT:    pxor %xmm3, %xmm2
107; SSE41-NEXT:    pmulld %xmm1, %xmm0
108; SSE41-NEXT:    movq %xmm0, (%rdi)
109; SSE41-NEXT:    movdqa %xmm2, %xmm0
110; SSE41-NEXT:    retq
111;
112; AVX1-LABEL: umulo_v2i32:
113; AVX1:       # %bb.0:
114; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
115; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
116; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
117; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
118; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
119; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
120; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
121; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
122; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
123; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
124; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
125; AVX1-NEXT:    vmovq %xmm0, (%rdi)
126; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
127; AVX1-NEXT:    retq
128;
129; AVX2-LABEL: umulo_v2i32:
130; AVX2:       # %bb.0:
131; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
132; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
133; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
134; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
135; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
136; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
137; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
138; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
139; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
140; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
141; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
142; AVX2-NEXT:    vmovq %xmm0, (%rdi)
143; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
144; AVX2-NEXT:    retq
145;
146; AVX512-LABEL: umulo_v2i32:
147; AVX512:       # %bb.0:
148; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
149; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
150; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
151; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
152; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
153; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
154; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
155; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
156; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
157; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
158; AVX512-NEXT:    vmovq %xmm1, (%rdi)
159; AVX512-NEXT:    retq
160  %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
161  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
162  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
163  %res = sext <2 x i1> %obit to <2 x i32>
164  store <2 x i32> %val, <2 x i32>* %p2
165  ret <2 x i32> %res
166}
167
168define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
169; SSE2-LABEL: umulo_v3i32:
170; SSE2:       # %bb.0:
171; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
172; SSE2-NEXT:    pmuludq %xmm1, %xmm0
173; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
174; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
175; SSE2-NEXT:    pmuludq %xmm2, %xmm4
176; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
177; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
178; SSE2-NEXT:    pxor %xmm2, %xmm2
179; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
180; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
181; SSE2-NEXT:    pxor %xmm2, %xmm1
182; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
183; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
184; SSE2-NEXT:    movd %xmm2, 8(%rdi)
185; SSE2-NEXT:    movq %xmm0, (%rdi)
186; SSE2-NEXT:    movdqa %xmm1, %xmm0
187; SSE2-NEXT:    retq
188;
189; SSSE3-LABEL: umulo_v3i32:
190; SSSE3:       # %bb.0:
191; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
192; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
193; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
194; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
195; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
196; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
197; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
198; SSSE3-NEXT:    pxor %xmm2, %xmm2
199; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
200; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
201; SSSE3-NEXT:    pxor %xmm2, %xmm1
202; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
203; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
204; SSSE3-NEXT:    movd %xmm2, 8(%rdi)
205; SSSE3-NEXT:    movq %xmm0, (%rdi)
206; SSSE3-NEXT:    movdqa %xmm1, %xmm0
207; SSSE3-NEXT:    retq
208;
209; SSE41-LABEL: umulo_v3i32:
210; SSE41:       # %bb.0:
211; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
212; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
213; SSE41-NEXT:    pmuludq %xmm2, %xmm3
214; SSE41-NEXT:    movdqa %xmm0, %xmm2
215; SSE41-NEXT:    pmuludq %xmm1, %xmm2
216; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
217; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
218; SSE41-NEXT:    pxor %xmm3, %xmm3
219; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
220; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
221; SSE41-NEXT:    pxor %xmm3, %xmm2
222; SSE41-NEXT:    pmulld %xmm1, %xmm0
223; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
224; SSE41-NEXT:    movq %xmm0, (%rdi)
225; SSE41-NEXT:    movdqa %xmm2, %xmm0
226; SSE41-NEXT:    retq
227;
228; AVX1-LABEL: umulo_v3i32:
229; AVX1:       # %bb.0:
230; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
231; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
232; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
233; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
234; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
235; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
236; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
237; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
238; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
239; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
240; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
241; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
242; AVX1-NEXT:    vmovq %xmm0, (%rdi)
243; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
244; AVX1-NEXT:    retq
245;
246; AVX2-LABEL: umulo_v3i32:
247; AVX2:       # %bb.0:
248; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
249; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
250; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
251; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
252; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
253; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
254; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
255; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
256; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
257; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
258; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
259; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
260; AVX2-NEXT:    vmovq %xmm0, (%rdi)
261; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
262; AVX2-NEXT:    retq
263;
264; AVX512-LABEL: umulo_v3i32:
265; AVX512:       # %bb.0:
266; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
267; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
268; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
269; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
270; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
271; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
272; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
273; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
274; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
275; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
276; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
277; AVX512-NEXT:    vmovq %xmm1, (%rdi)
278; AVX512-NEXT:    retq
279  %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
280  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
281  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
282  %res = sext <3 x i1> %obit to <3 x i32>
283  store <3 x i32> %val, <3 x i32>* %p2
284  ret <3 x i32> %res
285}
286
287define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
288; SSE2-LABEL: umulo_v4i32:
289; SSE2:       # %bb.0:
290; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
291; SSE2-NEXT:    pmuludq %xmm1, %xmm0
292; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
293; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
294; SSE2-NEXT:    pmuludq %xmm2, %xmm4
295; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
296; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
297; SSE2-NEXT:    pxor %xmm2, %xmm2
298; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
299; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
300; SSE2-NEXT:    pxor %xmm2, %xmm1
301; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
302; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
303; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
304; SSE2-NEXT:    movdqa %xmm0, (%rdi)
305; SSE2-NEXT:    movdqa %xmm1, %xmm0
306; SSE2-NEXT:    retq
307;
308; SSSE3-LABEL: umulo_v4i32:
309; SSSE3:       # %bb.0:
310; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
311; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
312; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
313; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
314; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
315; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
316; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
317; SSSE3-NEXT:    pxor %xmm2, %xmm2
318; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
319; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
320; SSSE3-NEXT:    pxor %xmm2, %xmm1
321; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
322; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
323; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
324; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
325; SSSE3-NEXT:    movdqa %xmm1, %xmm0
326; SSSE3-NEXT:    retq
327;
328; SSE41-LABEL: umulo_v4i32:
329; SSE41:       # %bb.0:
330; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
331; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
332; SSE41-NEXT:    pmuludq %xmm2, %xmm3
333; SSE41-NEXT:    movdqa %xmm0, %xmm2
334; SSE41-NEXT:    pmuludq %xmm1, %xmm2
335; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
336; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
337; SSE41-NEXT:    pxor %xmm3, %xmm3
338; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
339; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
340; SSE41-NEXT:    pxor %xmm3, %xmm2
341; SSE41-NEXT:    pmulld %xmm1, %xmm0
342; SSE41-NEXT:    movdqa %xmm0, (%rdi)
343; SSE41-NEXT:    movdqa %xmm2, %xmm0
344; SSE41-NEXT:    retq
345;
346; AVX1-LABEL: umulo_v4i32:
347; AVX1:       # %bb.0:
348; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
349; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
350; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
351; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
352; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
353; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
354; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
355; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
356; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
357; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
358; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
359; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
360; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
361; AVX1-NEXT:    retq
362;
363; AVX2-LABEL: umulo_v4i32:
364; AVX2:       # %bb.0:
365; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
366; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
367; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
368; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
369; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
370; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
371; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
372; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
373; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
374; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
375; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
376; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
377; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
378; AVX2-NEXT:    retq
379;
380; AVX512-LABEL: umulo_v4i32:
381; AVX512:       # %bb.0:
382; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
383; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
384; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
385; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
386; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
387; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
388; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
389; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
390; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
391; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
392; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
393; AVX512-NEXT:    retq
394  %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
395  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
396  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
397  %res = sext <4 x i1> %obit to <4 x i32>
398  store <4 x i32> %val, <4 x i32>* %p2
399  ret <4 x i32> %res
400}
401
402define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
403; SSE2-LABEL: umulo_v6i32:
404; SSE2:       # %bb.0:
405; SSE2-NEXT:    movq %rdi, %rax
406; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
407; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
408; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
409; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
410; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
411; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
412; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
413; SSE2-NEXT:    movd %r8d, %xmm4
414; SSE2-NEXT:    movd %ecx, %xmm0
415; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
416; SSE2-NEXT:    movd %edx, %xmm5
417; SSE2-NEXT:    movd %esi, %xmm6
418; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
419; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
420; SSE2-NEXT:    pmuludq %xmm1, %xmm6
421; SSE2-NEXT:    movd %r9d, %xmm0
422; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
423; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
424; SSE2-NEXT:    pmuludq %xmm0, %xmm1
425; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
426; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
427; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm4[0,0]
428; SSE2-NEXT:    pmuludq %xmm3, %xmm5
429; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,3,2,3]
430; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
431; SSE2-NEXT:    pxor %xmm2, %xmm2
432; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
433; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
434; SSE2-NEXT:    pxor %xmm3, %xmm0
435; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
436; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
437; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
438; SSE2-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
439; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
440; SSE2-NEXT:    pmuludq %xmm5, %xmm6
441; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]
442; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,3,2,3]
443; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
444; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
445; SSE2-NEXT:    pxor %xmm3, %xmm7
446; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
447; SSE2-NEXT:    movq %xmm1, 16(%rcx)
448; SSE2-NEXT:    movdqa %xmm4, (%rcx)
449; SSE2-NEXT:    movq %xmm7, 16(%rdi)
450; SSE2-NEXT:    movdqa %xmm0, (%rdi)
451; SSE2-NEXT:    retq
452;
453; SSSE3-LABEL: umulo_v6i32:
454; SSSE3:       # %bb.0:
455; SSSE3-NEXT:    movq %rdi, %rax
456; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
457; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
458; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
459; SSSE3-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
460; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
461; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
462; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
463; SSSE3-NEXT:    movd %r8d, %xmm4
464; SSSE3-NEXT:    movd %ecx, %xmm0
465; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
466; SSSE3-NEXT:    movd %edx, %xmm5
467; SSSE3-NEXT:    movd %esi, %xmm6
468; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
469; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
470; SSSE3-NEXT:    pmuludq %xmm1, %xmm6
471; SSSE3-NEXT:    movd %r9d, %xmm0
472; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
473; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
474; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
475; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
476; SSSE3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
477; SSSE3-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm4[0,0]
478; SSSE3-NEXT:    pmuludq %xmm3, %xmm5
479; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,3,2,3]
480; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
481; SSSE3-NEXT:    pxor %xmm2, %xmm2
482; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
483; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
484; SSSE3-NEXT:    pxor %xmm3, %xmm0
485; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
486; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
487; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
488; SSSE3-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
489; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
490; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
491; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]
492; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,3,2,3]
493; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
494; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
495; SSSE3-NEXT:    pxor %xmm3, %xmm7
496; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
497; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
498; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
499; SSSE3-NEXT:    movq %xmm7, 16(%rdi)
500; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
501; SSSE3-NEXT:    retq
502;
503; SSE41-LABEL: umulo_v6i32:
504; SSE41:       # %bb.0:
505; SSE41-NEXT:    movq %rdi, %rax
506; SSE41-NEXT:    movd %esi, %xmm2
507; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
508; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
509; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
510; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
511; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
512; SSE41-NEXT:    movdqa %xmm1, %xmm0
513; SSE41-NEXT:    pmuludq %xmm2, %xmm1
514; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
515; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
516; SSE41-NEXT:    movd %r9d, %xmm4
517; SSE41-NEXT:    movdqa %xmm4, %xmm5
518; SSE41-NEXT:    pmuludq %xmm3, %xmm4
519; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
520; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
521; SSE41-NEXT:    pmulld %xmm3, %xmm5
522; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
523; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
524; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
525; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
526; SSE41-NEXT:    pmuludq %xmm3, %xmm6
527; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
528; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
529; SSE41-NEXT:    pxor %xmm8, %xmm8
530; SSE41-NEXT:    pcmpeqd %xmm8, %xmm1
531; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
532; SSE41-NEXT:    pxor %xmm6, %xmm1
533; SSE41-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
534; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
535; SSE41-NEXT:    pmuludq %xmm7, %xmm3
536; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
537; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
538; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
539; SSE41-NEXT:    pxor %xmm6, %xmm4
540; SSE41-NEXT:    pmulld %xmm2, %xmm0
541; SSE41-NEXT:    movq %xmm5, 16(%rcx)
542; SSE41-NEXT:    movdqa %xmm0, (%rcx)
543; SSE41-NEXT:    movq %xmm4, 16(%rdi)
544; SSE41-NEXT:    movdqa %xmm1, (%rdi)
545; SSE41-NEXT:    retq
546;
547; AVX1-LABEL: umulo_v6i32:
548; AVX1:       # %bb.0:
549; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
550; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
551; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
552; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
553; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
554; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
555; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
556; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
557; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
558; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
559; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
560; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
561; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
562; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
563; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
564; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
565; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
566; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
567; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
568; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
569; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
570; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
571; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
572; AVX1-NEXT:    vmovq %xmm1, 16(%rdi)
573; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
574; AVX1-NEXT:    vmovaps %ymm2, %ymm0
575; AVX1-NEXT:    retq
576;
577; AVX2-LABEL: umulo_v6i32:
578; AVX2:       # %bb.0:
579; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
580; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
581; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
582; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
583; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
584; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
585; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
586; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
587; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
588; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
589; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
590; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
591; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
592; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
593; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
594; AVX2-NEXT:    retq
595;
596; AVX512-LABEL: umulo_v6i32:
597; AVX512:       # %bb.0:
598; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
599; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
600; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
601; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
602; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
603; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
604; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
605; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
606; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
607; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
608; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
609; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
610; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
611; AVX512-NEXT:    retq
612  %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
613  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
614  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
615  %res = sext <6 x i1> %obit to <6 x i32>
616  store <6 x i32> %val, <6 x i32>* %p2
617  ret <6 x i32> %res
618}
619
620define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
621; SSE2-LABEL: umulo_v8i32:
622; SSE2:       # %bb.0:
623; SSE2-NEXT:    movdqa %xmm0, %xmm4
624; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
625; SSE2-NEXT:    pmuludq %xmm2, %xmm4
626; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
627; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
628; SSE2-NEXT:    pmuludq %xmm5, %xmm6
629; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
630; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
631; SSE2-NEXT:    pxor %xmm8, %xmm8
632; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
633; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
634; SSE2-NEXT:    pxor %xmm7, %xmm0
635; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
636; SSE2-NEXT:    pmuludq %xmm3, %xmm1
637; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
638; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
639; SSE2-NEXT:    pmuludq %xmm5, %xmm3
640; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
641; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
642; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
643; SSE2-NEXT:    pxor %xmm7, %xmm2
644; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
645; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
646; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
647; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
648; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
649; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
650; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
651; SSE2-NEXT:    movdqa %xmm4, (%rdi)
652; SSE2-NEXT:    movdqa %xmm2, %xmm1
653; SSE2-NEXT:    retq
654;
655; SSSE3-LABEL: umulo_v8i32:
656; SSSE3:       # %bb.0:
657; SSSE3-NEXT:    movdqa %xmm0, %xmm4
658; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
659; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
660; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
661; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
662; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
663; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
664; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
665; SSSE3-NEXT:    pxor %xmm8, %xmm8
666; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
667; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
668; SSSE3-NEXT:    pxor %xmm7, %xmm0
669; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
670; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
671; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
672; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
673; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
674; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
675; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
676; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
677; SSSE3-NEXT:    pxor %xmm7, %xmm2
678; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
679; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
680; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
681; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
682; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
683; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
684; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
685; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
686; SSSE3-NEXT:    movdqa %xmm2, %xmm1
687; SSSE3-NEXT:    retq
688;
689; SSE41-LABEL: umulo_v8i32:
690; SSE41:       # %bb.0:
691; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
692; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
693; SSE41-NEXT:    pmuludq %xmm4, %xmm5
694; SSE41-NEXT:    movdqa %xmm0, %xmm4
695; SSE41-NEXT:    pmuludq %xmm2, %xmm4
696; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
697; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
698; SSE41-NEXT:    pxor %xmm8, %xmm8
699; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
700; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
701; SSE41-NEXT:    pxor %xmm7, %xmm4
702; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
703; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
704; SSE41-NEXT:    pmuludq %xmm5, %xmm6
705; SSE41-NEXT:    movdqa %xmm1, %xmm5
706; SSE41-NEXT:    pmuludq %xmm3, %xmm5
707; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
708; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
709; SSE41-NEXT:    pcmpeqd %xmm8, %xmm5
710; SSE41-NEXT:    pxor %xmm7, %xmm5
711; SSE41-NEXT:    pmulld %xmm2, %xmm0
712; SSE41-NEXT:    pmulld %xmm3, %xmm1
713; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
714; SSE41-NEXT:    movdqa %xmm0, (%rdi)
715; SSE41-NEXT:    movdqa %xmm4, %xmm0
716; SSE41-NEXT:    movdqa %xmm5, %xmm1
717; SSE41-NEXT:    retq
718;
719; AVX1-LABEL: umulo_v8i32:
720; AVX1:       # %bb.0:
721; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
722; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
723; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
724; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
725; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
726; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
727; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
728; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
729; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
730; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
731; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
732; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
733; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
734; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
735; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
736; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
737; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
738; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
739; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
740; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
741; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
742; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
743; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
744; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)
745; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
746; AVX1-NEXT:    vmovaps %ymm2, %ymm0
747; AVX1-NEXT:    retq
748;
749; AVX2-LABEL: umulo_v8i32:
750; AVX2:       # %bb.0:
751; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
752; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
753; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
754; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
755; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
756; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
757; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
758; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
759; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
760; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
761; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
762; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
763; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
764; AVX2-NEXT:    retq
765;
766; AVX512-LABEL: umulo_v8i32:
767; AVX512:       # %bb.0:
768; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
769; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
770; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
771; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
772; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
773; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
774; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
775; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
776; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
777; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
778; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
779; AVX512-NEXT:    retq
780  %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
781  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
782  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
783  %res = sext <8 x i1> %obit to <8 x i32>
784  store <8 x i32> %val, <8 x i32>* %p2
785  ret <8 x i32> %res
786}
787
788define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
789; SSE2-LABEL: umulo_v16i32:
790; SSE2:       # %bb.0:
791; SSE2-NEXT:    movdqa %xmm0, %xmm8
792; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
793; SSE2-NEXT:    pmuludq %xmm4, %xmm8
794; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
795; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
796; SSE2-NEXT:    pmuludq %xmm10, %xmm9
797; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
798; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
799; SSE2-NEXT:    pxor %xmm10, %xmm10
800; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
801; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
802; SSE2-NEXT:    pxor %xmm11, %xmm0
803; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
804; SSE2-NEXT:    pmuludq %xmm5, %xmm1
805; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
806; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
807; SSE2-NEXT:    pmuludq %xmm13, %xmm12
808; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
809; SSE2-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
810; SSE2-NEXT:    pcmpeqd %xmm10, %xmm15
811; SSE2-NEXT:    pxor %xmm11, %xmm15
812; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
813; SSE2-NEXT:    pmuludq %xmm6, %xmm2
814; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
815; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
816; SSE2-NEXT:    pmuludq %xmm14, %xmm13
817; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
818; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
819; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
820; SSE2-NEXT:    pxor %xmm11, %xmm5
821; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
822; SSE2-NEXT:    pmuludq %xmm7, %xmm3
823; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
824; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
825; SSE2-NEXT:    pmuludq %xmm14, %xmm7
826; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
827; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
828; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
829; SSE2-NEXT:    pxor %xmm11, %xmm6
830; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
831; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
832; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
833; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
834; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
835; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
836; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
837; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
838; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
839; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
840; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
841; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
842; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
843; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
844; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
845; SSE2-NEXT:    movdqa %xmm8, (%rdi)
846; SSE2-NEXT:    movdqa %xmm15, %xmm1
847; SSE2-NEXT:    movdqa %xmm5, %xmm2
848; SSE2-NEXT:    movdqa %xmm6, %xmm3
849; SSE2-NEXT:    retq
850;
851; SSSE3-LABEL: umulo_v16i32:
852; SSSE3:       # %bb.0:
853; SSSE3-NEXT:    movdqa %xmm0, %xmm8
854; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
855; SSSE3-NEXT:    pmuludq %xmm4, %xmm8
856; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
857; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
858; SSSE3-NEXT:    pmuludq %xmm10, %xmm9
859; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
860; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
861; SSSE3-NEXT:    pxor %xmm10, %xmm10
862; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
863; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm11
864; SSSE3-NEXT:    pxor %xmm11, %xmm0
865; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
866; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
867; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
868; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
869; SSSE3-NEXT:    pmuludq %xmm13, %xmm12
870; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
871; SSSE3-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
872; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm15
873; SSSE3-NEXT:    pxor %xmm11, %xmm15
874; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
875; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
876; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
877; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
878; SSSE3-NEXT:    pmuludq %xmm14, %xmm13
879; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
880; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
881; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm5
882; SSSE3-NEXT:    pxor %xmm11, %xmm5
883; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
884; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
885; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
886; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
887; SSSE3-NEXT:    pmuludq %xmm14, %xmm7
888; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
889; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
890; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm6
891; SSSE3-NEXT:    pxor %xmm11, %xmm6
892; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
893; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
894; SSSE3-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
895; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
896; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
897; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
898; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
899; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
900; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
901; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
902; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
903; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
904; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
905; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
906; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
907; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
908; SSSE3-NEXT:    movdqa %xmm15, %xmm1
909; SSSE3-NEXT:    movdqa %xmm5, %xmm2
910; SSSE3-NEXT:    movdqa %xmm6, %xmm3
911; SSSE3-NEXT:    retq
912;
913; SSE41-LABEL: umulo_v16i32:
914; SSE41:       # %bb.0:
915; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
916; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
917; SSE41-NEXT:    pmuludq %xmm8, %xmm9
918; SSE41-NEXT:    movdqa %xmm0, %xmm8
919; SSE41-NEXT:    pmuludq %xmm4, %xmm8
920; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
921; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
922; SSE41-NEXT:    pxor %xmm12, %xmm12
923; SSE41-NEXT:    pcmpeqd %xmm12, %xmm8
924; SSE41-NEXT:    pcmpeqd %xmm13, %xmm13
925; SSE41-NEXT:    pxor %xmm13, %xmm8
926; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
927; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
928; SSE41-NEXT:    pmuludq %xmm9, %xmm10
929; SSE41-NEXT:    movdqa %xmm1, %xmm9
930; SSE41-NEXT:    pmuludq %xmm5, %xmm9
931; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
932; SSE41-NEXT:    pblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3],xmm9[4,5],xmm10[6,7]
933; SSE41-NEXT:    pcmpeqd %xmm12, %xmm9
934; SSE41-NEXT:    pxor %xmm13, %xmm9
935; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
936; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]
937; SSE41-NEXT:    pmuludq %xmm10, %xmm11
938; SSE41-NEXT:    movdqa %xmm2, %xmm10
939; SSE41-NEXT:    pmuludq %xmm6, %xmm10
940; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
941; SSE41-NEXT:    pblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]
942; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
943; SSE41-NEXT:    pxor %xmm13, %xmm10
944; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
945; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
946; SSE41-NEXT:    pmuludq %xmm11, %xmm14
947; SSE41-NEXT:    movdqa %xmm3, %xmm11
948; SSE41-NEXT:    pmuludq %xmm7, %xmm11
949; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
950; SSE41-NEXT:    pblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3],xmm11[4,5],xmm14[6,7]
951; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
952; SSE41-NEXT:    pxor %xmm13, %xmm11
953; SSE41-NEXT:    pmulld %xmm4, %xmm0
954; SSE41-NEXT:    pmulld %xmm5, %xmm1
955; SSE41-NEXT:    pmulld %xmm6, %xmm2
956; SSE41-NEXT:    pmulld %xmm7, %xmm3
957; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
958; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
959; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
960; SSE41-NEXT:    movdqa %xmm0, (%rdi)
961; SSE41-NEXT:    movdqa %xmm8, %xmm0
962; SSE41-NEXT:    movdqa %xmm9, %xmm1
963; SSE41-NEXT:    movdqa %xmm10, %xmm2
964; SSE41-NEXT:    movdqa %xmm11, %xmm3
965; SSE41-NEXT:    retq
966;
967; AVX1-LABEL: umulo_v16i32:
968; AVX1:       # %bb.0:
969; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
970; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
971; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
972; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
973; AVX1-NEXT:    vpmuludq %xmm6, %xmm7, %xmm6
974; AVX1-NEXT:    vpmuludq %xmm10, %xmm12, %xmm7
975; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
976; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
977; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
978; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm8, %xmm7
979; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
980; AVX1-NEXT:    vpxor %xmm7, %xmm9, %xmm7
981; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
982; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
983; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
984; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm6
985; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
986; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5],xmm4[6,7]
987; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
988; AVX1-NEXT:    vpxor %xmm4, %xmm9, %xmm4
989; AVX1-NEXT:    vpackssdw %xmm7, %xmm4, %xmm11
990; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
991; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
992; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
993; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
994; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
995; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm7
996; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
997; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
998; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
999; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm13
1000; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
1001; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
1002; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
1003; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm7
1004; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1005; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1006; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
1007; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
1008; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
1009; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm7
1010; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1011; AVX1-NEXT:    vpmulld %xmm6, %xmm4, %xmm4
1012; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1013; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
1014; AVX1-NEXT:    vpmovsxbd %xmm7, %xmm0
1015; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm7[1,1,2,3]
1016; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1017; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1018; AVX1-NEXT:    vpacksswb %xmm5, %xmm11, %xmm1
1019; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1020; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1021; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1022; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1023; AVX1-NEXT:    vmovdqa %xmm6, 48(%rdi)
1024; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1025; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1026; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1027; AVX1-NEXT:    retq
1028;
1029; AVX2-LABEL: umulo_v16i32:
1030; AVX2:       # %bb.0:
1031; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1032; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1033; AVX2-NEXT:    vpmuludq %ymm4, %ymm5, %ymm4
1034; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm5
1035; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1036; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1037; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1038; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
1039; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
1040; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
1041; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm7
1042; AVX2-NEXT:    vpackssdw %xmm7, %xmm4, %xmm4
1043; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[1,1,3,3,5,5,7,7]
1044; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm0[1,1,3,3,5,5,7,7]
1045; AVX2-NEXT:    vpmuludq %ymm7, %ymm8, %ymm7
1046; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm8
1047; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,1,3,3,5,5,7,7]
1048; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
1049; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
1050; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
1051; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1052; AVX2-NEXT:    vpackssdw %xmm6, %xmm5, %xmm5
1053; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
1054; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1055; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1056; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm0
1057; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm1
1058; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1059; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1060; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1061; AVX2-NEXT:    retq
1062;
1063; AVX512-LABEL: umulo_v16i32:
1064; AVX512:       # %bb.0:
1065; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm2
1066; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1067; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1068; AVX512-NEXT:    vpmuludq %zmm3, %zmm4, %zmm3
1069; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1070; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1071; AVX512-NEXT:    vptestmd %zmm4, %zmm4, %k1
1072; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1073; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1074; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1075; AVX512-NEXT:    retq
1076  %t = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1077  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1078  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1079  %res = sext <16 x i1> %obit to <16 x i32>
1080  store <16 x i32> %val, <16 x i32>* %p2
1081  ret <16 x i32> %res
1082}
1083
1084define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1085; SSE2-LABEL: umulo_v16i8:
1086; SSE2:       # %bb.0:
1087; SSE2-NEXT:    movdqa %xmm1, %xmm2
1088; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1089; SSE2-NEXT:    movdqa %xmm0, %xmm3
1090; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1091; SSE2-NEXT:    pmullw %xmm2, %xmm3
1092; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1093; SSE2-NEXT:    pand %xmm2, %xmm3
1094; SSE2-NEXT:    movdqa %xmm1, %xmm5
1095; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
1096; SSE2-NEXT:    movdqa %xmm0, %xmm4
1097; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1098; SSE2-NEXT:    pmullw %xmm5, %xmm4
1099; SSE2-NEXT:    pand %xmm2, %xmm4
1100; SSE2-NEXT:    packuswb %xmm3, %xmm4
1101; SSE2-NEXT:    pxor %xmm2, %xmm2
1102; SSE2-NEXT:    movdqa %xmm1, %xmm3
1103; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1104; SSE2-NEXT:    movdqa %xmm0, %xmm5
1105; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1106; SSE2-NEXT:    pmullw %xmm3, %xmm5
1107; SSE2-NEXT:    psrlw $8, %xmm5
1108; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1109; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1110; SSE2-NEXT:    pmullw %xmm1, %xmm0
1111; SSE2-NEXT:    psrlw $8, %xmm0
1112; SSE2-NEXT:    packuswb %xmm5, %xmm0
1113; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
1114; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1115; SSE2-NEXT:    pxor %xmm0, %xmm3
1116; SSE2-NEXT:    movdqa %xmm3, %xmm1
1117; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1118; SSE2-NEXT:    movdqa %xmm1, %xmm0
1119; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1120; SSE2-NEXT:    pslld $31, %xmm0
1121; SSE2-NEXT:    psrad $31, %xmm0
1122; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1123; SSE2-NEXT:    pslld $31, %xmm1
1124; SSE2-NEXT:    psrad $31, %xmm1
1125; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1126; SSE2-NEXT:    movdqa %xmm3, %xmm2
1127; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1128; SSE2-NEXT:    pslld $31, %xmm2
1129; SSE2-NEXT:    psrad $31, %xmm2
1130; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1131; SSE2-NEXT:    pslld $31, %xmm3
1132; SSE2-NEXT:    psrad $31, %xmm3
1133; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1134; SSE2-NEXT:    retq
1135;
1136; SSSE3-LABEL: umulo_v16i8:
1137; SSSE3:       # %bb.0:
1138; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1139; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1140; SSSE3-NEXT:    movdqa %xmm0, %xmm3
1141; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1142; SSSE3-NEXT:    pmullw %xmm2, %xmm3
1143; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1144; SSSE3-NEXT:    pand %xmm2, %xmm3
1145; SSSE3-NEXT:    movdqa %xmm1, %xmm5
1146; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
1147; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1148; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1149; SSSE3-NEXT:    pmullw %xmm5, %xmm4
1150; SSSE3-NEXT:    pand %xmm2, %xmm4
1151; SSSE3-NEXT:    packuswb %xmm3, %xmm4
1152; SSSE3-NEXT:    pxor %xmm2, %xmm2
1153; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1154; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1155; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1156; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1157; SSSE3-NEXT:    pmullw %xmm3, %xmm5
1158; SSSE3-NEXT:    psrlw $8, %xmm5
1159; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1160; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1161; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1162; SSSE3-NEXT:    psrlw $8, %xmm0
1163; SSSE3-NEXT:    packuswb %xmm5, %xmm0
1164; SSSE3-NEXT:    pcmpeqb %xmm2, %xmm0
1165; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1166; SSSE3-NEXT:    pxor %xmm0, %xmm3
1167; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1168; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1169; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1170; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1171; SSSE3-NEXT:    pslld $31, %xmm0
1172; SSSE3-NEXT:    psrad $31, %xmm0
1173; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1174; SSSE3-NEXT:    pslld $31, %xmm1
1175; SSSE3-NEXT:    psrad $31, %xmm1
1176; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1177; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1178; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1179; SSSE3-NEXT:    pslld $31, %xmm2
1180; SSSE3-NEXT:    psrad $31, %xmm2
1181; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1182; SSSE3-NEXT:    pslld $31, %xmm3
1183; SSSE3-NEXT:    psrad $31, %xmm3
1184; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1185; SSSE3-NEXT:    retq
1186;
1187; SSE41-LABEL: umulo_v16i8:
1188; SSE41:       # %bb.0:
1189; SSE41-NEXT:    movdqa %xmm1, %xmm2
1190; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1191; SSE41-NEXT:    movdqa %xmm0, %xmm3
1192; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1193; SSE41-NEXT:    pmullw %xmm2, %xmm3
1194; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
1195; SSE41-NEXT:    pand %xmm4, %xmm3
1196; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1197; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1198; SSE41-NEXT:    pmullw %xmm2, %xmm5
1199; SSE41-NEXT:    pand %xmm5, %xmm4
1200; SSE41-NEXT:    packuswb %xmm3, %xmm4
1201; SSE41-NEXT:    pxor %xmm2, %xmm2
1202; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1203; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1204; SSE41-NEXT:    pmullw %xmm1, %xmm0
1205; SSE41-NEXT:    psrlw $8, %xmm0
1206; SSE41-NEXT:    psrlw $8, %xmm5
1207; SSE41-NEXT:    packuswb %xmm0, %xmm5
1208; SSE41-NEXT:    pcmpeqb %xmm2, %xmm5
1209; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1210; SSE41-NEXT:    pxor %xmm5, %xmm3
1211; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1212; SSE41-NEXT:    pslld $31, %xmm0
1213; SSE41-NEXT:    psrad $31, %xmm0
1214; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
1215; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1216; SSE41-NEXT:    pslld $31, %xmm1
1217; SSE41-NEXT:    psrad $31, %xmm1
1218; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
1219; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1220; SSE41-NEXT:    pslld $31, %xmm2
1221; SSE41-NEXT:    psrad $31, %xmm2
1222; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
1223; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1224; SSE41-NEXT:    pslld $31, %xmm3
1225; SSE41-NEXT:    psrad $31, %xmm3
1226; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1227; SSE41-NEXT:    retq
1228;
1229; AVX1-LABEL: umulo_v16i8:
1230; AVX1:       # %bb.0:
1231; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1232; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1233; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1234; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1235; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1236; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1237; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1238; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1239; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1240; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1241; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1242; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1243; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1244; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1245; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1246; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm1
1247; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1248; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1249; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1250; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1251; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1252; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
1253; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1254; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1255; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
1256; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1257; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
1258; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1259; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1260; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1261; AVX1-NEXT:    retq
1262;
1263; AVX2-LABEL: umulo_v16i8:
1264; AVX2:       # %bb.0:
1265; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1266; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1267; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1268; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm1
1269; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
1270; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm2
1271; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1272; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1273; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1274; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1275; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
1276; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1277; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1278; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1279; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1280; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1281; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1282; AVX2-NEXT:    retq
1283;
1284; AVX512-LABEL: umulo_v16i8:
1285; AVX512:       # %bb.0:
1286; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1287; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1288; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1289; AVX512-NEXT:    vpsrlw $8, %ymm1, %ymm0
1290; AVX512-NEXT:    vpmovwb %ymm0, %xmm0
1291; AVX512-NEXT:    vptestmb %xmm0, %xmm0, %k1
1292; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1293; AVX512-NEXT:    vpmovwb %ymm1, (%rdi)
1294; AVX512-NEXT:    retq
1295  %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1296  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1297  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1298  %res = sext <16 x i1> %obit to <16 x i32>
1299  store <16 x i8> %val, <16 x i8>* %p2
1300  ret <16 x i32> %res
1301}
1302
1303define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
1304; SSE2-LABEL: umulo_v8i16:
1305; SSE2:       # %bb.0:
1306; SSE2-NEXT:    movdqa %xmm0, %xmm2
1307; SSE2-NEXT:    pmullw %xmm1, %xmm2
1308; SSE2-NEXT:    pmulhuw %xmm1, %xmm0
1309; SSE2-NEXT:    pxor %xmm3, %xmm3
1310; SSE2-NEXT:    pcmpeqw %xmm0, %xmm3
1311; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
1312; SSE2-NEXT:    pxor %xmm3, %xmm1
1313; SSE2-NEXT:    movdqa %xmm1, %xmm0
1314; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1315; SSE2-NEXT:    pslld $31, %xmm0
1316; SSE2-NEXT:    psrad $31, %xmm0
1317; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1318; SSE2-NEXT:    pslld $31, %xmm1
1319; SSE2-NEXT:    psrad $31, %xmm1
1320; SSE2-NEXT:    movdqa %xmm2, (%rdi)
1321; SSE2-NEXT:    retq
1322;
1323; SSSE3-LABEL: umulo_v8i16:
1324; SSSE3:       # %bb.0:
1325; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1326; SSSE3-NEXT:    pmullw %xmm1, %xmm2
1327; SSSE3-NEXT:    pmulhuw %xmm1, %xmm0
1328; SSSE3-NEXT:    pxor %xmm3, %xmm3
1329; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm3
1330; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
1331; SSSE3-NEXT:    pxor %xmm3, %xmm1
1332; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1333; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1334; SSSE3-NEXT:    pslld $31, %xmm0
1335; SSSE3-NEXT:    psrad $31, %xmm0
1336; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1337; SSSE3-NEXT:    pslld $31, %xmm1
1338; SSSE3-NEXT:    psrad $31, %xmm1
1339; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
1340; SSSE3-NEXT:    retq
1341;
1342; SSE41-LABEL: umulo_v8i16:
1343; SSE41:       # %bb.0:
1344; SSE41-NEXT:    movdqa %xmm0, %xmm2
1345; SSE41-NEXT:    pmullw %xmm1, %xmm2
1346; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
1347; SSE41-NEXT:    pxor %xmm3, %xmm3
1348; SSE41-NEXT:    pcmpeqw %xmm0, %xmm3
1349; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
1350; SSE41-NEXT:    pxor %xmm3, %xmm1
1351; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1352; SSE41-NEXT:    pslld $31, %xmm0
1353; SSE41-NEXT:    psrad $31, %xmm0
1354; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1355; SSE41-NEXT:    pslld $31, %xmm1
1356; SSE41-NEXT:    psrad $31, %xmm1
1357; SSE41-NEXT:    movdqa %xmm2, (%rdi)
1358; SSE41-NEXT:    retq
1359;
1360; AVX1-LABEL: umulo_v8i16:
1361; AVX1:       # %bb.0:
1362; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1363; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1364; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1365; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
1366; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1367; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
1368; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
1369; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1370; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
1371; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1372; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1373; AVX1-NEXT:    retq
1374;
1375; AVX2-LABEL: umulo_v8i16:
1376; AVX2:       # %bb.0:
1377; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1378; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1379; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1380; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
1381; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1382; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
1383; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
1384; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1385; AVX2-NEXT:    retq
1386;
1387; AVX512-LABEL: umulo_v8i16:
1388; AVX512:       # %bb.0:
1389; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1390; AVX512-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1391; AVX512-NEXT:    vptestmw %xmm0, %xmm0, %k1
1392; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
1393; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
1394; AVX512-NEXT:    vmovdqa %xmm2, (%rdi)
1395; AVX512-NEXT:    retq
1396  %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
1397  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
1398  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
1399  %res = sext <8 x i1> %obit to <8 x i32>
1400  store <8 x i16> %val, <8 x i16>* %p2
1401  ret <8 x i32> %res
1402}
1403
1404define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
1405; SSE2-LABEL: umulo_v2i64:
1406; SSE2:       # %bb.0:
1407; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1408; SSE2-NEXT:    movq %xmm2, %r8
1409; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1410; SSE2-NEXT:    movq %xmm2, %r10
1411; SSE2-NEXT:    movq %xmm0, %rax
1412; SSE2-NEXT:    movq %xmm1, %rdx
1413; SSE2-NEXT:    xorl %ecx, %ecx
1414; SSE2-NEXT:    mulq %rdx
1415; SSE2-NEXT:    movq $-1, %r9
1416; SSE2-NEXT:    movl $0, %esi
1417; SSE2-NEXT:    cmovoq %r9, %rsi
1418; SSE2-NEXT:    movq %rax, %xmm1
1419; SSE2-NEXT:    movq %r8, %rax
1420; SSE2-NEXT:    mulq %r10
1421; SSE2-NEXT:    movq %rax, %xmm0
1422; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1423; SSE2-NEXT:    movq %rsi, %xmm0
1424; SSE2-NEXT:    cmovoq %r9, %rcx
1425; SSE2-NEXT:    movq %rcx, %xmm2
1426; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1427; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1428; SSE2-NEXT:    movdqa %xmm1, (%rdi)
1429; SSE2-NEXT:    retq
1430;
1431; SSSE3-LABEL: umulo_v2i64:
1432; SSSE3:       # %bb.0:
1433; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1434; SSSE3-NEXT:    movq %xmm2, %r8
1435; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1436; SSSE3-NEXT:    movq %xmm2, %r10
1437; SSSE3-NEXT:    movq %xmm0, %rax
1438; SSSE3-NEXT:    movq %xmm1, %rdx
1439; SSSE3-NEXT:    xorl %ecx, %ecx
1440; SSSE3-NEXT:    mulq %rdx
1441; SSSE3-NEXT:    movq $-1, %r9
1442; SSSE3-NEXT:    movl $0, %esi
1443; SSSE3-NEXT:    cmovoq %r9, %rsi
1444; SSSE3-NEXT:    movq %rax, %xmm1
1445; SSSE3-NEXT:    movq %r8, %rax
1446; SSSE3-NEXT:    mulq %r10
1447; SSSE3-NEXT:    movq %rax, %xmm0
1448; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1449; SSSE3-NEXT:    movq %rsi, %xmm0
1450; SSSE3-NEXT:    cmovoq %r9, %rcx
1451; SSSE3-NEXT:    movq %rcx, %xmm2
1452; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1453; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1454; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
1455; SSSE3-NEXT:    retq
1456;
1457; SSE41-LABEL: umulo_v2i64:
1458; SSE41:       # %bb.0:
1459; SSE41-NEXT:    movq %xmm0, %r10
1460; SSE41-NEXT:    movq %xmm1, %r8
1461; SSE41-NEXT:    pextrq $1, %xmm0, %rax
1462; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
1463; SSE41-NEXT:    xorl %esi, %esi
1464; SSE41-NEXT:    mulq %rdx
1465; SSE41-NEXT:    movq $-1, %r9
1466; SSE41-NEXT:    movl $0, %ecx
1467; SSE41-NEXT:    cmovoq %r9, %rcx
1468; SSE41-NEXT:    movq %rax, %xmm0
1469; SSE41-NEXT:    movq %r10, %rax
1470; SSE41-NEXT:    mulq %r8
1471; SSE41-NEXT:    movq %rax, %xmm1
1472; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1473; SSE41-NEXT:    movq %rcx, %xmm0
1474; SSE41-NEXT:    cmovoq %r9, %rsi
1475; SSE41-NEXT:    movq %rsi, %xmm2
1476; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
1477; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
1478; SSE41-NEXT:    movdqa %xmm1, (%rdi)
1479; SSE41-NEXT:    retq
1480;
1481; AVX1-LABEL: umulo_v2i64:
1482; AVX1:       # %bb.0:
1483; AVX1-NEXT:    vmovq %xmm0, %r10
1484; AVX1-NEXT:    vmovq %xmm1, %r8
1485; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
1486; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
1487; AVX1-NEXT:    xorl %esi, %esi
1488; AVX1-NEXT:    mulq %rdx
1489; AVX1-NEXT:    movq $-1, %r9
1490; AVX1-NEXT:    movl $0, %ecx
1491; AVX1-NEXT:    cmovoq %r9, %rcx
1492; AVX1-NEXT:    vmovq %rax, %xmm0
1493; AVX1-NEXT:    movq %r10, %rax
1494; AVX1-NEXT:    mulq %r8
1495; AVX1-NEXT:    vmovq %rax, %xmm1
1496; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1497; AVX1-NEXT:    vmovq %rcx, %xmm0
1498; AVX1-NEXT:    cmovoq %r9, %rsi
1499; AVX1-NEXT:    vmovq %rsi, %xmm2
1500; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1501; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1502; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
1503; AVX1-NEXT:    retq
1504;
1505; AVX2-LABEL: umulo_v2i64:
1506; AVX2:       # %bb.0:
1507; AVX2-NEXT:    vmovq %xmm0, %r10
1508; AVX2-NEXT:    vmovq %xmm1, %r8
1509; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
1510; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
1511; AVX2-NEXT:    xorl %esi, %esi
1512; AVX2-NEXT:    mulq %rdx
1513; AVX2-NEXT:    movq $-1, %r9
1514; AVX2-NEXT:    movl $0, %ecx
1515; AVX2-NEXT:    cmovoq %r9, %rcx
1516; AVX2-NEXT:    vmovq %rax, %xmm0
1517; AVX2-NEXT:    movq %r10, %rax
1518; AVX2-NEXT:    mulq %r8
1519; AVX2-NEXT:    vmovq %rax, %xmm1
1520; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1521; AVX2-NEXT:    vmovq %rcx, %xmm0
1522; AVX2-NEXT:    cmovoq %r9, %rsi
1523; AVX2-NEXT:    vmovq %rsi, %xmm2
1524; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1525; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1526; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
1527; AVX2-NEXT:    retq
1528;
1529; AVX512-LABEL: umulo_v2i64:
1530; AVX512:       # %bb.0:
1531; AVX512-NEXT:    vmovq %xmm0, %rcx
1532; AVX512-NEXT:    vmovq %xmm1, %rsi
1533; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
1534; AVX512-NEXT:    vpextrq $1, %xmm1, %rdx
1535; AVX512-NEXT:    mulq %rdx
1536; AVX512-NEXT:    seto %r8b
1537; AVX512-NEXT:    vmovq %rax, %xmm0
1538; AVX512-NEXT:    movq %rcx, %rax
1539; AVX512-NEXT:    mulq %rsi
1540; AVX512-NEXT:    vmovq %rax, %xmm1
1541; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1542; AVX512-NEXT:    seto %al
1543; AVX512-NEXT:    movw $-3, %cx
1544; AVX512-NEXT:    kmovd %ecx, %k0
1545; AVX512-NEXT:    kmovd %eax, %k1
1546; AVX512-NEXT:    kandw %k0, %k1, %k0
1547; AVX512-NEXT:    kmovd %r8d, %k1
1548; AVX512-NEXT:    kshiftlw $15, %k1, %k1
1549; AVX512-NEXT:    kshiftrw $14, %k1, %k1
1550; AVX512-NEXT:    korw %k1, %k0, %k1
1551; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
1552; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1553; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
1554; AVX512-NEXT:    retq
1555  %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
1556  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
1557  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
1558  %res = sext <2 x i1> %obit to <2 x i32>
1559  store <2 x i64> %val, <2 x i64>* %p2
1560  ret <2 x i32> %res
1561}
1562
1563define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
1564; SSE2-LABEL: umulo_v4i24:
1565; SSE2:       # %bb.0:
1566; SSE2-NEXT:    movdqa %xmm0, %xmm2
1567; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1568; SSE2-NEXT:    pand %xmm0, %xmm1
1569; SSE2-NEXT:    pand %xmm0, %xmm2
1570; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
1571; SSE2-NEXT:    pmuludq %xmm1, %xmm2
1572; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
1573; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1574; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1575; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1576; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
1577; SSE2-NEXT:    pxor %xmm4, %xmm4
1578; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
1579; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
1580; SSE2-NEXT:    pxor %xmm5, %xmm3
1581; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
1582; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
1583; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
1584; SSE2-NEXT:    psrld $24, %xmm0
1585; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
1586; SSE2-NEXT:    pxor %xmm5, %xmm0
1587; SSE2-NEXT:    por %xmm3, %xmm0
1588; SSE2-NEXT:    movd %xmm2, %eax
1589; SSE2-NEXT:    movw %ax, (%rdi)
1590; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1591; SSE2-NEXT:    movd %xmm2, %ecx
1592; SSE2-NEXT:    movw %cx, 6(%rdi)
1593; SSE2-NEXT:    movd %xmm1, %edx
1594; SSE2-NEXT:    movw %dx, 3(%rdi)
1595; SSE2-NEXT:    shrl $16, %eax
1596; SSE2-NEXT:    movb %al, 2(%rdi)
1597; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1598; SSE2-NEXT:    movd %xmm1, %eax
1599; SSE2-NEXT:    movw %ax, 9(%rdi)
1600; SSE2-NEXT:    shrl $16, %ecx
1601; SSE2-NEXT:    movb %cl, 8(%rdi)
1602; SSE2-NEXT:    shrl $16, %edx
1603; SSE2-NEXT:    movb %dl, 5(%rdi)
1604; SSE2-NEXT:    shrl $16, %eax
1605; SSE2-NEXT:    movb %al, 11(%rdi)
1606; SSE2-NEXT:    retq
1607;
1608; SSSE3-LABEL: umulo_v4i24:
1609; SSSE3:       # %bb.0:
1610; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1611; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1612; SSSE3-NEXT:    pand %xmm0, %xmm1
1613; SSSE3-NEXT:    pand %xmm0, %xmm2
1614; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
1615; SSSE3-NEXT:    pmuludq %xmm1, %xmm2
1616; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
1617; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1618; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
1619; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1620; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
1621; SSSE3-NEXT:    pxor %xmm4, %xmm4
1622; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
1623; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
1624; SSSE3-NEXT:    pxor %xmm5, %xmm3
1625; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
1626; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
1627; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
1628; SSSE3-NEXT:    psrld $24, %xmm0
1629; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
1630; SSSE3-NEXT:    pxor %xmm5, %xmm0
1631; SSSE3-NEXT:    por %xmm3, %xmm0
1632; SSSE3-NEXT:    movd %xmm2, %eax
1633; SSSE3-NEXT:    movw %ax, (%rdi)
1634; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1635; SSSE3-NEXT:    movd %xmm2, %ecx
1636; SSSE3-NEXT:    movw %cx, 6(%rdi)
1637; SSSE3-NEXT:    movd %xmm1, %edx
1638; SSSE3-NEXT:    movw %dx, 3(%rdi)
1639; SSSE3-NEXT:    shrl $16, %eax
1640; SSSE3-NEXT:    movb %al, 2(%rdi)
1641; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1642; SSSE3-NEXT:    movd %xmm1, %eax
1643; SSSE3-NEXT:    movw %ax, 9(%rdi)
1644; SSSE3-NEXT:    shrl $16, %ecx
1645; SSSE3-NEXT:    movb %cl, 8(%rdi)
1646; SSSE3-NEXT:    shrl $16, %edx
1647; SSSE3-NEXT:    movb %dl, 5(%rdi)
1648; SSSE3-NEXT:    shrl $16, %eax
1649; SSSE3-NEXT:    movb %al, 11(%rdi)
1650; SSSE3-NEXT:    retq
1651;
1652; SSE41-LABEL: umulo_v4i24:
1653; SSE41:       # %bb.0:
1654; SSE41-NEXT:    movdqa %xmm0, %xmm2
1655; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1656; SSE41-NEXT:    pand %xmm0, %xmm2
1657; SSE41-NEXT:    pand %xmm0, %xmm1
1658; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
1659; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
1660; SSE41-NEXT:    pmuludq %xmm0, %xmm3
1661; SSE41-NEXT:    movdqa %xmm2, %xmm0
1662; SSE41-NEXT:    pmuludq %xmm1, %xmm0
1663; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1664; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
1665; SSE41-NEXT:    pxor %xmm0, %xmm0
1666; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
1667; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1668; SSE41-NEXT:    pxor %xmm3, %xmm4
1669; SSE41-NEXT:    pmulld %xmm2, %xmm1
1670; SSE41-NEXT:    pextrd $3, %xmm1, %eax
1671; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
1672; SSE41-NEXT:    pextrd $1, %xmm1, %edx
1673; SSE41-NEXT:    movd %xmm1, %esi
1674; SSE41-NEXT:    psrld $24, %xmm1
1675; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
1676; SSE41-NEXT:    pxor %xmm3, %xmm0
1677; SSE41-NEXT:    por %xmm4, %xmm0
1678; SSE41-NEXT:    movw %ax, 9(%rdi)
1679; SSE41-NEXT:    movw %cx, 6(%rdi)
1680; SSE41-NEXT:    movw %dx, 3(%rdi)
1681; SSE41-NEXT:    movw %si, (%rdi)
1682; SSE41-NEXT:    shrl $16, %eax
1683; SSE41-NEXT:    movb %al, 11(%rdi)
1684; SSE41-NEXT:    shrl $16, %ecx
1685; SSE41-NEXT:    movb %cl, 8(%rdi)
1686; SSE41-NEXT:    shrl $16, %edx
1687; SSE41-NEXT:    movb %dl, 5(%rdi)
1688; SSE41-NEXT:    shrl $16, %esi
1689; SSE41-NEXT:    movb %sil, 2(%rdi)
1690; SSE41-NEXT:    retq
1691;
1692; AVX1-LABEL: umulo_v4i24:
1693; AVX1:       # %bb.0:
1694; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
1695; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
1696; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
1697; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[1,1,3,3]
1698; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[1,1,3,3]
1699; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1700; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1701; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1702; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1703; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1704; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1705; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1706; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1707; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1708; AVX1-NEXT:    vpsrld $24, %xmm1, %xmm0
1709; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1710; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1711; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
1712; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
1713; AVX1-NEXT:    movw %ax, 9(%rdi)
1714; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
1715; AVX1-NEXT:    movw %cx, 6(%rdi)
1716; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
1717; AVX1-NEXT:    movw %dx, 3(%rdi)
1718; AVX1-NEXT:    vmovd %xmm1, %esi
1719; AVX1-NEXT:    movw %si, (%rdi)
1720; AVX1-NEXT:    shrl $16, %eax
1721; AVX1-NEXT:    movb %al, 11(%rdi)
1722; AVX1-NEXT:    shrl $16, %ecx
1723; AVX1-NEXT:    movb %cl, 8(%rdi)
1724; AVX1-NEXT:    shrl $16, %edx
1725; AVX1-NEXT:    movb %dl, 5(%rdi)
1726; AVX1-NEXT:    shrl $16, %esi
1727; AVX1-NEXT:    movb %sil, 2(%rdi)
1728; AVX1-NEXT:    retq
1729;
1730; AVX2-LABEL: umulo_v4i24:
1731; AVX2:       # %bb.0:
1732; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
1733; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
1734; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
1735; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1736; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1737; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1738; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1739; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1740; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
1741; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1742; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1743; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1744; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1745; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1746; AVX2-NEXT:    vpsrld $24, %xmm1, %xmm0
1747; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1748; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1749; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
1750; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
1751; AVX2-NEXT:    movw %ax, 9(%rdi)
1752; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
1753; AVX2-NEXT:    movw %cx, 6(%rdi)
1754; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
1755; AVX2-NEXT:    movw %dx, 3(%rdi)
1756; AVX2-NEXT:    vmovd %xmm1, %esi
1757; AVX2-NEXT:    movw %si, (%rdi)
1758; AVX2-NEXT:    shrl $16, %eax
1759; AVX2-NEXT:    movb %al, 11(%rdi)
1760; AVX2-NEXT:    shrl $16, %ecx
1761; AVX2-NEXT:    movb %cl, 8(%rdi)
1762; AVX2-NEXT:    shrl $16, %edx
1763; AVX2-NEXT:    movb %dl, 5(%rdi)
1764; AVX2-NEXT:    shrl $16, %esi
1765; AVX2-NEXT:    movb %sil, 2(%rdi)
1766; AVX2-NEXT:    retq
1767;
1768; AVX512-LABEL: umulo_v4i24:
1769; AVX512:       # %bb.0:
1770; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
1771; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
1772; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
1773; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
1774; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
1775; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1776; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
1777; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
1778; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
1779; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1780; AVX512-NEXT:    vpsrld $24, %xmm1, %xmm0
1781; AVX512-NEXT:    vpor %xmm4, %xmm0, %xmm0
1782; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
1783; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
1784; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1785; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
1786; AVX512-NEXT:    movw %ax, 9(%rdi)
1787; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
1788; AVX512-NEXT:    movw %cx, 6(%rdi)
1789; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
1790; AVX512-NEXT:    movw %dx, 3(%rdi)
1791; AVX512-NEXT:    vmovd %xmm1, %esi
1792; AVX512-NEXT:    movw %si, (%rdi)
1793; AVX512-NEXT:    shrl $16, %eax
1794; AVX512-NEXT:    movb %al, 11(%rdi)
1795; AVX512-NEXT:    shrl $16, %ecx
1796; AVX512-NEXT:    movb %cl, 8(%rdi)
1797; AVX512-NEXT:    shrl $16, %edx
1798; AVX512-NEXT:    movb %dl, 5(%rdi)
1799; AVX512-NEXT:    shrl $16, %esi
1800; AVX512-NEXT:    movb %sil, 2(%rdi)
1801; AVX512-NEXT:    retq
1802  %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
1803  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
1804  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
1805  %res = sext <4 x i1> %obit to <4 x i32>
1806  store <4 x i24> %val, <4 x i24>* %p2
1807  ret <4 x i32> %res
1808}
1809
1810define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
1811; SSE2-LABEL: umulo_v4i1:
1812; SSE2:       # %bb.0:
1813; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1814; SSE2-NEXT:    pand %xmm2, %xmm1
1815; SSE2-NEXT:    pand %xmm2, %xmm0
1816; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
1817; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1818; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1819; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1820; SSE2-NEXT:    pmuludq %xmm2, %xmm1
1821; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
1822; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1823; SSE2-NEXT:    pxor %xmm2, %xmm2
1824; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
1825; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
1826; SSE2-NEXT:    pxor %xmm4, %xmm3
1827; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,2,2,3]
1828; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1829; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
1830; SSE2-NEXT:    movdqa %xmm5, %xmm0
1831; SSE2-NEXT:    psrld $1, %xmm0
1832; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
1833; SSE2-NEXT:    pxor %xmm4, %xmm0
1834; SSE2-NEXT:    por %xmm3, %xmm0
1835; SSE2-NEXT:    pslld $31, %xmm5
1836; SSE2-NEXT:    movmskps %xmm5, %eax
1837; SSE2-NEXT:    movb %al, (%rdi)
1838; SSE2-NEXT:    retq
1839;
1840; SSSE3-LABEL: umulo_v4i1:
1841; SSSE3:       # %bb.0:
1842; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1843; SSSE3-NEXT:    pand %xmm2, %xmm1
1844; SSSE3-NEXT:    pand %xmm2, %xmm0
1845; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
1846; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
1847; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1848; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1849; SSSE3-NEXT:    pmuludq %xmm2, %xmm1
1850; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
1851; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1852; SSSE3-NEXT:    pxor %xmm2, %xmm2
1853; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
1854; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
1855; SSSE3-NEXT:    pxor %xmm4, %xmm3
1856; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,2,2,3]
1857; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1858; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
1859; SSSE3-NEXT:    movdqa %xmm5, %xmm0
1860; SSSE3-NEXT:    psrld $1, %xmm0
1861; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
1862; SSSE3-NEXT:    pxor %xmm4, %xmm0
1863; SSSE3-NEXT:    por %xmm3, %xmm0
1864; SSSE3-NEXT:    pslld $31, %xmm5
1865; SSSE3-NEXT:    movmskps %xmm5, %eax
1866; SSSE3-NEXT:    movb %al, (%rdi)
1867; SSSE3-NEXT:    retq
1868;
1869; SSE41-LABEL: umulo_v4i1:
1870; SSE41:       # %bb.0:
1871; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1872; SSE41-NEXT:    pand %xmm2, %xmm0
1873; SSE41-NEXT:    pand %xmm2, %xmm1
1874; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1875; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1876; SSE41-NEXT:    pmuludq %xmm2, %xmm3
1877; SSE41-NEXT:    movdqa %xmm0, %xmm2
1878; SSE41-NEXT:    pmuludq %xmm1, %xmm2
1879; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
1880; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
1881; SSE41-NEXT:    pxor %xmm2, %xmm2
1882; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
1883; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1884; SSE41-NEXT:    pxor %xmm3, %xmm4
1885; SSE41-NEXT:    pmaddwd %xmm0, %xmm1
1886; SSE41-NEXT:    movdqa %xmm1, %xmm0
1887; SSE41-NEXT:    psrld $1, %xmm0
1888; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
1889; SSE41-NEXT:    pxor %xmm3, %xmm2
1890; SSE41-NEXT:    por %xmm4, %xmm2
1891; SSE41-NEXT:    pslld $31, %xmm1
1892; SSE41-NEXT:    movmskps %xmm1, %eax
1893; SSE41-NEXT:    movb %al, (%rdi)
1894; SSE41-NEXT:    movdqa %xmm2, %xmm0
1895; SSE41-NEXT:    retq
1896;
1897; AVX1-LABEL: umulo_v4i1:
1898; AVX1:       # %bb.0:
1899; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1]
1900; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
1901; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
1902; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1903; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1904; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1905; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1906; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1907; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1908; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1909; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1910; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1911; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1912; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm1
1913; AVX1-NEXT:    vpsrld $1, %xmm1, %xmm0
1914; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1915; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1916; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
1917; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
1918; AVX1-NEXT:    vmovmskps %xmm1, %eax
1919; AVX1-NEXT:    movb %al, (%rdi)
1920; AVX1-NEXT:    retq
1921;
1922; AVX2-LABEL: umulo_v4i1:
1923; AVX2:       # %bb.0:
1924; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
1925; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
1926; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
1927; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1928; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1929; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1930; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1931; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1932; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
1933; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1934; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1935; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1936; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1937; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm1
1938; AVX2-NEXT:    vpsrld $1, %xmm1, %xmm0
1939; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1940; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1941; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
1942; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
1943; AVX2-NEXT:    vmovmskps %xmm1, %eax
1944; AVX2-NEXT:    movb %al, (%rdi)
1945; AVX2-NEXT:    retq
1946;
1947; AVX512-LABEL: umulo_v4i1:
1948; AVX512:       # %bb.0:
1949; AVX512-NEXT:    pushq %rbx
1950; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
1951; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
1952; AVX512-NEXT:    kshiftrw $3, %k0, %k1
1953; AVX512-NEXT:    kmovd %k1, %r9d
1954; AVX512-NEXT:    andb $1, %r9b
1955; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
1956; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
1957; AVX512-NEXT:    kshiftrw $3, %k1, %k2
1958; AVX512-NEXT:    kmovd %k2, %r10d
1959; AVX512-NEXT:    andb $1, %r10b
1960; AVX512-NEXT:    kshiftrw $2, %k0, %k2
1961; AVX512-NEXT:    kmovd %k2, %r11d
1962; AVX512-NEXT:    andb $1, %r11b
1963; AVX512-NEXT:    kshiftrw $2, %k1, %k2
1964; AVX512-NEXT:    kmovd %k2, %ebx
1965; AVX512-NEXT:    andb $1, %bl
1966; AVX512-NEXT:    kshiftrw $1, %k0, %k2
1967; AVX512-NEXT:    kmovd %k2, %edx
1968; AVX512-NEXT:    andb $1, %dl
1969; AVX512-NEXT:    kshiftrw $1, %k1, %k2
1970; AVX512-NEXT:    kmovd %k2, %esi
1971; AVX512-NEXT:    andb $1, %sil
1972; AVX512-NEXT:    kmovd %k0, %eax
1973; AVX512-NEXT:    andb $1, %al
1974; AVX512-NEXT:    kmovd %k1, %ecx
1975; AVX512-NEXT:    andb $1, %cl
1976; AVX512-NEXT:    # kill: def $al killed $al killed $eax
1977; AVX512-NEXT:    mulb %cl
1978; AVX512-NEXT:    movl %eax, %r8d
1979; AVX512-NEXT:    seto %al
1980; AVX512-NEXT:    testb $-2, %r8b
1981; AVX512-NEXT:    setne %cl
1982; AVX512-NEXT:    orb %al, %cl
1983; AVX512-NEXT:    setne %al
1984; AVX512-NEXT:    kmovd %eax, %k1
1985; AVX512-NEXT:    movw $-3, %ax
1986; AVX512-NEXT:    kmovd %eax, %k0
1987; AVX512-NEXT:    kandw %k0, %k1, %k1
1988; AVX512-NEXT:    movl %edx, %eax
1989; AVX512-NEXT:    mulb %sil
1990; AVX512-NEXT:    movl %eax, %edx
1991; AVX512-NEXT:    seto %al
1992; AVX512-NEXT:    testb $-2, %dl
1993; AVX512-NEXT:    setne %cl
1994; AVX512-NEXT:    orb %al, %cl
1995; AVX512-NEXT:    setne %al
1996; AVX512-NEXT:    kmovd %eax, %k2
1997; AVX512-NEXT:    kshiftlw $15, %k2, %k2
1998; AVX512-NEXT:    kshiftrw $14, %k2, %k2
1999; AVX512-NEXT:    korw %k2, %k1, %k2
2000; AVX512-NEXT:    movw $-5, %ax
2001; AVX512-NEXT:    kmovd %eax, %k1
2002; AVX512-NEXT:    kandw %k1, %k2, %k2
2003; AVX512-NEXT:    movl %r11d, %eax
2004; AVX512-NEXT:    mulb %bl
2005; AVX512-NEXT:    movl %eax, %esi
2006; AVX512-NEXT:    seto %al
2007; AVX512-NEXT:    testb $-2, %sil
2008; AVX512-NEXT:    setne %cl
2009; AVX512-NEXT:    orb %al, %cl
2010; AVX512-NEXT:    setne %al
2011; AVX512-NEXT:    kmovd %eax, %k3
2012; AVX512-NEXT:    kshiftlw $2, %k3, %k3
2013; AVX512-NEXT:    korw %k3, %k2, %k2
2014; AVX512-NEXT:    kshiftlw $13, %k2, %k2
2015; AVX512-NEXT:    kshiftrw $13, %k2, %k2
2016; AVX512-NEXT:    movl %r9d, %eax
2017; AVX512-NEXT:    mulb %r10b
2018; AVX512-NEXT:    # kill: def $al killed $al def $eax
2019; AVX512-NEXT:    seto %cl
2020; AVX512-NEXT:    testb $-2, %al
2021; AVX512-NEXT:    setne %bl
2022; AVX512-NEXT:    orb %cl, %bl
2023; AVX512-NEXT:    setne %cl
2024; AVX512-NEXT:    kmovd %ecx, %k3
2025; AVX512-NEXT:    kshiftlw $3, %k3, %k3
2026; AVX512-NEXT:    korw %k3, %k2, %k2
2027; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2028; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
2029; AVX512-NEXT:    kmovd %r8d, %k2
2030; AVX512-NEXT:    kandw %k0, %k2, %k0
2031; AVX512-NEXT:    kmovd %edx, %k2
2032; AVX512-NEXT:    kshiftlw $15, %k2, %k2
2033; AVX512-NEXT:    kshiftrw $14, %k2, %k2
2034; AVX512-NEXT:    korw %k2, %k0, %k0
2035; AVX512-NEXT:    kandw %k1, %k0, %k0
2036; AVX512-NEXT:    kmovd %esi, %k1
2037; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2038; AVX512-NEXT:    kshiftrw $13, %k1, %k1
2039; AVX512-NEXT:    korw %k1, %k0, %k0
2040; AVX512-NEXT:    movw $-9, %cx
2041; AVX512-NEXT:    kmovd %ecx, %k1
2042; AVX512-NEXT:    kandw %k1, %k0, %k0
2043; AVX512-NEXT:    kmovd %eax, %k1
2044; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2045; AVX512-NEXT:    kshiftrw $12, %k1, %k1
2046; AVX512-NEXT:    korw %k1, %k0, %k0
2047; AVX512-NEXT:    kmovd %k0, %eax
2048; AVX512-NEXT:    movb %al, (%rdi)
2049; AVX512-NEXT:    popq %rbx
2050; AVX512-NEXT:    retq
2051  %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
2052  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
2053  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
2054  %res = sext <4 x i1> %obit to <4 x i32>
2055  store <4 x i1> %val, <4 x i1>* %p2
2056  ret <4 x i32> %res
2057}
2058
2059define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
2060; SSE2-LABEL: umulo_v2i128:
2061; SSE2:       # %bb.0:
2062; SSE2-NEXT:    pushq %rbp
2063; SSE2-NEXT:    pushq %r15
2064; SSE2-NEXT:    pushq %r14
2065; SSE2-NEXT:    pushq %r13
2066; SSE2-NEXT:    pushq %r12
2067; SSE2-NEXT:    pushq %rbx
2068; SSE2-NEXT:    movq %r9, %r10
2069; SSE2-NEXT:    movq %rcx, %r12
2070; SSE2-NEXT:    movq %rdx, %r11
2071; SSE2-NEXT:    movq %rsi, %rax
2072; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2073; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2074; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2075; SSE2-NEXT:    testq %r10, %r10
2076; SSE2-NEXT:    setne %cl
2077; SSE2-NEXT:    testq %rsi, %rsi
2078; SSE2-NEXT:    setne %r13b
2079; SSE2-NEXT:    andb %cl, %r13b
2080; SSE2-NEXT:    mulq %r8
2081; SSE2-NEXT:    movq %rax, %rsi
2082; SSE2-NEXT:    seto %bpl
2083; SSE2-NEXT:    movq %r10, %rax
2084; SSE2-NEXT:    mulq %rdi
2085; SSE2-NEXT:    movq %rax, %rcx
2086; SSE2-NEXT:    seto %bl
2087; SSE2-NEXT:    orb %bpl, %bl
2088; SSE2-NEXT:    addq %rsi, %rcx
2089; SSE2-NEXT:    movq %rdi, %rax
2090; SSE2-NEXT:    mulq %r8
2091; SSE2-NEXT:    movq %rax, %rdi
2092; SSE2-NEXT:    movq %rdx, %rsi
2093; SSE2-NEXT:    addq %rcx, %rsi
2094; SSE2-NEXT:    setb %cl
2095; SSE2-NEXT:    orb %bl, %cl
2096; SSE2-NEXT:    orb %r13b, %cl
2097; SSE2-NEXT:    testq %r9, %r9
2098; SSE2-NEXT:    setne %al
2099; SSE2-NEXT:    testq %r12, %r12
2100; SSE2-NEXT:    setne %r8b
2101; SSE2-NEXT:    andb %al, %r8b
2102; SSE2-NEXT:    movq %r12, %rax
2103; SSE2-NEXT:    mulq %r15
2104; SSE2-NEXT:    movq %rax, %rbp
2105; SSE2-NEXT:    seto %r10b
2106; SSE2-NEXT:    movq %r9, %rax
2107; SSE2-NEXT:    mulq %r11
2108; SSE2-NEXT:    movq %rax, %rbx
2109; SSE2-NEXT:    seto %r9b
2110; SSE2-NEXT:    orb %r10b, %r9b
2111; SSE2-NEXT:    addq %rbp, %rbx
2112; SSE2-NEXT:    movq %r11, %rax
2113; SSE2-NEXT:    mulq %r15
2114; SSE2-NEXT:    addq %rbx, %rdx
2115; SSE2-NEXT:    setb %bl
2116; SSE2-NEXT:    orb %r9b, %bl
2117; SSE2-NEXT:    orb %r8b, %bl
2118; SSE2-NEXT:    movzbl %bl, %ebp
2119; SSE2-NEXT:    negl %ebp
2120; SSE2-NEXT:    movd %ebp, %xmm1
2121; SSE2-NEXT:    movzbl %cl, %ecx
2122; SSE2-NEXT:    negl %ecx
2123; SSE2-NEXT:    movd %ecx, %xmm0
2124; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2125; SSE2-NEXT:    movq %rax, 16(%r14)
2126; SSE2-NEXT:    movq %rdi, (%r14)
2127; SSE2-NEXT:    movq %rdx, 24(%r14)
2128; SSE2-NEXT:    movq %rsi, 8(%r14)
2129; SSE2-NEXT:    popq %rbx
2130; SSE2-NEXT:    popq %r12
2131; SSE2-NEXT:    popq %r13
2132; SSE2-NEXT:    popq %r14
2133; SSE2-NEXT:    popq %r15
2134; SSE2-NEXT:    popq %rbp
2135; SSE2-NEXT:    retq
2136;
2137; SSSE3-LABEL: umulo_v2i128:
2138; SSSE3:       # %bb.0:
2139; SSSE3-NEXT:    pushq %rbp
2140; SSSE3-NEXT:    pushq %r15
2141; SSSE3-NEXT:    pushq %r14
2142; SSSE3-NEXT:    pushq %r13
2143; SSSE3-NEXT:    pushq %r12
2144; SSSE3-NEXT:    pushq %rbx
2145; SSSE3-NEXT:    movq %r9, %r10
2146; SSSE3-NEXT:    movq %rcx, %r12
2147; SSSE3-NEXT:    movq %rdx, %r11
2148; SSSE3-NEXT:    movq %rsi, %rax
2149; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2150; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2151; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2152; SSSE3-NEXT:    testq %r10, %r10
2153; SSSE3-NEXT:    setne %cl
2154; SSSE3-NEXT:    testq %rsi, %rsi
2155; SSSE3-NEXT:    setne %r13b
2156; SSSE3-NEXT:    andb %cl, %r13b
2157; SSSE3-NEXT:    mulq %r8
2158; SSSE3-NEXT:    movq %rax, %rsi
2159; SSSE3-NEXT:    seto %bpl
2160; SSSE3-NEXT:    movq %r10, %rax
2161; SSSE3-NEXT:    mulq %rdi
2162; SSSE3-NEXT:    movq %rax, %rcx
2163; SSSE3-NEXT:    seto %bl
2164; SSSE3-NEXT:    orb %bpl, %bl
2165; SSSE3-NEXT:    addq %rsi, %rcx
2166; SSSE3-NEXT:    movq %rdi, %rax
2167; SSSE3-NEXT:    mulq %r8
2168; SSSE3-NEXT:    movq %rax, %rdi
2169; SSSE3-NEXT:    movq %rdx, %rsi
2170; SSSE3-NEXT:    addq %rcx, %rsi
2171; SSSE3-NEXT:    setb %cl
2172; SSSE3-NEXT:    orb %bl, %cl
2173; SSSE3-NEXT:    orb %r13b, %cl
2174; SSSE3-NEXT:    testq %r9, %r9
2175; SSSE3-NEXT:    setne %al
2176; SSSE3-NEXT:    testq %r12, %r12
2177; SSSE3-NEXT:    setne %r8b
2178; SSSE3-NEXT:    andb %al, %r8b
2179; SSSE3-NEXT:    movq %r12, %rax
2180; SSSE3-NEXT:    mulq %r15
2181; SSSE3-NEXT:    movq %rax, %rbp
2182; SSSE3-NEXT:    seto %r10b
2183; SSSE3-NEXT:    movq %r9, %rax
2184; SSSE3-NEXT:    mulq %r11
2185; SSSE3-NEXT:    movq %rax, %rbx
2186; SSSE3-NEXT:    seto %r9b
2187; SSSE3-NEXT:    orb %r10b, %r9b
2188; SSSE3-NEXT:    addq %rbp, %rbx
2189; SSSE3-NEXT:    movq %r11, %rax
2190; SSSE3-NEXT:    mulq %r15
2191; SSSE3-NEXT:    addq %rbx, %rdx
2192; SSSE3-NEXT:    setb %bl
2193; SSSE3-NEXT:    orb %r9b, %bl
2194; SSSE3-NEXT:    orb %r8b, %bl
2195; SSSE3-NEXT:    movzbl %bl, %ebp
2196; SSSE3-NEXT:    negl %ebp
2197; SSSE3-NEXT:    movd %ebp, %xmm1
2198; SSSE3-NEXT:    movzbl %cl, %ecx
2199; SSSE3-NEXT:    negl %ecx
2200; SSSE3-NEXT:    movd %ecx, %xmm0
2201; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2202; SSSE3-NEXT:    movq %rax, 16(%r14)
2203; SSSE3-NEXT:    movq %rdi, (%r14)
2204; SSSE3-NEXT:    movq %rdx, 24(%r14)
2205; SSSE3-NEXT:    movq %rsi, 8(%r14)
2206; SSSE3-NEXT:    popq %rbx
2207; SSSE3-NEXT:    popq %r12
2208; SSSE3-NEXT:    popq %r13
2209; SSSE3-NEXT:    popq %r14
2210; SSSE3-NEXT:    popq %r15
2211; SSSE3-NEXT:    popq %rbp
2212; SSSE3-NEXT:    retq
2213;
2214; SSE41-LABEL: umulo_v2i128:
2215; SSE41:       # %bb.0:
2216; SSE41-NEXT:    pushq %rbp
2217; SSE41-NEXT:    pushq %r15
2218; SSE41-NEXT:    pushq %r14
2219; SSE41-NEXT:    pushq %r13
2220; SSE41-NEXT:    pushq %r12
2221; SSE41-NEXT:    pushq %rbx
2222; SSE41-NEXT:    movq %r9, %r10
2223; SSE41-NEXT:    movq %rcx, %r12
2224; SSE41-NEXT:    movq %rdx, %r11
2225; SSE41-NEXT:    movq %rsi, %rax
2226; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2227; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2228; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2229; SSE41-NEXT:    testq %r10, %r10
2230; SSE41-NEXT:    setne %cl
2231; SSE41-NEXT:    testq %rsi, %rsi
2232; SSE41-NEXT:    setne %r13b
2233; SSE41-NEXT:    andb %cl, %r13b
2234; SSE41-NEXT:    mulq %r8
2235; SSE41-NEXT:    movq %rax, %rsi
2236; SSE41-NEXT:    seto %bpl
2237; SSE41-NEXT:    movq %r10, %rax
2238; SSE41-NEXT:    mulq %rdi
2239; SSE41-NEXT:    movq %rax, %rcx
2240; SSE41-NEXT:    seto %bl
2241; SSE41-NEXT:    orb %bpl, %bl
2242; SSE41-NEXT:    addq %rsi, %rcx
2243; SSE41-NEXT:    movq %rdi, %rax
2244; SSE41-NEXT:    mulq %r8
2245; SSE41-NEXT:    movq %rax, %rdi
2246; SSE41-NEXT:    movq %rdx, %rsi
2247; SSE41-NEXT:    addq %rcx, %rsi
2248; SSE41-NEXT:    setb %cl
2249; SSE41-NEXT:    orb %bl, %cl
2250; SSE41-NEXT:    orb %r13b, %cl
2251; SSE41-NEXT:    testq %r9, %r9
2252; SSE41-NEXT:    setne %al
2253; SSE41-NEXT:    testq %r12, %r12
2254; SSE41-NEXT:    setne %r8b
2255; SSE41-NEXT:    andb %al, %r8b
2256; SSE41-NEXT:    movq %r12, %rax
2257; SSE41-NEXT:    mulq %r15
2258; SSE41-NEXT:    movq %rax, %rbp
2259; SSE41-NEXT:    seto %r10b
2260; SSE41-NEXT:    movq %r9, %rax
2261; SSE41-NEXT:    mulq %r11
2262; SSE41-NEXT:    movq %rax, %rbx
2263; SSE41-NEXT:    seto %r9b
2264; SSE41-NEXT:    orb %r10b, %r9b
2265; SSE41-NEXT:    addq %rbp, %rbx
2266; SSE41-NEXT:    movq %r11, %rax
2267; SSE41-NEXT:    mulq %r15
2268; SSE41-NEXT:    addq %rbx, %rdx
2269; SSE41-NEXT:    setb %bl
2270; SSE41-NEXT:    orb %r9b, %bl
2271; SSE41-NEXT:    orb %r8b, %bl
2272; SSE41-NEXT:    movzbl %bl, %ebp
2273; SSE41-NEXT:    negl %ebp
2274; SSE41-NEXT:    movzbl %cl, %ecx
2275; SSE41-NEXT:    negl %ecx
2276; SSE41-NEXT:    movd %ecx, %xmm0
2277; SSE41-NEXT:    pinsrd $1, %ebp, %xmm0
2278; SSE41-NEXT:    movq %rax, 16(%r14)
2279; SSE41-NEXT:    movq %rdi, (%r14)
2280; SSE41-NEXT:    movq %rdx, 24(%r14)
2281; SSE41-NEXT:    movq %rsi, 8(%r14)
2282; SSE41-NEXT:    popq %rbx
2283; SSE41-NEXT:    popq %r12
2284; SSE41-NEXT:    popq %r13
2285; SSE41-NEXT:    popq %r14
2286; SSE41-NEXT:    popq %r15
2287; SSE41-NEXT:    popq %rbp
2288; SSE41-NEXT:    retq
2289;
2290; AVX1-LABEL: umulo_v2i128:
2291; AVX1:       # %bb.0:
2292; AVX1-NEXT:    pushq %rbp
2293; AVX1-NEXT:    pushq %r15
2294; AVX1-NEXT:    pushq %r14
2295; AVX1-NEXT:    pushq %r13
2296; AVX1-NEXT:    pushq %r12
2297; AVX1-NEXT:    pushq %rbx
2298; AVX1-NEXT:    movq %r9, %r10
2299; AVX1-NEXT:    movq %rcx, %r12
2300; AVX1-NEXT:    movq %rdx, %r11
2301; AVX1-NEXT:    movq %rsi, %rax
2302; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2303; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2304; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2305; AVX1-NEXT:    testq %r10, %r10
2306; AVX1-NEXT:    setne %cl
2307; AVX1-NEXT:    testq %rsi, %rsi
2308; AVX1-NEXT:    setne %r13b
2309; AVX1-NEXT:    andb %cl, %r13b
2310; AVX1-NEXT:    mulq %r8
2311; AVX1-NEXT:    movq %rax, %rsi
2312; AVX1-NEXT:    seto %bpl
2313; AVX1-NEXT:    movq %r10, %rax
2314; AVX1-NEXT:    mulq %rdi
2315; AVX1-NEXT:    movq %rax, %rcx
2316; AVX1-NEXT:    seto %bl
2317; AVX1-NEXT:    orb %bpl, %bl
2318; AVX1-NEXT:    addq %rsi, %rcx
2319; AVX1-NEXT:    movq %rdi, %rax
2320; AVX1-NEXT:    mulq %r8
2321; AVX1-NEXT:    movq %rax, %rdi
2322; AVX1-NEXT:    movq %rdx, %rsi
2323; AVX1-NEXT:    addq %rcx, %rsi
2324; AVX1-NEXT:    setb %cl
2325; AVX1-NEXT:    orb %bl, %cl
2326; AVX1-NEXT:    orb %r13b, %cl
2327; AVX1-NEXT:    testq %r9, %r9
2328; AVX1-NEXT:    setne %al
2329; AVX1-NEXT:    testq %r12, %r12
2330; AVX1-NEXT:    setne %r8b
2331; AVX1-NEXT:    andb %al, %r8b
2332; AVX1-NEXT:    movq %r12, %rax
2333; AVX1-NEXT:    mulq %r15
2334; AVX1-NEXT:    movq %rax, %rbp
2335; AVX1-NEXT:    seto %r10b
2336; AVX1-NEXT:    movq %r9, %rax
2337; AVX1-NEXT:    mulq %r11
2338; AVX1-NEXT:    movq %rax, %rbx
2339; AVX1-NEXT:    seto %r9b
2340; AVX1-NEXT:    orb %r10b, %r9b
2341; AVX1-NEXT:    addq %rbp, %rbx
2342; AVX1-NEXT:    movq %r11, %rax
2343; AVX1-NEXT:    mulq %r15
2344; AVX1-NEXT:    addq %rbx, %rdx
2345; AVX1-NEXT:    setb %bl
2346; AVX1-NEXT:    orb %r9b, %bl
2347; AVX1-NEXT:    orb %r8b, %bl
2348; AVX1-NEXT:    movzbl %bl, %ebp
2349; AVX1-NEXT:    negl %ebp
2350; AVX1-NEXT:    movzbl %cl, %ecx
2351; AVX1-NEXT:    negl %ecx
2352; AVX1-NEXT:    vmovd %ecx, %xmm0
2353; AVX1-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
2354; AVX1-NEXT:    movq %rax, 16(%r14)
2355; AVX1-NEXT:    movq %rdi, (%r14)
2356; AVX1-NEXT:    movq %rdx, 24(%r14)
2357; AVX1-NEXT:    movq %rsi, 8(%r14)
2358; AVX1-NEXT:    popq %rbx
2359; AVX1-NEXT:    popq %r12
2360; AVX1-NEXT:    popq %r13
2361; AVX1-NEXT:    popq %r14
2362; AVX1-NEXT:    popq %r15
2363; AVX1-NEXT:    popq %rbp
2364; AVX1-NEXT:    retq
2365;
2366; AVX2-LABEL: umulo_v2i128:
2367; AVX2:       # %bb.0:
2368; AVX2-NEXT:    pushq %rbp
2369; AVX2-NEXT:    pushq %r15
2370; AVX2-NEXT:    pushq %r14
2371; AVX2-NEXT:    pushq %r13
2372; AVX2-NEXT:    pushq %r12
2373; AVX2-NEXT:    pushq %rbx
2374; AVX2-NEXT:    movq %r9, %r10
2375; AVX2-NEXT:    movq %rcx, %r12
2376; AVX2-NEXT:    movq %rdx, %r11
2377; AVX2-NEXT:    movq %rsi, %rax
2378; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2379; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2380; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2381; AVX2-NEXT:    testq %r10, %r10
2382; AVX2-NEXT:    setne %cl
2383; AVX2-NEXT:    testq %rsi, %rsi
2384; AVX2-NEXT:    setne %r13b
2385; AVX2-NEXT:    andb %cl, %r13b
2386; AVX2-NEXT:    mulq %r8
2387; AVX2-NEXT:    movq %rax, %rsi
2388; AVX2-NEXT:    seto %bpl
2389; AVX2-NEXT:    movq %r10, %rax
2390; AVX2-NEXT:    mulq %rdi
2391; AVX2-NEXT:    movq %rax, %rcx
2392; AVX2-NEXT:    seto %bl
2393; AVX2-NEXT:    orb %bpl, %bl
2394; AVX2-NEXT:    addq %rsi, %rcx
2395; AVX2-NEXT:    movq %rdi, %rax
2396; AVX2-NEXT:    mulq %r8
2397; AVX2-NEXT:    movq %rax, %rdi
2398; AVX2-NEXT:    movq %rdx, %rsi
2399; AVX2-NEXT:    addq %rcx, %rsi
2400; AVX2-NEXT:    setb %cl
2401; AVX2-NEXT:    orb %bl, %cl
2402; AVX2-NEXT:    orb %r13b, %cl
2403; AVX2-NEXT:    testq %r9, %r9
2404; AVX2-NEXT:    setne %al
2405; AVX2-NEXT:    testq %r12, %r12
2406; AVX2-NEXT:    setne %r8b
2407; AVX2-NEXT:    andb %al, %r8b
2408; AVX2-NEXT:    movq %r12, %rax
2409; AVX2-NEXT:    mulq %r15
2410; AVX2-NEXT:    movq %rax, %rbp
2411; AVX2-NEXT:    seto %r10b
2412; AVX2-NEXT:    movq %r9, %rax
2413; AVX2-NEXT:    mulq %r11
2414; AVX2-NEXT:    movq %rax, %rbx
2415; AVX2-NEXT:    seto %r9b
2416; AVX2-NEXT:    orb %r10b, %r9b
2417; AVX2-NEXT:    addq %rbp, %rbx
2418; AVX2-NEXT:    movq %r11, %rax
2419; AVX2-NEXT:    mulq %r15
2420; AVX2-NEXT:    addq %rbx, %rdx
2421; AVX2-NEXT:    setb %bl
2422; AVX2-NEXT:    orb %r9b, %bl
2423; AVX2-NEXT:    orb %r8b, %bl
2424; AVX2-NEXT:    movzbl %bl, %ebp
2425; AVX2-NEXT:    negl %ebp
2426; AVX2-NEXT:    movzbl %cl, %ecx
2427; AVX2-NEXT:    negl %ecx
2428; AVX2-NEXT:    vmovd %ecx, %xmm0
2429; AVX2-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
2430; AVX2-NEXT:    movq %rax, 16(%r14)
2431; AVX2-NEXT:    movq %rdi, (%r14)
2432; AVX2-NEXT:    movq %rdx, 24(%r14)
2433; AVX2-NEXT:    movq %rsi, 8(%r14)
2434; AVX2-NEXT:    popq %rbx
2435; AVX2-NEXT:    popq %r12
2436; AVX2-NEXT:    popq %r13
2437; AVX2-NEXT:    popq %r14
2438; AVX2-NEXT:    popq %r15
2439; AVX2-NEXT:    popq %rbp
2440; AVX2-NEXT:    retq
2441;
2442; AVX512-LABEL: umulo_v2i128:
2443; AVX512:       # %bb.0:
2444; AVX512-NEXT:    pushq %rbp
2445; AVX512-NEXT:    pushq %r15
2446; AVX512-NEXT:    pushq %r14
2447; AVX512-NEXT:    pushq %r13
2448; AVX512-NEXT:    pushq %r12
2449; AVX512-NEXT:    pushq %rbx
2450; AVX512-NEXT:    movq %r9, %r10
2451; AVX512-NEXT:    movq %rcx, %r9
2452; AVX512-NEXT:    movq %rdx, %r11
2453; AVX512-NEXT:    movq %rsi, %rax
2454; AVX512-NEXT:    movq %rdi, %rsi
2455; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2456; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2457; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r12
2458; AVX512-NEXT:    testq %r10, %r10
2459; AVX512-NEXT:    setne %dl
2460; AVX512-NEXT:    testq %rax, %rax
2461; AVX512-NEXT:    setne %bl
2462; AVX512-NEXT:    andb %dl, %bl
2463; AVX512-NEXT:    mulq %r8
2464; AVX512-NEXT:    movq %rax, %r13
2465; AVX512-NEXT:    seto %bpl
2466; AVX512-NEXT:    movq %r10, %rax
2467; AVX512-NEXT:    mulq %rdi
2468; AVX512-NEXT:    movq %rax, %rdi
2469; AVX512-NEXT:    seto %cl
2470; AVX512-NEXT:    orb %bpl, %cl
2471; AVX512-NEXT:    addq %r13, %rdi
2472; AVX512-NEXT:    movq %rsi, %rax
2473; AVX512-NEXT:    mulq %r8
2474; AVX512-NEXT:    movq %rax, %r8
2475; AVX512-NEXT:    movq %rdx, %r10
2476; AVX512-NEXT:    addq %rdi, %r10
2477; AVX512-NEXT:    setb %sil
2478; AVX512-NEXT:    orb %cl, %sil
2479; AVX512-NEXT:    orb %bl, %sil
2480; AVX512-NEXT:    testq %r12, %r12
2481; AVX512-NEXT:    setne %al
2482; AVX512-NEXT:    testq %r9, %r9
2483; AVX512-NEXT:    setne %bpl
2484; AVX512-NEXT:    andb %al, %bpl
2485; AVX512-NEXT:    movq %r9, %rax
2486; AVX512-NEXT:    mulq %r15
2487; AVX512-NEXT:    movq %rax, %rdi
2488; AVX512-NEXT:    seto %r9b
2489; AVX512-NEXT:    movq %r12, %rax
2490; AVX512-NEXT:    mulq %r11
2491; AVX512-NEXT:    movq %rax, %rbx
2492; AVX512-NEXT:    seto %cl
2493; AVX512-NEXT:    orb %r9b, %cl
2494; AVX512-NEXT:    addq %rdi, %rbx
2495; AVX512-NEXT:    movq %r11, %rax
2496; AVX512-NEXT:    mulq %r15
2497; AVX512-NEXT:    addq %rbx, %rdx
2498; AVX512-NEXT:    setb %dil
2499; AVX512-NEXT:    orb %cl, %dil
2500; AVX512-NEXT:    orb %bpl, %dil
2501; AVX512-NEXT:    kmovd %edi, %k0
2502; AVX512-NEXT:    kshiftlw $1, %k0, %k0
2503; AVX512-NEXT:    andl $1, %esi
2504; AVX512-NEXT:    kmovw %esi, %k1
2505; AVX512-NEXT:    korw %k0, %k1, %k1
2506; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2507; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2508; AVX512-NEXT:    movq %rax, 16(%r14)
2509; AVX512-NEXT:    movq %r8, (%r14)
2510; AVX512-NEXT:    movq %rdx, 24(%r14)
2511; AVX512-NEXT:    movq %r10, 8(%r14)
2512; AVX512-NEXT:    popq %rbx
2513; AVX512-NEXT:    popq %r12
2514; AVX512-NEXT:    popq %r13
2515; AVX512-NEXT:    popq %r14
2516; AVX512-NEXT:    popq %r15
2517; AVX512-NEXT:    popq %rbp
2518; AVX512-NEXT:    retq
2519  %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
2520  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
2521  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
2522  %res = sext <2 x i1> %obit to <2 x i32>
2523  store <2 x i128> %val, <2 x i128>* %p2
2524  ret <2 x i32> %res
2525}
2526