1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
8
9declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
10declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
11declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
12declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
13declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
14declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
15declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
16
17declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
18declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
19declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
20
21declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
22declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
23declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
24
25define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
26; SSE-LABEL: umulo_v1i32:
27; SSE:       # %bb.0:
28; SSE-NEXT:    movq %rdx, %rcx
29; SSE-NEXT:    movl %edi, %eax
30; SSE-NEXT:    xorl %edi, %edi
31; SSE-NEXT:    mull %esi
32; SSE-NEXT:    seto %dil
33; SSE-NEXT:    negl %edi
34; SSE-NEXT:    movl %eax, (%rcx)
35; SSE-NEXT:    movl %edi, %eax
36; SSE-NEXT:    retq
37;
38; AVX-LABEL: umulo_v1i32:
39; AVX:       # %bb.0:
40; AVX-NEXT:    movq %rdx, %rcx
41; AVX-NEXT:    movl %edi, %eax
42; AVX-NEXT:    xorl %edi, %edi
43; AVX-NEXT:    mull %esi
44; AVX-NEXT:    seto %dil
45; AVX-NEXT:    negl %edi
46; AVX-NEXT:    movl %eax, (%rcx)
47; AVX-NEXT:    movl %edi, %eax
48; AVX-NEXT:    retq
49  %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
50  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
51  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
52  %res = sext <1 x i1> %obit to <1 x i32>
53  store <1 x i32> %val, <1 x i32>* %p2
54  ret <1 x i32> %res
55}
56
57define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
58; SSE2-LABEL: umulo_v2i32:
59; SSE2:       # %bb.0:
60; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
61; SSE2-NEXT:    pmuludq %xmm1, %xmm0
62; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
63; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
64; SSE2-NEXT:    pmuludq %xmm2, %xmm4
65; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
66; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
67; SSE2-NEXT:    pxor %xmm2, %xmm2
68; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
69; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
70; SSE2-NEXT:    pxor %xmm2, %xmm1
71; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
72; SSE2-NEXT:    movq %xmm0, (%rdi)
73; SSE2-NEXT:    movdqa %xmm1, %xmm0
74; SSE2-NEXT:    retq
75;
76; SSSE3-LABEL: umulo_v2i32:
77; SSSE3:       # %bb.0:
78; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
79; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
80; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
81; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
82; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
83; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
84; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
85; SSSE3-NEXT:    pxor %xmm2, %xmm2
86; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
87; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
88; SSSE3-NEXT:    pxor %xmm2, %xmm1
89; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
90; SSSE3-NEXT:    movq %xmm0, (%rdi)
91; SSSE3-NEXT:    movdqa %xmm1, %xmm0
92; SSSE3-NEXT:    retq
93;
94; SSE41-LABEL: umulo_v2i32:
95; SSE41:       # %bb.0:
96; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
97; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
98; SSE41-NEXT:    pmuludq %xmm2, %xmm3
99; SSE41-NEXT:    movdqa %xmm0, %xmm2
100; SSE41-NEXT:    pmuludq %xmm1, %xmm2
101; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
102; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
103; SSE41-NEXT:    pxor %xmm3, %xmm3
104; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
105; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
106; SSE41-NEXT:    pxor %xmm3, %xmm2
107; SSE41-NEXT:    pmulld %xmm1, %xmm0
108; SSE41-NEXT:    movq %xmm0, (%rdi)
109; SSE41-NEXT:    movdqa %xmm2, %xmm0
110; SSE41-NEXT:    retq
111;
112; AVX1-LABEL: umulo_v2i32:
113; AVX1:       # %bb.0:
114; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
115; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
116; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
117; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
118; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
119; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
120; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
121; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
122; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
123; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
124; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
125; AVX1-NEXT:    vmovq %xmm0, (%rdi)
126; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
127; AVX1-NEXT:    retq
128;
129; AVX2-LABEL: umulo_v2i32:
130; AVX2:       # %bb.0:
131; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
132; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
133; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
134; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
135; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
136; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
137; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
138; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
139; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
140; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
141; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
142; AVX2-NEXT:    vmovq %xmm0, (%rdi)
143; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
144; AVX2-NEXT:    retq
145;
146; AVX512-LABEL: umulo_v2i32:
147; AVX512:       # %bb.0:
148; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
149; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
150; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
151; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
152; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
153; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
154; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
155; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
156; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
157; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
158; AVX512-NEXT:    vmovq %xmm1, (%rdi)
159; AVX512-NEXT:    retq
160  %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
161  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
162  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
163  %res = sext <2 x i1> %obit to <2 x i32>
164  store <2 x i32> %val, <2 x i32>* %p2
165  ret <2 x i32> %res
166}
167
168define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
169; SSE2-LABEL: umulo_v3i32:
170; SSE2:       # %bb.0:
171; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
172; SSE2-NEXT:    pmuludq %xmm1, %xmm0
173; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
174; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
175; SSE2-NEXT:    pmuludq %xmm2, %xmm4
176; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
177; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
178; SSE2-NEXT:    pxor %xmm2, %xmm2
179; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
180; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
181; SSE2-NEXT:    pxor %xmm2, %xmm1
182; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
183; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
184; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
185; SSE2-NEXT:    movq %xmm0, (%rdi)
186; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
187; SSE2-NEXT:    movd %xmm0, 8(%rdi)
188; SSE2-NEXT:    movdqa %xmm1, %xmm0
189; SSE2-NEXT:    retq
190;
191; SSSE3-LABEL: umulo_v3i32:
192; SSSE3:       # %bb.0:
193; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
194; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
195; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
196; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
197; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
198; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
199; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
200; SSSE3-NEXT:    pxor %xmm2, %xmm2
201; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
202; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
203; SSSE3-NEXT:    pxor %xmm2, %xmm1
204; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
205; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
206; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
207; SSSE3-NEXT:    movq %xmm0, (%rdi)
208; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
209; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
210; SSSE3-NEXT:    movdqa %xmm1, %xmm0
211; SSSE3-NEXT:    retq
212;
213; SSE41-LABEL: umulo_v3i32:
214; SSE41:       # %bb.0:
215; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
216; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
217; SSE41-NEXT:    pmuludq %xmm2, %xmm3
218; SSE41-NEXT:    movdqa %xmm0, %xmm2
219; SSE41-NEXT:    pmuludq %xmm1, %xmm2
220; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
221; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
222; SSE41-NEXT:    pxor %xmm3, %xmm3
223; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
224; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
225; SSE41-NEXT:    pxor %xmm3, %xmm2
226; SSE41-NEXT:    pmulld %xmm1, %xmm0
227; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
228; SSE41-NEXT:    movq %xmm0, (%rdi)
229; SSE41-NEXT:    movdqa %xmm2, %xmm0
230; SSE41-NEXT:    retq
231;
232; AVX1-LABEL: umulo_v3i32:
233; AVX1:       # %bb.0:
234; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
235; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
236; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
237; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
238; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
239; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
240; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
241; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
242; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
243; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
244; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
245; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
246; AVX1-NEXT:    vmovq %xmm0, (%rdi)
247; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
248; AVX1-NEXT:    retq
249;
250; AVX2-LABEL: umulo_v3i32:
251; AVX2:       # %bb.0:
252; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
253; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
254; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
255; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
256; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
257; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
258; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
259; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
260; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
261; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
262; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
263; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
264; AVX2-NEXT:    vmovq %xmm0, (%rdi)
265; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
266; AVX2-NEXT:    retq
267;
268; AVX512-LABEL: umulo_v3i32:
269; AVX512:       # %bb.0:
270; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
271; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
272; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
273; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
274; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
275; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
276; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
277; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
278; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
279; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
280; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
281; AVX512-NEXT:    vmovq %xmm1, (%rdi)
282; AVX512-NEXT:    retq
283  %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
284  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
285  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
286  %res = sext <3 x i1> %obit to <3 x i32>
287  store <3 x i32> %val, <3 x i32>* %p2
288  ret <3 x i32> %res
289}
290
291define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
292; SSE2-LABEL: umulo_v4i32:
293; SSE2:       # %bb.0:
294; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
295; SSE2-NEXT:    pmuludq %xmm1, %xmm0
296; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
297; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
298; SSE2-NEXT:    pmuludq %xmm2, %xmm4
299; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
300; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
301; SSE2-NEXT:    pxor %xmm2, %xmm2
302; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
303; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
304; SSE2-NEXT:    pxor %xmm2, %xmm1
305; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
306; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
307; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
308; SSE2-NEXT:    movdqa %xmm0, (%rdi)
309; SSE2-NEXT:    movdqa %xmm1, %xmm0
310; SSE2-NEXT:    retq
311;
312; SSSE3-LABEL: umulo_v4i32:
313; SSSE3:       # %bb.0:
314; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
315; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
316; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
317; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
318; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
319; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
320; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
321; SSSE3-NEXT:    pxor %xmm2, %xmm2
322; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
323; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
324; SSSE3-NEXT:    pxor %xmm2, %xmm1
325; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
326; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
327; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
328; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
329; SSSE3-NEXT:    movdqa %xmm1, %xmm0
330; SSSE3-NEXT:    retq
331;
332; SSE41-LABEL: umulo_v4i32:
333; SSE41:       # %bb.0:
334; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
335; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
336; SSE41-NEXT:    pmuludq %xmm2, %xmm3
337; SSE41-NEXT:    movdqa %xmm0, %xmm2
338; SSE41-NEXT:    pmuludq %xmm1, %xmm2
339; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
340; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
341; SSE41-NEXT:    pxor %xmm3, %xmm3
342; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
343; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
344; SSE41-NEXT:    pxor %xmm3, %xmm2
345; SSE41-NEXT:    pmulld %xmm1, %xmm0
346; SSE41-NEXT:    movdqa %xmm0, (%rdi)
347; SSE41-NEXT:    movdqa %xmm2, %xmm0
348; SSE41-NEXT:    retq
349;
350; AVX1-LABEL: umulo_v4i32:
351; AVX1:       # %bb.0:
352; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
353; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
354; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
355; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
356; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
357; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
358; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
359; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
360; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
361; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
362; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
363; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
364; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
365; AVX1-NEXT:    retq
366;
367; AVX2-LABEL: umulo_v4i32:
368; AVX2:       # %bb.0:
369; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
370; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
371; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
372; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
373; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
374; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
375; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
376; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
377; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
378; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
379; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
380; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
381; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
382; AVX2-NEXT:    retq
383;
384; AVX512-LABEL: umulo_v4i32:
385; AVX512:       # %bb.0:
386; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
387; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
388; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
389; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
390; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
391; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
392; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
393; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
394; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
395; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
396; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
397; AVX512-NEXT:    retq
398  %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
399  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
400  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
401  %res = sext <4 x i1> %obit to <4 x i32>
402  store <4 x i32> %val, <4 x i32>* %p2
403  ret <4 x i32> %res
404}
405
406define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
407; SSE2-LABEL: umulo_v6i32:
408; SSE2:       # %bb.0:
409; SSE2-NEXT:    movq %rdi, %rax
410; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
411; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
412; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
413; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
414; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
415; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
416; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
417; SSE2-NEXT:    movd %r8d, %xmm4
418; SSE2-NEXT:    movd %ecx, %xmm0
419; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
420; SSE2-NEXT:    movd %edx, %xmm5
421; SSE2-NEXT:    movd %esi, %xmm6
422; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
423; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
424; SSE2-NEXT:    pmuludq %xmm1, %xmm6
425; SSE2-NEXT:    movd %r9d, %xmm0
426; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
427; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
428; SSE2-NEXT:    pmuludq %xmm0, %xmm1
429; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
430; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
431; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm4[0,0]
432; SSE2-NEXT:    pmuludq %xmm3, %xmm5
433; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,3,2,3]
434; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
435; SSE2-NEXT:    pxor %xmm2, %xmm2
436; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
437; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
438; SSE2-NEXT:    pxor %xmm3, %xmm0
439; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
440; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
441; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
442; SSE2-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
443; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
444; SSE2-NEXT:    pmuludq %xmm5, %xmm6
445; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]
446; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,3,2,3]
447; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
448; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
449; SSE2-NEXT:    pxor %xmm3, %xmm7
450; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
451; SSE2-NEXT:    movq %xmm1, 16(%rcx)
452; SSE2-NEXT:    movdqa %xmm4, (%rcx)
453; SSE2-NEXT:    movq %xmm7, 16(%rdi)
454; SSE2-NEXT:    movdqa %xmm0, (%rdi)
455; SSE2-NEXT:    retq
456;
457; SSSE3-LABEL: umulo_v6i32:
458; SSSE3:       # %bb.0:
459; SSSE3-NEXT:    movq %rdi, %rax
460; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
461; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
462; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
463; SSSE3-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
464; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
465; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
466; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
467; SSSE3-NEXT:    movd %r8d, %xmm4
468; SSSE3-NEXT:    movd %ecx, %xmm0
469; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
470; SSSE3-NEXT:    movd %edx, %xmm5
471; SSSE3-NEXT:    movd %esi, %xmm6
472; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
473; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
474; SSSE3-NEXT:    pmuludq %xmm1, %xmm6
475; SSSE3-NEXT:    movd %r9d, %xmm0
476; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
477; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
478; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
479; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
480; SSSE3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
481; SSSE3-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm4[0,0]
482; SSSE3-NEXT:    pmuludq %xmm3, %xmm5
483; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,3,2,3]
484; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
485; SSSE3-NEXT:    pxor %xmm2, %xmm2
486; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
487; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
488; SSSE3-NEXT:    pxor %xmm3, %xmm0
489; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
490; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
491; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
492; SSSE3-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
493; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
494; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
495; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]
496; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,3,2,3]
497; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
498; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
499; SSSE3-NEXT:    pxor %xmm3, %xmm7
500; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
501; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
502; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
503; SSSE3-NEXT:    movq %xmm7, 16(%rdi)
504; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
505; SSSE3-NEXT:    retq
506;
507; SSE41-LABEL: umulo_v6i32:
508; SSE41:       # %bb.0:
509; SSE41-NEXT:    movq %rdi, %rax
510; SSE41-NEXT:    movd %esi, %xmm2
511; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
512; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
513; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
514; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
515; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
516; SSE41-NEXT:    movdqa %xmm1, %xmm0
517; SSE41-NEXT:    pmuludq %xmm2, %xmm1
518; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
519; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
520; SSE41-NEXT:    movd %r9d, %xmm4
521; SSE41-NEXT:    movdqa %xmm4, %xmm5
522; SSE41-NEXT:    pmuludq %xmm3, %xmm4
523; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
524; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
525; SSE41-NEXT:    pmulld %xmm3, %xmm5
526; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
527; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
528; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
529; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
530; SSE41-NEXT:    pmuludq %xmm3, %xmm6
531; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
532; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
533; SSE41-NEXT:    pxor %xmm8, %xmm8
534; SSE41-NEXT:    pcmpeqd %xmm8, %xmm1
535; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
536; SSE41-NEXT:    pxor %xmm6, %xmm1
537; SSE41-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
538; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
539; SSE41-NEXT:    pmuludq %xmm7, %xmm3
540; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
541; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
542; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
543; SSE41-NEXT:    pxor %xmm6, %xmm4
544; SSE41-NEXT:    pmulld %xmm2, %xmm0
545; SSE41-NEXT:    movq %xmm5, 16(%rcx)
546; SSE41-NEXT:    movdqa %xmm0, (%rcx)
547; SSE41-NEXT:    movq %xmm4, 16(%rdi)
548; SSE41-NEXT:    movdqa %xmm1, (%rdi)
549; SSE41-NEXT:    retq
550;
551; AVX1-LABEL: umulo_v6i32:
552; AVX1:       # %bb.0:
553; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
554; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
555; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
556; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
557; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
558; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
559; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
560; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
561; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
562; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm2
563; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
564; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
565; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
566; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
567; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
568; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
569; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
570; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
571; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
572; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
573; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
574; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
575; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
576; AVX1-NEXT:    vmovq %xmm1, 16(%rdi)
577; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
578; AVX1-NEXT:    vmovaps %ymm2, %ymm0
579; AVX1-NEXT:    retq
580;
581; AVX2-LABEL: umulo_v6i32:
582; AVX2:       # %bb.0:
583; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
584; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
585; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
586; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
587; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
588; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
589; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
590; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
591; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
592; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
593; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
594; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
595; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
596; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
597; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
598; AVX2-NEXT:    retq
599;
600; AVX512-LABEL: umulo_v6i32:
601; AVX512:       # %bb.0:
602; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
603; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
604; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
605; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
606; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
607; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
608; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
609; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
610; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
611; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
612; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
613; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
614; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
615; AVX512-NEXT:    retq
616  %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
617  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
618  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
619  %res = sext <6 x i1> %obit to <6 x i32>
620  store <6 x i32> %val, <6 x i32>* %p2
621  ret <6 x i32> %res
622}
623
624define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
625; SSE2-LABEL: umulo_v8i32:
626; SSE2:       # %bb.0:
627; SSE2-NEXT:    movdqa %xmm0, %xmm4
628; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
629; SSE2-NEXT:    pmuludq %xmm2, %xmm4
630; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
631; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
632; SSE2-NEXT:    pmuludq %xmm5, %xmm6
633; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
634; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
635; SSE2-NEXT:    pxor %xmm8, %xmm8
636; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
637; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
638; SSE2-NEXT:    pxor %xmm7, %xmm0
639; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
640; SSE2-NEXT:    pmuludq %xmm3, %xmm1
641; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
642; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
643; SSE2-NEXT:    pmuludq %xmm5, %xmm3
644; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
645; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
646; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
647; SSE2-NEXT:    pxor %xmm7, %xmm2
648; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
649; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
650; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
651; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
652; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
653; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
654; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
655; SSE2-NEXT:    movdqa %xmm4, (%rdi)
656; SSE2-NEXT:    movdqa %xmm2, %xmm1
657; SSE2-NEXT:    retq
658;
659; SSSE3-LABEL: umulo_v8i32:
660; SSSE3:       # %bb.0:
661; SSSE3-NEXT:    movdqa %xmm0, %xmm4
662; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
663; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
664; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
665; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
666; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
667; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
668; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
669; SSSE3-NEXT:    pxor %xmm8, %xmm8
670; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
671; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
672; SSSE3-NEXT:    pxor %xmm7, %xmm0
673; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
674; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
675; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
676; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
677; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
678; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
679; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
680; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
681; SSSE3-NEXT:    pxor %xmm7, %xmm2
682; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
683; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
684; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
685; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
686; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
687; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
688; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
689; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
690; SSSE3-NEXT:    movdqa %xmm2, %xmm1
691; SSSE3-NEXT:    retq
692;
693; SSE41-LABEL: umulo_v8i32:
694; SSE41:       # %bb.0:
695; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
696; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
697; SSE41-NEXT:    pmuludq %xmm4, %xmm5
698; SSE41-NEXT:    movdqa %xmm0, %xmm4
699; SSE41-NEXT:    pmuludq %xmm2, %xmm4
700; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
701; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
702; SSE41-NEXT:    pxor %xmm8, %xmm8
703; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
704; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
705; SSE41-NEXT:    pxor %xmm7, %xmm4
706; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
707; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
708; SSE41-NEXT:    pmuludq %xmm5, %xmm6
709; SSE41-NEXT:    movdqa %xmm1, %xmm5
710; SSE41-NEXT:    pmuludq %xmm3, %xmm5
711; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
712; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
713; SSE41-NEXT:    pcmpeqd %xmm8, %xmm5
714; SSE41-NEXT:    pxor %xmm7, %xmm5
715; SSE41-NEXT:    pmulld %xmm2, %xmm0
716; SSE41-NEXT:    pmulld %xmm3, %xmm1
717; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
718; SSE41-NEXT:    movdqa %xmm0, (%rdi)
719; SSE41-NEXT:    movdqa %xmm4, %xmm0
720; SSE41-NEXT:    movdqa %xmm5, %xmm1
721; SSE41-NEXT:    retq
722;
723; AVX1-LABEL: umulo_v8i32:
724; AVX1:       # %bb.0:
725; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
726; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
727; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
728; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
729; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
730; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
731; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
732; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
733; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
734; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm2
735; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
736; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
737; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
738; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
739; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
740; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
741; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
742; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
743; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
744; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
745; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
746; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
747; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
748; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)
749; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
750; AVX1-NEXT:    vmovaps %ymm2, %ymm0
751; AVX1-NEXT:    retq
752;
753; AVX2-LABEL: umulo_v8i32:
754; AVX2:       # %bb.0:
755; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
756; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
757; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
758; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
759; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
760; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
761; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
762; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
763; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
764; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
765; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
766; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
767; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
768; AVX2-NEXT:    retq
769;
770; AVX512-LABEL: umulo_v8i32:
771; AVX512:       # %bb.0:
772; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
773; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
774; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
775; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
776; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
777; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
778; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
779; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
780; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
781; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
782; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
783; AVX512-NEXT:    retq
784  %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
785  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
786  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
787  %res = sext <8 x i1> %obit to <8 x i32>
788  store <8 x i32> %val, <8 x i32>* %p2
789  ret <8 x i32> %res
790}
791
792define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
793; SSE2-LABEL: umulo_v16i32:
794; SSE2:       # %bb.0:
795; SSE2-NEXT:    movdqa %xmm0, %xmm8
796; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
797; SSE2-NEXT:    pmuludq %xmm4, %xmm8
798; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
799; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
800; SSE2-NEXT:    pmuludq %xmm10, %xmm9
801; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
802; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
803; SSE2-NEXT:    pxor %xmm10, %xmm10
804; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
805; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
806; SSE2-NEXT:    pxor %xmm11, %xmm0
807; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
808; SSE2-NEXT:    pmuludq %xmm5, %xmm1
809; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
810; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
811; SSE2-NEXT:    pmuludq %xmm13, %xmm12
812; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
813; SSE2-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
814; SSE2-NEXT:    pcmpeqd %xmm10, %xmm15
815; SSE2-NEXT:    pxor %xmm11, %xmm15
816; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
817; SSE2-NEXT:    pmuludq %xmm6, %xmm2
818; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
819; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
820; SSE2-NEXT:    pmuludq %xmm14, %xmm13
821; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
822; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
823; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
824; SSE2-NEXT:    pxor %xmm11, %xmm5
825; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
826; SSE2-NEXT:    pmuludq %xmm7, %xmm3
827; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
828; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
829; SSE2-NEXT:    pmuludq %xmm14, %xmm7
830; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
831; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
832; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
833; SSE2-NEXT:    pxor %xmm11, %xmm6
834; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
835; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
836; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
837; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
838; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
839; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
840; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
841; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
842; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
843; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
844; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
845; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
846; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
847; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
848; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
849; SSE2-NEXT:    movdqa %xmm8, (%rdi)
850; SSE2-NEXT:    movdqa %xmm15, %xmm1
851; SSE2-NEXT:    movdqa %xmm5, %xmm2
852; SSE2-NEXT:    movdqa %xmm6, %xmm3
853; SSE2-NEXT:    retq
854;
855; SSSE3-LABEL: umulo_v16i32:
856; SSSE3:       # %bb.0:
857; SSSE3-NEXT:    movdqa %xmm0, %xmm8
858; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
859; SSSE3-NEXT:    pmuludq %xmm4, %xmm8
860; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
861; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
862; SSSE3-NEXT:    pmuludq %xmm10, %xmm9
863; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
864; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
865; SSSE3-NEXT:    pxor %xmm10, %xmm10
866; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
867; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm11
868; SSSE3-NEXT:    pxor %xmm11, %xmm0
869; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
870; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
871; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
872; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
873; SSSE3-NEXT:    pmuludq %xmm13, %xmm12
874; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
875; SSSE3-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
876; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm15
877; SSSE3-NEXT:    pxor %xmm11, %xmm15
878; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
879; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
880; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
881; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
882; SSSE3-NEXT:    pmuludq %xmm14, %xmm13
883; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
884; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
885; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm5
886; SSSE3-NEXT:    pxor %xmm11, %xmm5
887; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
888; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
889; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
890; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
891; SSSE3-NEXT:    pmuludq %xmm14, %xmm7
892; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
893; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
894; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm6
895; SSSE3-NEXT:    pxor %xmm11, %xmm6
896; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
897; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
898; SSSE3-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
899; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
900; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
901; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
902; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
903; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
904; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
905; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
906; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
907; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
908; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
909; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
910; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
911; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
912; SSSE3-NEXT:    movdqa %xmm15, %xmm1
913; SSSE3-NEXT:    movdqa %xmm5, %xmm2
914; SSSE3-NEXT:    movdqa %xmm6, %xmm3
915; SSSE3-NEXT:    retq
916;
917; SSE41-LABEL: umulo_v16i32:
918; SSE41:       # %bb.0:
919; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
920; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
921; SSE41-NEXT:    pmuludq %xmm8, %xmm9
922; SSE41-NEXT:    movdqa %xmm0, %xmm8
923; SSE41-NEXT:    pmuludq %xmm4, %xmm8
924; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
925; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
926; SSE41-NEXT:    pxor %xmm12, %xmm12
927; SSE41-NEXT:    pcmpeqd %xmm12, %xmm8
928; SSE41-NEXT:    pcmpeqd %xmm13, %xmm13
929; SSE41-NEXT:    pxor %xmm13, %xmm8
930; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
931; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
932; SSE41-NEXT:    pmuludq %xmm9, %xmm10
933; SSE41-NEXT:    movdqa %xmm1, %xmm9
934; SSE41-NEXT:    pmuludq %xmm5, %xmm9
935; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
936; SSE41-NEXT:    pblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3],xmm9[4,5],xmm10[6,7]
937; SSE41-NEXT:    pcmpeqd %xmm12, %xmm9
938; SSE41-NEXT:    pxor %xmm13, %xmm9
939; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
940; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]
941; SSE41-NEXT:    pmuludq %xmm10, %xmm11
942; SSE41-NEXT:    movdqa %xmm2, %xmm10
943; SSE41-NEXT:    pmuludq %xmm6, %xmm10
944; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
945; SSE41-NEXT:    pblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]
946; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
947; SSE41-NEXT:    pxor %xmm13, %xmm10
948; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
949; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
950; SSE41-NEXT:    pmuludq %xmm11, %xmm14
951; SSE41-NEXT:    movdqa %xmm3, %xmm11
952; SSE41-NEXT:    pmuludq %xmm7, %xmm11
953; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
954; SSE41-NEXT:    pblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3],xmm11[4,5],xmm14[6,7]
955; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
956; SSE41-NEXT:    pxor %xmm13, %xmm11
957; SSE41-NEXT:    pmulld %xmm4, %xmm0
958; SSE41-NEXT:    pmulld %xmm5, %xmm1
959; SSE41-NEXT:    pmulld %xmm6, %xmm2
960; SSE41-NEXT:    pmulld %xmm7, %xmm3
961; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
962; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
963; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
964; SSE41-NEXT:    movdqa %xmm0, (%rdi)
965; SSE41-NEXT:    movdqa %xmm8, %xmm0
966; SSE41-NEXT:    movdqa %xmm9, %xmm1
967; SSE41-NEXT:    movdqa %xmm10, %xmm2
968; SSE41-NEXT:    movdqa %xmm11, %xmm3
969; SSE41-NEXT:    retq
970;
971; AVX1-LABEL: umulo_v16i32:
972; AVX1:       # %bb.0:
973; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
974; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
975; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
976; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
977; AVX1-NEXT:    vpmuludq %xmm6, %xmm7, %xmm6
978; AVX1-NEXT:    vpmuludq %xmm10, %xmm12, %xmm7
979; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
980; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
981; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
982; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm7, %xmm7
983; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
984; AVX1-NEXT:    vpxor %xmm9, %xmm7, %xmm7
985; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
986; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
987; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
988; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm6
989; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
990; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5],xmm4[6,7]
991; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm4
992; AVX1-NEXT:    vpxor %xmm9, %xmm4, %xmm4
993; AVX1-NEXT:    vpackssdw %xmm7, %xmm4, %xmm11
994; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
995; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
996; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
997; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
998; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
999; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm7
1000; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1001; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1002; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
1003; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm13
1004; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
1005; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
1006; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
1007; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm7
1008; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
1009; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
1010; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
1011; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm5
1012; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
1013; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm5
1014; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1015; AVX1-NEXT:    vpmulld %xmm6, %xmm4, %xmm4
1016; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1017; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
1018; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm0
1019; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,2,3]
1020; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1021; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1022; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[2,3,0,1]
1023; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1024; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,3,0,1]
1025; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
1026; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
1027; AVX1-NEXT:    vmovdqa %xmm6, 48(%rdi)
1028; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1029; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1030; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1031; AVX1-NEXT:    retq
1032;
1033; AVX2-LABEL: umulo_v16i32:
1034; AVX2:       # %bb.0:
1035; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1036; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1037; AVX2-NEXT:    vpmuludq %ymm4, %ymm5, %ymm4
1038; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm5
1039; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1040; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1041; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1042; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
1043; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
1044; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
1045; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm7
1046; AVX2-NEXT:    vpackssdw %xmm7, %xmm4, %xmm4
1047; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[1,1,3,3,5,5,7,7]
1048; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm0[1,1,3,3,5,5,7,7]
1049; AVX2-NEXT:    vpmuludq %ymm7, %ymm8, %ymm7
1050; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm8
1051; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,1,3,3,5,5,7,7]
1052; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
1053; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
1054; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
1055; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1056; AVX2-NEXT:    vpackssdw %xmm6, %xmm5, %xmm5
1057; AVX2-NEXT:    vpacksswb %xmm4, %xmm5, %xmm4
1058; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1059; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1060; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm0
1061; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,3,0,1]
1062; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1063; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1064; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1065; AVX2-NEXT:    retq
1066;
1067; AVX512-LABEL: umulo_v16i32:
1068; AVX512:       # %bb.0:
1069; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm2
1070; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1071; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1072; AVX512-NEXT:    vpmuludq %zmm3, %zmm4, %zmm3
1073; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1074; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1075; AVX512-NEXT:    vptestmd %zmm4, %zmm4, %k1
1076; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1077; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1078; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1079; AVX512-NEXT:    retq
1080  %t = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1081  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1082  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1083  %res = sext <16 x i1> %obit to <16 x i32>
1084  store <16 x i32> %val, <16 x i32>* %p2
1085  ret <16 x i32> %res
1086}
1087
1088define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1089; SSE2-LABEL: umulo_v16i8:
1090; SSE2:       # %bb.0:
1091; SSE2-NEXT:    movdqa %xmm1, %xmm2
1092; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1093; SSE2-NEXT:    movdqa %xmm0, %xmm3
1094; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1095; SSE2-NEXT:    pmullw %xmm2, %xmm3
1096; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1097; SSE2-NEXT:    pand %xmm2, %xmm3
1098; SSE2-NEXT:    movdqa %xmm1, %xmm5
1099; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
1100; SSE2-NEXT:    movdqa %xmm0, %xmm4
1101; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1102; SSE2-NEXT:    pmullw %xmm5, %xmm4
1103; SSE2-NEXT:    pand %xmm2, %xmm4
1104; SSE2-NEXT:    packuswb %xmm3, %xmm4
1105; SSE2-NEXT:    pxor %xmm2, %xmm2
1106; SSE2-NEXT:    movdqa %xmm1, %xmm3
1107; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1108; SSE2-NEXT:    movdqa %xmm0, %xmm5
1109; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1110; SSE2-NEXT:    pmullw %xmm3, %xmm5
1111; SSE2-NEXT:    psrlw $8, %xmm5
1112; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1113; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1114; SSE2-NEXT:    pmullw %xmm1, %xmm0
1115; SSE2-NEXT:    psrlw $8, %xmm0
1116; SSE2-NEXT:    packuswb %xmm5, %xmm0
1117; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
1118; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1119; SSE2-NEXT:    pxor %xmm0, %xmm3
1120; SSE2-NEXT:    movdqa %xmm3, %xmm1
1121; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1122; SSE2-NEXT:    movdqa %xmm1, %xmm0
1123; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1124; SSE2-NEXT:    pslld $31, %xmm0
1125; SSE2-NEXT:    psrad $31, %xmm0
1126; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1127; SSE2-NEXT:    pslld $31, %xmm1
1128; SSE2-NEXT:    psrad $31, %xmm1
1129; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1130; SSE2-NEXT:    movdqa %xmm3, %xmm2
1131; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1132; SSE2-NEXT:    pslld $31, %xmm2
1133; SSE2-NEXT:    psrad $31, %xmm2
1134; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1135; SSE2-NEXT:    pslld $31, %xmm3
1136; SSE2-NEXT:    psrad $31, %xmm3
1137; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1138; SSE2-NEXT:    retq
1139;
1140; SSSE3-LABEL: umulo_v16i8:
1141; SSSE3:       # %bb.0:
1142; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1143; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1144; SSSE3-NEXT:    movdqa %xmm0, %xmm3
1145; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1146; SSSE3-NEXT:    pmullw %xmm2, %xmm3
1147; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1148; SSSE3-NEXT:    pand %xmm2, %xmm3
1149; SSSE3-NEXT:    movdqa %xmm1, %xmm5
1150; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
1151; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1152; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
1153; SSSE3-NEXT:    pmullw %xmm5, %xmm4
1154; SSSE3-NEXT:    pand %xmm2, %xmm4
1155; SSSE3-NEXT:    packuswb %xmm3, %xmm4
1156; SSSE3-NEXT:    pxor %xmm2, %xmm2
1157; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1158; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1159; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1160; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1161; SSSE3-NEXT:    pmullw %xmm3, %xmm5
1162; SSSE3-NEXT:    psrlw $8, %xmm5
1163; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1164; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1165; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1166; SSSE3-NEXT:    psrlw $8, %xmm0
1167; SSSE3-NEXT:    packuswb %xmm5, %xmm0
1168; SSSE3-NEXT:    pcmpeqb %xmm2, %xmm0
1169; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1170; SSSE3-NEXT:    pxor %xmm0, %xmm3
1171; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1172; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1173; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1174; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1175; SSSE3-NEXT:    pslld $31, %xmm0
1176; SSSE3-NEXT:    psrad $31, %xmm0
1177; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1178; SSSE3-NEXT:    pslld $31, %xmm1
1179; SSSE3-NEXT:    psrad $31, %xmm1
1180; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1181; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1182; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1183; SSSE3-NEXT:    pslld $31, %xmm2
1184; SSSE3-NEXT:    psrad $31, %xmm2
1185; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1186; SSSE3-NEXT:    pslld $31, %xmm3
1187; SSSE3-NEXT:    psrad $31, %xmm3
1188; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1189; SSSE3-NEXT:    retq
1190;
1191; SSE41-LABEL: umulo_v16i8:
1192; SSE41:       # %bb.0:
1193; SSE41-NEXT:    movdqa %xmm1, %xmm2
1194; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1195; SSE41-NEXT:    movdqa %xmm0, %xmm3
1196; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1197; SSE41-NEXT:    pmullw %xmm2, %xmm3
1198; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
1199; SSE41-NEXT:    pand %xmm4, %xmm3
1200; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1201; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1202; SSE41-NEXT:    pmullw %xmm2, %xmm5
1203; SSE41-NEXT:    pand %xmm5, %xmm4
1204; SSE41-NEXT:    packuswb %xmm3, %xmm4
1205; SSE41-NEXT:    pxor %xmm2, %xmm2
1206; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1207; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1208; SSE41-NEXT:    pmullw %xmm1, %xmm0
1209; SSE41-NEXT:    psrlw $8, %xmm0
1210; SSE41-NEXT:    psrlw $8, %xmm5
1211; SSE41-NEXT:    packuswb %xmm0, %xmm5
1212; SSE41-NEXT:    pcmpeqb %xmm2, %xmm5
1213; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1214; SSE41-NEXT:    pxor %xmm5, %xmm3
1215; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1216; SSE41-NEXT:    pslld $31, %xmm0
1217; SSE41-NEXT:    psrad $31, %xmm0
1218; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
1219; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1220; SSE41-NEXT:    pslld $31, %xmm1
1221; SSE41-NEXT:    psrad $31, %xmm1
1222; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
1223; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1224; SSE41-NEXT:    pslld $31, %xmm2
1225; SSE41-NEXT:    psrad $31, %xmm2
1226; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
1227; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1228; SSE41-NEXT:    pslld $31, %xmm3
1229; SSE41-NEXT:    psrad $31, %xmm3
1230; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1231; SSE41-NEXT:    retq
1232;
1233; AVX1-LABEL: umulo_v16i8:
1234; AVX1:       # %bb.0:
1235; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1236; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1237; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1238; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1239; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1240; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1241; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1242; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1243; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1244; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1245; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1246; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1247; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1248; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1249; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1250; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm1
1251; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1252; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1253; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1254; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1255; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1256; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
1257; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1258; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1259; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
1260; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1261; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
1262; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1263; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1264; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1265; AVX1-NEXT:    retq
1266;
1267; AVX2-LABEL: umulo_v16i8:
1268; AVX2:       # %bb.0:
1269; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1270; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1271; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1272; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm1
1273; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
1274; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm2
1275; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1276; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1277; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1278; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1279; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
1280; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1281; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1282; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1283; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1284; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1285; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1286; AVX2-NEXT:    retq
1287;
1288; AVX512-LABEL: umulo_v16i8:
1289; AVX512:       # %bb.0:
1290; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1291; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1292; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1293; AVX512-NEXT:    vpsrlw $8, %ymm1, %ymm0
1294; AVX512-NEXT:    vpmovwb %ymm0, %xmm0
1295; AVX512-NEXT:    vptestmb %xmm0, %xmm0, %k1
1296; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1297; AVX512-NEXT:    vpmovwb %ymm1, (%rdi)
1298; AVX512-NEXT:    retq
1299  %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1300  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1301  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1302  %res = sext <16 x i1> %obit to <16 x i32>
1303  store <16 x i8> %val, <16 x i8>* %p2
1304  ret <16 x i32> %res
1305}
1306
1307define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
1308; SSE2-LABEL: umulo_v8i16:
1309; SSE2:       # %bb.0:
1310; SSE2-NEXT:    movdqa %xmm0, %xmm2
1311; SSE2-NEXT:    pmullw %xmm1, %xmm2
1312; SSE2-NEXT:    pmulhuw %xmm1, %xmm0
1313; SSE2-NEXT:    pxor %xmm3, %xmm3
1314; SSE2-NEXT:    pcmpeqw %xmm0, %xmm3
1315; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
1316; SSE2-NEXT:    pxor %xmm3, %xmm1
1317; SSE2-NEXT:    movdqa %xmm1, %xmm0
1318; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1319; SSE2-NEXT:    pslld $31, %xmm0
1320; SSE2-NEXT:    psrad $31, %xmm0
1321; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1322; SSE2-NEXT:    pslld $31, %xmm1
1323; SSE2-NEXT:    psrad $31, %xmm1
1324; SSE2-NEXT:    movdqa %xmm2, (%rdi)
1325; SSE2-NEXT:    retq
1326;
1327; SSSE3-LABEL: umulo_v8i16:
1328; SSSE3:       # %bb.0:
1329; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1330; SSSE3-NEXT:    pmullw %xmm1, %xmm2
1331; SSSE3-NEXT:    pmulhuw %xmm1, %xmm0
1332; SSSE3-NEXT:    pxor %xmm3, %xmm3
1333; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm3
1334; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
1335; SSSE3-NEXT:    pxor %xmm3, %xmm1
1336; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1337; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1338; SSSE3-NEXT:    pslld $31, %xmm0
1339; SSSE3-NEXT:    psrad $31, %xmm0
1340; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1341; SSSE3-NEXT:    pslld $31, %xmm1
1342; SSSE3-NEXT:    psrad $31, %xmm1
1343; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
1344; SSSE3-NEXT:    retq
1345;
1346; SSE41-LABEL: umulo_v8i16:
1347; SSE41:       # %bb.0:
1348; SSE41-NEXT:    movdqa %xmm0, %xmm2
1349; SSE41-NEXT:    pmullw %xmm1, %xmm2
1350; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
1351; SSE41-NEXT:    pxor %xmm3, %xmm3
1352; SSE41-NEXT:    pcmpeqw %xmm0, %xmm3
1353; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
1354; SSE41-NEXT:    pxor %xmm3, %xmm1
1355; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1356; SSE41-NEXT:    pslld $31, %xmm0
1357; SSE41-NEXT:    psrad $31, %xmm0
1358; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1359; SSE41-NEXT:    pslld $31, %xmm1
1360; SSE41-NEXT:    psrad $31, %xmm1
1361; SSE41-NEXT:    movdqa %xmm2, (%rdi)
1362; SSE41-NEXT:    retq
1363;
1364; AVX1-LABEL: umulo_v8i16:
1365; AVX1:       # %bb.0:
1366; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1367; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1368; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1369; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
1370; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1371; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
1372; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
1373; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1374; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
1375; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1376; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1377; AVX1-NEXT:    retq
1378;
1379; AVX2-LABEL: umulo_v8i16:
1380; AVX2:       # %bb.0:
1381; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1382; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1383; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1384; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
1385; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1386; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
1387; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
1388; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1389; AVX2-NEXT:    retq
1390;
1391; AVX512-LABEL: umulo_v8i16:
1392; AVX512:       # %bb.0:
1393; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
1394; AVX512-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
1395; AVX512-NEXT:    vptestmw %xmm0, %xmm0, %k1
1396; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
1397; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
1398; AVX512-NEXT:    vmovdqa %xmm2, (%rdi)
1399; AVX512-NEXT:    retq
1400  %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
1401  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
1402  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
1403  %res = sext <8 x i1> %obit to <8 x i32>
1404  store <8 x i16> %val, <8 x i16>* %p2
1405  ret <8 x i32> %res
1406}
1407
1408define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
1409; SSE2-LABEL: umulo_v2i64:
1410; SSE2:       # %bb.0:
1411; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1412; SSE2-NEXT:    movq %xmm2, %r8
1413; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1414; SSE2-NEXT:    movq %xmm2, %r10
1415; SSE2-NEXT:    movq %xmm0, %rax
1416; SSE2-NEXT:    movq %xmm1, %rdx
1417; SSE2-NEXT:    xorl %ecx, %ecx
1418; SSE2-NEXT:    mulq %rdx
1419; SSE2-NEXT:    movq $-1, %r9
1420; SSE2-NEXT:    movl $0, %esi
1421; SSE2-NEXT:    cmovoq %r9, %rsi
1422; SSE2-NEXT:    movq %rax, %xmm1
1423; SSE2-NEXT:    movq %r8, %rax
1424; SSE2-NEXT:    mulq %r10
1425; SSE2-NEXT:    movq %rax, %xmm0
1426; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1427; SSE2-NEXT:    movq %rsi, %xmm0
1428; SSE2-NEXT:    cmovoq %r9, %rcx
1429; SSE2-NEXT:    movq %rcx, %xmm2
1430; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1431; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1432; SSE2-NEXT:    movdqa %xmm1, (%rdi)
1433; SSE2-NEXT:    retq
1434;
1435; SSSE3-LABEL: umulo_v2i64:
1436; SSSE3:       # %bb.0:
1437; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1438; SSSE3-NEXT:    movq %xmm2, %r8
1439; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1440; SSSE3-NEXT:    movq %xmm2, %r10
1441; SSSE3-NEXT:    movq %xmm0, %rax
1442; SSSE3-NEXT:    movq %xmm1, %rdx
1443; SSSE3-NEXT:    xorl %ecx, %ecx
1444; SSSE3-NEXT:    mulq %rdx
1445; SSSE3-NEXT:    movq $-1, %r9
1446; SSSE3-NEXT:    movl $0, %esi
1447; SSSE3-NEXT:    cmovoq %r9, %rsi
1448; SSSE3-NEXT:    movq %rax, %xmm1
1449; SSSE3-NEXT:    movq %r8, %rax
1450; SSSE3-NEXT:    mulq %r10
1451; SSSE3-NEXT:    movq %rax, %xmm0
1452; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1453; SSSE3-NEXT:    movq %rsi, %xmm0
1454; SSSE3-NEXT:    cmovoq %r9, %rcx
1455; SSSE3-NEXT:    movq %rcx, %xmm2
1456; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1457; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1458; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
1459; SSSE3-NEXT:    retq
1460;
1461; SSE41-LABEL: umulo_v2i64:
1462; SSE41:       # %bb.0:
1463; SSE41-NEXT:    movq %xmm0, %r10
1464; SSE41-NEXT:    movq %xmm1, %r8
1465; SSE41-NEXT:    pextrq $1, %xmm0, %rax
1466; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
1467; SSE41-NEXT:    xorl %esi, %esi
1468; SSE41-NEXT:    mulq %rdx
1469; SSE41-NEXT:    movq $-1, %r9
1470; SSE41-NEXT:    movl $0, %ecx
1471; SSE41-NEXT:    cmovoq %r9, %rcx
1472; SSE41-NEXT:    movq %rax, %xmm0
1473; SSE41-NEXT:    movq %r10, %rax
1474; SSE41-NEXT:    mulq %r8
1475; SSE41-NEXT:    movq %rax, %xmm1
1476; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1477; SSE41-NEXT:    movq %rcx, %xmm0
1478; SSE41-NEXT:    cmovoq %r9, %rsi
1479; SSE41-NEXT:    movq %rsi, %xmm2
1480; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
1481; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
1482; SSE41-NEXT:    movdqa %xmm1, (%rdi)
1483; SSE41-NEXT:    retq
1484;
1485; AVX1-LABEL: umulo_v2i64:
1486; AVX1:       # %bb.0:
1487; AVX1-NEXT:    vmovq %xmm0, %r10
1488; AVX1-NEXT:    vmovq %xmm1, %r8
1489; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
1490; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
1491; AVX1-NEXT:    xorl %esi, %esi
1492; AVX1-NEXT:    mulq %rdx
1493; AVX1-NEXT:    movq $-1, %r9
1494; AVX1-NEXT:    movl $0, %ecx
1495; AVX1-NEXT:    cmovoq %r9, %rcx
1496; AVX1-NEXT:    vmovq %rax, %xmm0
1497; AVX1-NEXT:    movq %r10, %rax
1498; AVX1-NEXT:    mulq %r8
1499; AVX1-NEXT:    vmovq %rax, %xmm1
1500; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1501; AVX1-NEXT:    vmovq %rcx, %xmm0
1502; AVX1-NEXT:    cmovoq %r9, %rsi
1503; AVX1-NEXT:    vmovq %rsi, %xmm2
1504; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1505; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1506; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
1507; AVX1-NEXT:    retq
1508;
1509; AVX2-LABEL: umulo_v2i64:
1510; AVX2:       # %bb.0:
1511; AVX2-NEXT:    vmovq %xmm0, %r10
1512; AVX2-NEXT:    vmovq %xmm1, %r8
1513; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
1514; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
1515; AVX2-NEXT:    xorl %esi, %esi
1516; AVX2-NEXT:    mulq %rdx
1517; AVX2-NEXT:    movq $-1, %r9
1518; AVX2-NEXT:    movl $0, %ecx
1519; AVX2-NEXT:    cmovoq %r9, %rcx
1520; AVX2-NEXT:    vmovq %rax, %xmm0
1521; AVX2-NEXT:    movq %r10, %rax
1522; AVX2-NEXT:    mulq %r8
1523; AVX2-NEXT:    vmovq %rax, %xmm1
1524; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1525; AVX2-NEXT:    vmovq %rcx, %xmm0
1526; AVX2-NEXT:    cmovoq %r9, %rsi
1527; AVX2-NEXT:    vmovq %rsi, %xmm2
1528; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
1529; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1530; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
1531; AVX2-NEXT:    retq
1532;
1533; AVX512-LABEL: umulo_v2i64:
1534; AVX512:       # %bb.0:
1535; AVX512-NEXT:    vpextrq $1, %xmm0, %rcx
1536; AVX512-NEXT:    vpextrq $1, %xmm1, %r8
1537; AVX512-NEXT:    vmovq %xmm0, %rax
1538; AVX512-NEXT:    vmovq %xmm1, %rdx
1539; AVX512-NEXT:    mulq %rdx
1540; AVX512-NEXT:    movq %rax, %rsi
1541; AVX512-NEXT:    seto %r9b
1542; AVX512-NEXT:    movq %rcx, %rax
1543; AVX512-NEXT:    mulq %r8
1544; AVX512-NEXT:    vmovq %rax, %xmm0
1545; AVX512-NEXT:    vmovq %rsi, %xmm1
1546; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1547; AVX512-NEXT:    seto %al
1548; AVX512-NEXT:    kmovd %eax, %k0
1549; AVX512-NEXT:    kmovd %r9d, %k1
1550; AVX512-NEXT:    kshiftrw $1, %k1, %k2
1551; AVX512-NEXT:    kxorw %k0, %k2, %k0
1552; AVX512-NEXT:    kshiftlw $15, %k0, %k0
1553; AVX512-NEXT:    kshiftrw $14, %k0, %k0
1554; AVX512-NEXT:    kxorw %k0, %k1, %k1
1555; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
1556; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1557; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
1558; AVX512-NEXT:    retq
1559  %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
1560  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
1561  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
1562  %res = sext <2 x i1> %obit to <2 x i32>
1563  store <2 x i64> %val, <2 x i64>* %p2
1564  ret <2 x i32> %res
1565}
1566
1567define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
1568; SSE2-LABEL: umulo_v4i24:
1569; SSE2:       # %bb.0:
1570; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1571; SSE2-NEXT:    pand %xmm2, %xmm1
1572; SSE2-NEXT:    pand %xmm2, %xmm0
1573; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1574; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1575; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1576; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1577; SSE2-NEXT:    pmuludq %xmm4, %xmm2
1578; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
1579; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1580; SSE2-NEXT:    pxor %xmm4, %xmm4
1581; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
1582; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
1583; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
1584; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
1585; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[3,1,2,3]
1586; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
1587; SSE2-NEXT:    psrld $24, %xmm1
1588; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
1589; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
1590; SSE2-NEXT:    pxor %xmm4, %xmm3
1591; SSE2-NEXT:    pxor %xmm4, %xmm1
1592; SSE2-NEXT:    por %xmm3, %xmm1
1593; SSE2-NEXT:    movd %xmm0, %eax
1594; SSE2-NEXT:    movw %ax, (%rdi)
1595; SSE2-NEXT:    movd %xmm2, %ecx
1596; SSE2-NEXT:    movw %cx, 3(%rdi)
1597; SSE2-NEXT:    shrl $16, %eax
1598; SSE2-NEXT:    movb %al, 2(%rdi)
1599; SSE2-NEXT:    shrl $16, %ecx
1600; SSE2-NEXT:    movb %cl, 5(%rdi)
1601; SSE2-NEXT:    movd %xmm5, %eax
1602; SSE2-NEXT:    movw %ax, 9(%rdi)
1603; SSE2-NEXT:    movd %xmm6, %ecx
1604; SSE2-NEXT:    movw %cx, 6(%rdi)
1605; SSE2-NEXT:    shrl $16, %eax
1606; SSE2-NEXT:    movb %al, 11(%rdi)
1607; SSE2-NEXT:    shrl $16, %ecx
1608; SSE2-NEXT:    movb %cl, 8(%rdi)
1609; SSE2-NEXT:    movdqa %xmm1, %xmm0
1610; SSE2-NEXT:    retq
1611;
1612; SSSE3-LABEL: umulo_v4i24:
1613; SSSE3:       # %bb.0:
1614; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1615; SSSE3-NEXT:    pand %xmm2, %xmm1
1616; SSSE3-NEXT:    pand %xmm2, %xmm0
1617; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1618; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
1619; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1620; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1621; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
1622; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
1623; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1624; SSSE3-NEXT:    pxor %xmm4, %xmm4
1625; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
1626; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
1627; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
1628; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
1629; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[3,1,2,3]
1630; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
1631; SSSE3-NEXT:    psrld $24, %xmm1
1632; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
1633; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
1634; SSSE3-NEXT:    pxor %xmm4, %xmm3
1635; SSSE3-NEXT:    pxor %xmm4, %xmm1
1636; SSSE3-NEXT:    por %xmm3, %xmm1
1637; SSSE3-NEXT:    movd %xmm0, %eax
1638; SSSE3-NEXT:    movw %ax, (%rdi)
1639; SSSE3-NEXT:    movd %xmm2, %ecx
1640; SSSE3-NEXT:    movw %cx, 3(%rdi)
1641; SSSE3-NEXT:    shrl $16, %eax
1642; SSSE3-NEXT:    movb %al, 2(%rdi)
1643; SSSE3-NEXT:    shrl $16, %ecx
1644; SSSE3-NEXT:    movb %cl, 5(%rdi)
1645; SSSE3-NEXT:    movd %xmm5, %eax
1646; SSSE3-NEXT:    movw %ax, 9(%rdi)
1647; SSSE3-NEXT:    movd %xmm6, %ecx
1648; SSSE3-NEXT:    movw %cx, 6(%rdi)
1649; SSSE3-NEXT:    shrl $16, %eax
1650; SSSE3-NEXT:    movb %al, 11(%rdi)
1651; SSSE3-NEXT:    shrl $16, %ecx
1652; SSSE3-NEXT:    movb %cl, 8(%rdi)
1653; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1654; SSSE3-NEXT:    retq
1655;
1656; SSE41-LABEL: umulo_v4i24:
1657; SSE41:       # %bb.0:
1658; SSE41-NEXT:    movdqa %xmm0, %xmm2
1659; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
1660; SSE41-NEXT:    pand %xmm0, %xmm2
1661; SSE41-NEXT:    pand %xmm0, %xmm1
1662; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
1663; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
1664; SSE41-NEXT:    pmuludq %xmm0, %xmm3
1665; SSE41-NEXT:    movdqa %xmm2, %xmm0
1666; SSE41-NEXT:    pmuludq %xmm1, %xmm0
1667; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1668; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
1669; SSE41-NEXT:    pxor %xmm0, %xmm0
1670; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
1671; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1672; SSE41-NEXT:    pxor %xmm3, %xmm4
1673; SSE41-NEXT:    pmulld %xmm2, %xmm1
1674; SSE41-NEXT:    pextrd $3, %xmm1, %eax
1675; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
1676; SSE41-NEXT:    pextrd $1, %xmm1, %edx
1677; SSE41-NEXT:    movd %xmm1, %esi
1678; SSE41-NEXT:    psrld $24, %xmm1
1679; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
1680; SSE41-NEXT:    pxor %xmm3, %xmm0
1681; SSE41-NEXT:    por %xmm4, %xmm0
1682; SSE41-NEXT:    movw %ax, 9(%rdi)
1683; SSE41-NEXT:    movw %cx, 6(%rdi)
1684; SSE41-NEXT:    movw %dx, 3(%rdi)
1685; SSE41-NEXT:    movw %si, (%rdi)
1686; SSE41-NEXT:    shrl $16, %eax
1687; SSE41-NEXT:    movb %al, 11(%rdi)
1688; SSE41-NEXT:    shrl $16, %ecx
1689; SSE41-NEXT:    movb %cl, 8(%rdi)
1690; SSE41-NEXT:    shrl $16, %edx
1691; SSE41-NEXT:    movb %dl, 5(%rdi)
1692; SSE41-NEXT:    shrl $16, %esi
1693; SSE41-NEXT:    movb %sil, 2(%rdi)
1694; SSE41-NEXT:    retq
1695;
1696; AVX1-LABEL: umulo_v4i24:
1697; AVX1:       # %bb.0:
1698; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.35098856E-38,2.35098856E-38,2.35098856E-38,2.35098856E-38]
1699; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
1700; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
1701; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[1,1,3,3]
1702; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[1,1,3,3]
1703; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1704; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1705; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1706; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1707; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1708; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1709; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1710; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1711; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1712; AVX1-NEXT:    vpsrld $24, %xmm1, %xmm0
1713; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1714; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1715; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
1716; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
1717; AVX1-NEXT:    movw %ax, 9(%rdi)
1718; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
1719; AVX1-NEXT:    movw %cx, 6(%rdi)
1720; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
1721; AVX1-NEXT:    movw %dx, 3(%rdi)
1722; AVX1-NEXT:    vmovd %xmm1, %esi
1723; AVX1-NEXT:    movw %si, (%rdi)
1724; AVX1-NEXT:    shrl $16, %eax
1725; AVX1-NEXT:    movb %al, 11(%rdi)
1726; AVX1-NEXT:    shrl $16, %ecx
1727; AVX1-NEXT:    movb %cl, 8(%rdi)
1728; AVX1-NEXT:    shrl $16, %edx
1729; AVX1-NEXT:    movb %dl, 5(%rdi)
1730; AVX1-NEXT:    shrl $16, %esi
1731; AVX1-NEXT:    movb %sil, 2(%rdi)
1732; AVX1-NEXT:    retq
1733;
1734; AVX2-LABEL: umulo_v4i24:
1735; AVX2:       # %bb.0:
1736; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
1737; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
1738; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
1739; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1740; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1741; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1742; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1743; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1744; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
1745; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1746; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1747; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1748; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1749; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1750; AVX2-NEXT:    vpsrld $24, %xmm1, %xmm0
1751; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1752; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1753; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
1754; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
1755; AVX2-NEXT:    movw %ax, 9(%rdi)
1756; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
1757; AVX2-NEXT:    movw %cx, 6(%rdi)
1758; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
1759; AVX2-NEXT:    movw %dx, 3(%rdi)
1760; AVX2-NEXT:    vmovd %xmm1, %esi
1761; AVX2-NEXT:    movw %si, (%rdi)
1762; AVX2-NEXT:    shrl $16, %eax
1763; AVX2-NEXT:    movb %al, 11(%rdi)
1764; AVX2-NEXT:    shrl $16, %ecx
1765; AVX2-NEXT:    movb %cl, 8(%rdi)
1766; AVX2-NEXT:    shrl $16, %edx
1767; AVX2-NEXT:    movb %dl, 5(%rdi)
1768; AVX2-NEXT:    shrl $16, %esi
1769; AVX2-NEXT:    movb %sil, 2(%rdi)
1770; AVX2-NEXT:    retq
1771;
1772; AVX512-LABEL: umulo_v4i24:
1773; AVX512:       # %bb.0:
1774; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
1775; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
1776; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
1777; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
1778; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
1779; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
1780; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
1781; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
1782; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
1783; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
1784; AVX512-NEXT:    vpsrld $24, %xmm1, %xmm0
1785; AVX512-NEXT:    vpor %xmm4, %xmm0, %xmm0
1786; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
1787; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
1788; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1789; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
1790; AVX512-NEXT:    movw %ax, 9(%rdi)
1791; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
1792; AVX512-NEXT:    movw %cx, 6(%rdi)
1793; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
1794; AVX512-NEXT:    movw %dx, 3(%rdi)
1795; AVX512-NEXT:    vmovd %xmm1, %esi
1796; AVX512-NEXT:    movw %si, (%rdi)
1797; AVX512-NEXT:    shrl $16, %eax
1798; AVX512-NEXT:    movb %al, 11(%rdi)
1799; AVX512-NEXT:    shrl $16, %ecx
1800; AVX512-NEXT:    movb %cl, 8(%rdi)
1801; AVX512-NEXT:    shrl $16, %edx
1802; AVX512-NEXT:    movb %dl, 5(%rdi)
1803; AVX512-NEXT:    shrl $16, %esi
1804; AVX512-NEXT:    movb %sil, 2(%rdi)
1805; AVX512-NEXT:    retq
1806  %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
1807  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
1808  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
1809  %res = sext <4 x i1> %obit to <4 x i32>
1810  store <4 x i24> %val, <4 x i24>* %p2
1811  ret <4 x i32> %res
1812}
1813
1814define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
1815; SSE2-LABEL: umulo_v4i1:
1816; SSE2:       # %bb.0:
1817; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1818; SSE2-NEXT:    pand %xmm2, %xmm1
1819; SSE2-NEXT:    pand %xmm2, %xmm0
1820; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
1821; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1822; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1823; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1824; SSE2-NEXT:    pmuludq %xmm2, %xmm1
1825; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
1826; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1827; SSE2-NEXT:    pxor %xmm2, %xmm2
1828; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
1829; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
1830; SSE2-NEXT:    pxor %xmm4, %xmm3
1831; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,2,2,3]
1832; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1833; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
1834; SSE2-NEXT:    movdqa %xmm5, %xmm0
1835; SSE2-NEXT:    psrld $1, %xmm0
1836; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
1837; SSE2-NEXT:    pxor %xmm4, %xmm0
1838; SSE2-NEXT:    por %xmm3, %xmm0
1839; SSE2-NEXT:    pslld $31, %xmm5
1840; SSE2-NEXT:    movmskps %xmm5, %eax
1841; SSE2-NEXT:    movb %al, (%rdi)
1842; SSE2-NEXT:    retq
1843;
1844; SSSE3-LABEL: umulo_v4i1:
1845; SSSE3:       # %bb.0:
1846; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1847; SSSE3-NEXT:    pand %xmm2, %xmm1
1848; SSSE3-NEXT:    pand %xmm2, %xmm0
1849; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
1850; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
1851; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
1852; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1853; SSSE3-NEXT:    pmuludq %xmm2, %xmm1
1854; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
1855; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1856; SSSE3-NEXT:    pxor %xmm2, %xmm2
1857; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
1858; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
1859; SSSE3-NEXT:    pxor %xmm4, %xmm3
1860; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,2,2,3]
1861; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1862; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
1863; SSSE3-NEXT:    movdqa %xmm5, %xmm0
1864; SSSE3-NEXT:    psrld $1, %xmm0
1865; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
1866; SSSE3-NEXT:    pxor %xmm4, %xmm0
1867; SSSE3-NEXT:    por %xmm3, %xmm0
1868; SSSE3-NEXT:    pslld $31, %xmm5
1869; SSSE3-NEXT:    movmskps %xmm5, %eax
1870; SSSE3-NEXT:    movb %al, (%rdi)
1871; SSSE3-NEXT:    retq
1872;
1873; SSE41-LABEL: umulo_v4i1:
1874; SSE41:       # %bb.0:
1875; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
1876; SSE41-NEXT:    pand %xmm2, %xmm0
1877; SSE41-NEXT:    pand %xmm2, %xmm1
1878; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1879; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1880; SSE41-NEXT:    pmuludq %xmm2, %xmm3
1881; SSE41-NEXT:    movdqa %xmm0, %xmm2
1882; SSE41-NEXT:    pmuludq %xmm1, %xmm2
1883; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
1884; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
1885; SSE41-NEXT:    pxor %xmm2, %xmm2
1886; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
1887; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1888; SSE41-NEXT:    pxor %xmm3, %xmm4
1889; SSE41-NEXT:    pmaddwd %xmm0, %xmm1
1890; SSE41-NEXT:    movdqa %xmm1, %xmm0
1891; SSE41-NEXT:    psrld $1, %xmm0
1892; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
1893; SSE41-NEXT:    pxor %xmm3, %xmm2
1894; SSE41-NEXT:    por %xmm4, %xmm2
1895; SSE41-NEXT:    pslld $31, %xmm1
1896; SSE41-NEXT:    movmskps %xmm1, %eax
1897; SSE41-NEXT:    movb %al, (%rdi)
1898; SSE41-NEXT:    movdqa %xmm2, %xmm0
1899; SSE41-NEXT:    retq
1900;
1901; AVX1-LABEL: umulo_v4i1:
1902; AVX1:       # %bb.0:
1903; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1]
1904; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
1905; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
1906; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1907; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1908; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1909; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1910; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1911; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1912; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1913; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1914; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1915; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1916; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm1
1917; AVX1-NEXT:    vpsrld $1, %xmm1, %xmm0
1918; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1919; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1920; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
1921; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
1922; AVX1-NEXT:    vmovmskps %xmm1, %eax
1923; AVX1-NEXT:    movb %al, (%rdi)
1924; AVX1-NEXT:    retq
1925;
1926; AVX2-LABEL: umulo_v4i1:
1927; AVX2:       # %bb.0:
1928; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
1929; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
1930; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
1931; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
1932; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
1933; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
1934; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
1935; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
1936; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
1937; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1938; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
1939; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1940; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
1941; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm1
1942; AVX2-NEXT:    vpsrld $1, %xmm1, %xmm0
1943; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
1944; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
1945; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
1946; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
1947; AVX2-NEXT:    vmovmskps %xmm1, %eax
1948; AVX2-NEXT:    movb %al, (%rdi)
1949; AVX2-NEXT:    retq
1950;
1951; AVX512-LABEL: umulo_v4i1:
1952; AVX512:       # %bb.0:
1953; AVX512-NEXT:    pushq %rbx
1954; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
1955; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
1956; AVX512-NEXT:    kshiftrw $3, %k0, %k1
1957; AVX512-NEXT:    kmovd %k1, %r9d
1958; AVX512-NEXT:    andb $1, %r9b
1959; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
1960; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
1961; AVX512-NEXT:    kshiftrw $3, %k1, %k2
1962; AVX512-NEXT:    kmovd %k2, %r10d
1963; AVX512-NEXT:    andb $1, %r10b
1964; AVX512-NEXT:    kshiftrw $2, %k0, %k2
1965; AVX512-NEXT:    kmovd %k2, %r11d
1966; AVX512-NEXT:    andb $1, %r11b
1967; AVX512-NEXT:    kshiftrw $2, %k1, %k2
1968; AVX512-NEXT:    kmovd %k2, %ebx
1969; AVX512-NEXT:    andb $1, %bl
1970; AVX512-NEXT:    kshiftrw $1, %k0, %k2
1971; AVX512-NEXT:    kmovd %k2, %edx
1972; AVX512-NEXT:    andb $1, %dl
1973; AVX512-NEXT:    kshiftrw $1, %k1, %k2
1974; AVX512-NEXT:    kmovd %k2, %esi
1975; AVX512-NEXT:    andb $1, %sil
1976; AVX512-NEXT:    kmovd %k0, %eax
1977; AVX512-NEXT:    andb $1, %al
1978; AVX512-NEXT:    kmovd %k1, %ecx
1979; AVX512-NEXT:    andb $1, %cl
1980; AVX512-NEXT:    # kill: def $al killed $al killed $eax
1981; AVX512-NEXT:    mulb %cl
1982; AVX512-NEXT:    movl %eax, %r8d
1983; AVX512-NEXT:    seto %al
1984; AVX512-NEXT:    testb $-2, %r8b
1985; AVX512-NEXT:    setne %cl
1986; AVX512-NEXT:    orb %al, %cl
1987; AVX512-NEXT:    setne %al
1988; AVX512-NEXT:    kmovd %eax, %k0
1989; AVX512-NEXT:    kshiftrw $1, %k0, %k1
1990; AVX512-NEXT:    movl %edx, %eax
1991; AVX512-NEXT:    mulb %sil
1992; AVX512-NEXT:    movl %eax, %edx
1993; AVX512-NEXT:    seto %al
1994; AVX512-NEXT:    testb $-2, %dl
1995; AVX512-NEXT:    setne %cl
1996; AVX512-NEXT:    orb %al, %cl
1997; AVX512-NEXT:    setne %al
1998; AVX512-NEXT:    kmovd %eax, %k2
1999; AVX512-NEXT:    kxorw %k2, %k1, %k1
2000; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2001; AVX512-NEXT:    kshiftrw $14, %k1, %k1
2002; AVX512-NEXT:    kxorw %k1, %k0, %k0
2003; AVX512-NEXT:    kshiftrw $2, %k0, %k1
2004; AVX512-NEXT:    movl %r11d, %eax
2005; AVX512-NEXT:    mulb %bl
2006; AVX512-NEXT:    movl %eax, %esi
2007; AVX512-NEXT:    seto %al
2008; AVX512-NEXT:    testb $-2, %sil
2009; AVX512-NEXT:    setne %cl
2010; AVX512-NEXT:    orb %al, %cl
2011; AVX512-NEXT:    setne %al
2012; AVX512-NEXT:    kmovd %eax, %k2
2013; AVX512-NEXT:    kxorw %k2, %k1, %k1
2014; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2015; AVX512-NEXT:    kshiftrw $13, %k1, %k1
2016; AVX512-NEXT:    kxorw %k1, %k0, %k0
2017; AVX512-NEXT:    kshiftlw $13, %k0, %k0
2018; AVX512-NEXT:    kshiftrw $13, %k0, %k0
2019; AVX512-NEXT:    movl %r9d, %eax
2020; AVX512-NEXT:    mulb %r10b
2021; AVX512-NEXT:    # kill: def $al killed $al def $eax
2022; AVX512-NEXT:    seto %cl
2023; AVX512-NEXT:    testb $-2, %al
2024; AVX512-NEXT:    setne %bl
2025; AVX512-NEXT:    orb %cl, %bl
2026; AVX512-NEXT:    setne %cl
2027; AVX512-NEXT:    kmovd %ecx, %k1
2028; AVX512-NEXT:    kshiftlw $3, %k1, %k1
2029; AVX512-NEXT:    korw %k1, %k0, %k1
2030; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2031; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2032; AVX512-NEXT:    kmovd %r8d, %k0
2033; AVX512-NEXT:    kshiftrw $1, %k0, %k1
2034; AVX512-NEXT:    kmovd %edx, %k2
2035; AVX512-NEXT:    kxorw %k2, %k1, %k1
2036; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2037; AVX512-NEXT:    kshiftrw $14, %k1, %k1
2038; AVX512-NEXT:    kxorw %k1, %k0, %k0
2039; AVX512-NEXT:    kshiftrw $2, %k0, %k1
2040; AVX512-NEXT:    kmovd %esi, %k2
2041; AVX512-NEXT:    kxorw %k2, %k1, %k1
2042; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2043; AVX512-NEXT:    kshiftrw $13, %k1, %k1
2044; AVX512-NEXT:    kxorw %k1, %k0, %k0
2045; AVX512-NEXT:    kshiftrw $3, %k0, %k1
2046; AVX512-NEXT:    kmovd %eax, %k2
2047; AVX512-NEXT:    kxorw %k2, %k1, %k1
2048; AVX512-NEXT:    kshiftlw $15, %k1, %k1
2049; AVX512-NEXT:    kshiftrw $12, %k1, %k1
2050; AVX512-NEXT:    kxorw %k1, %k0, %k0
2051; AVX512-NEXT:    kmovd %k0, %eax
2052; AVX512-NEXT:    movb %al, (%rdi)
2053; AVX512-NEXT:    popq %rbx
2054; AVX512-NEXT:    retq
2055  %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
2056  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
2057  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
2058  %res = sext <4 x i1> %obit to <4 x i32>
2059  store <4 x i1> %val, <4 x i1>* %p2
2060  ret <4 x i32> %res
2061}
2062
2063define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
2064; SSE2-LABEL: umulo_v2i128:
2065; SSE2:       # %bb.0:
2066; SSE2-NEXT:    pushq %rbp
2067; SSE2-NEXT:    pushq %r15
2068; SSE2-NEXT:    pushq %r14
2069; SSE2-NEXT:    pushq %r13
2070; SSE2-NEXT:    pushq %r12
2071; SSE2-NEXT:    pushq %rbx
2072; SSE2-NEXT:    movq %r9, %r10
2073; SSE2-NEXT:    movq %rcx, %r12
2074; SSE2-NEXT:    movq %rdx, %r11
2075; SSE2-NEXT:    movq %rsi, %rax
2076; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2077; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2078; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2079; SSE2-NEXT:    testq %r10, %r10
2080; SSE2-NEXT:    setne %cl
2081; SSE2-NEXT:    testq %rsi, %rsi
2082; SSE2-NEXT:    setne %r13b
2083; SSE2-NEXT:    andb %cl, %r13b
2084; SSE2-NEXT:    mulq %r8
2085; SSE2-NEXT:    movq %rax, %rsi
2086; SSE2-NEXT:    seto %bpl
2087; SSE2-NEXT:    movq %r10, %rax
2088; SSE2-NEXT:    mulq %rdi
2089; SSE2-NEXT:    movq %rax, %rcx
2090; SSE2-NEXT:    seto %bl
2091; SSE2-NEXT:    orb %bpl, %bl
2092; SSE2-NEXT:    addq %rsi, %rcx
2093; SSE2-NEXT:    movq %rdi, %rax
2094; SSE2-NEXT:    mulq %r8
2095; SSE2-NEXT:    movq %rax, %rdi
2096; SSE2-NEXT:    movq %rdx, %rsi
2097; SSE2-NEXT:    addq %rcx, %rsi
2098; SSE2-NEXT:    setb %cl
2099; SSE2-NEXT:    orb %bl, %cl
2100; SSE2-NEXT:    orb %r13b, %cl
2101; SSE2-NEXT:    testq %r9, %r9
2102; SSE2-NEXT:    setne %al
2103; SSE2-NEXT:    testq %r12, %r12
2104; SSE2-NEXT:    setne %r8b
2105; SSE2-NEXT:    andb %al, %r8b
2106; SSE2-NEXT:    movq %r12, %rax
2107; SSE2-NEXT:    mulq %r15
2108; SSE2-NEXT:    movq %rax, %rbp
2109; SSE2-NEXT:    seto %r10b
2110; SSE2-NEXT:    movq %r9, %rax
2111; SSE2-NEXT:    mulq %r11
2112; SSE2-NEXT:    movq %rax, %rbx
2113; SSE2-NEXT:    seto %r9b
2114; SSE2-NEXT:    orb %r10b, %r9b
2115; SSE2-NEXT:    addq %rbp, %rbx
2116; SSE2-NEXT:    movq %r11, %rax
2117; SSE2-NEXT:    mulq %r15
2118; SSE2-NEXT:    addq %rbx, %rdx
2119; SSE2-NEXT:    setb %bl
2120; SSE2-NEXT:    orb %r9b, %bl
2121; SSE2-NEXT:    orb %r8b, %bl
2122; SSE2-NEXT:    movzbl %bl, %ebp
2123; SSE2-NEXT:    negl %ebp
2124; SSE2-NEXT:    movd %ebp, %xmm1
2125; SSE2-NEXT:    movzbl %cl, %ecx
2126; SSE2-NEXT:    negl %ecx
2127; SSE2-NEXT:    movd %ecx, %xmm0
2128; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2129; SSE2-NEXT:    movq %rax, 16(%r14)
2130; SSE2-NEXT:    movq %rdi, (%r14)
2131; SSE2-NEXT:    movq %rdx, 24(%r14)
2132; SSE2-NEXT:    movq %rsi, 8(%r14)
2133; SSE2-NEXT:    popq %rbx
2134; SSE2-NEXT:    popq %r12
2135; SSE2-NEXT:    popq %r13
2136; SSE2-NEXT:    popq %r14
2137; SSE2-NEXT:    popq %r15
2138; SSE2-NEXT:    popq %rbp
2139; SSE2-NEXT:    retq
2140;
2141; SSSE3-LABEL: umulo_v2i128:
2142; SSSE3:       # %bb.0:
2143; SSSE3-NEXT:    pushq %rbp
2144; SSSE3-NEXT:    pushq %r15
2145; SSSE3-NEXT:    pushq %r14
2146; SSSE3-NEXT:    pushq %r13
2147; SSSE3-NEXT:    pushq %r12
2148; SSSE3-NEXT:    pushq %rbx
2149; SSSE3-NEXT:    movq %r9, %r10
2150; SSSE3-NEXT:    movq %rcx, %r12
2151; SSSE3-NEXT:    movq %rdx, %r11
2152; SSSE3-NEXT:    movq %rsi, %rax
2153; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2154; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2155; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2156; SSSE3-NEXT:    testq %r10, %r10
2157; SSSE3-NEXT:    setne %cl
2158; SSSE3-NEXT:    testq %rsi, %rsi
2159; SSSE3-NEXT:    setne %r13b
2160; SSSE3-NEXT:    andb %cl, %r13b
2161; SSSE3-NEXT:    mulq %r8
2162; SSSE3-NEXT:    movq %rax, %rsi
2163; SSSE3-NEXT:    seto %bpl
2164; SSSE3-NEXT:    movq %r10, %rax
2165; SSSE3-NEXT:    mulq %rdi
2166; SSSE3-NEXT:    movq %rax, %rcx
2167; SSSE3-NEXT:    seto %bl
2168; SSSE3-NEXT:    orb %bpl, %bl
2169; SSSE3-NEXT:    addq %rsi, %rcx
2170; SSSE3-NEXT:    movq %rdi, %rax
2171; SSSE3-NEXT:    mulq %r8
2172; SSSE3-NEXT:    movq %rax, %rdi
2173; SSSE3-NEXT:    movq %rdx, %rsi
2174; SSSE3-NEXT:    addq %rcx, %rsi
2175; SSSE3-NEXT:    setb %cl
2176; SSSE3-NEXT:    orb %bl, %cl
2177; SSSE3-NEXT:    orb %r13b, %cl
2178; SSSE3-NEXT:    testq %r9, %r9
2179; SSSE3-NEXT:    setne %al
2180; SSSE3-NEXT:    testq %r12, %r12
2181; SSSE3-NEXT:    setne %r8b
2182; SSSE3-NEXT:    andb %al, %r8b
2183; SSSE3-NEXT:    movq %r12, %rax
2184; SSSE3-NEXT:    mulq %r15
2185; SSSE3-NEXT:    movq %rax, %rbp
2186; SSSE3-NEXT:    seto %r10b
2187; SSSE3-NEXT:    movq %r9, %rax
2188; SSSE3-NEXT:    mulq %r11
2189; SSSE3-NEXT:    movq %rax, %rbx
2190; SSSE3-NEXT:    seto %r9b
2191; SSSE3-NEXT:    orb %r10b, %r9b
2192; SSSE3-NEXT:    addq %rbp, %rbx
2193; SSSE3-NEXT:    movq %r11, %rax
2194; SSSE3-NEXT:    mulq %r15
2195; SSSE3-NEXT:    addq %rbx, %rdx
2196; SSSE3-NEXT:    setb %bl
2197; SSSE3-NEXT:    orb %r9b, %bl
2198; SSSE3-NEXT:    orb %r8b, %bl
2199; SSSE3-NEXT:    movzbl %bl, %ebp
2200; SSSE3-NEXT:    negl %ebp
2201; SSSE3-NEXT:    movd %ebp, %xmm1
2202; SSSE3-NEXT:    movzbl %cl, %ecx
2203; SSSE3-NEXT:    negl %ecx
2204; SSSE3-NEXT:    movd %ecx, %xmm0
2205; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2206; SSSE3-NEXT:    movq %rax, 16(%r14)
2207; SSSE3-NEXT:    movq %rdi, (%r14)
2208; SSSE3-NEXT:    movq %rdx, 24(%r14)
2209; SSSE3-NEXT:    movq %rsi, 8(%r14)
2210; SSSE3-NEXT:    popq %rbx
2211; SSSE3-NEXT:    popq %r12
2212; SSSE3-NEXT:    popq %r13
2213; SSSE3-NEXT:    popq %r14
2214; SSSE3-NEXT:    popq %r15
2215; SSSE3-NEXT:    popq %rbp
2216; SSSE3-NEXT:    retq
2217;
2218; SSE41-LABEL: umulo_v2i128:
2219; SSE41:       # %bb.0:
2220; SSE41-NEXT:    pushq %rbp
2221; SSE41-NEXT:    pushq %r15
2222; SSE41-NEXT:    pushq %r14
2223; SSE41-NEXT:    pushq %r13
2224; SSE41-NEXT:    pushq %r12
2225; SSE41-NEXT:    pushq %rbx
2226; SSE41-NEXT:    movq %r9, %r10
2227; SSE41-NEXT:    movq %rcx, %r12
2228; SSE41-NEXT:    movq %rdx, %r11
2229; SSE41-NEXT:    movq %rsi, %rax
2230; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2231; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2232; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2233; SSE41-NEXT:    testq %r10, %r10
2234; SSE41-NEXT:    setne %cl
2235; SSE41-NEXT:    testq %rsi, %rsi
2236; SSE41-NEXT:    setne %r13b
2237; SSE41-NEXT:    andb %cl, %r13b
2238; SSE41-NEXT:    mulq %r8
2239; SSE41-NEXT:    movq %rax, %rsi
2240; SSE41-NEXT:    seto %bpl
2241; SSE41-NEXT:    movq %r10, %rax
2242; SSE41-NEXT:    mulq %rdi
2243; SSE41-NEXT:    movq %rax, %rcx
2244; SSE41-NEXT:    seto %bl
2245; SSE41-NEXT:    orb %bpl, %bl
2246; SSE41-NEXT:    addq %rsi, %rcx
2247; SSE41-NEXT:    movq %rdi, %rax
2248; SSE41-NEXT:    mulq %r8
2249; SSE41-NEXT:    movq %rax, %rdi
2250; SSE41-NEXT:    movq %rdx, %rsi
2251; SSE41-NEXT:    addq %rcx, %rsi
2252; SSE41-NEXT:    setb %cl
2253; SSE41-NEXT:    orb %bl, %cl
2254; SSE41-NEXT:    orb %r13b, %cl
2255; SSE41-NEXT:    testq %r9, %r9
2256; SSE41-NEXT:    setne %al
2257; SSE41-NEXT:    testq %r12, %r12
2258; SSE41-NEXT:    setne %r8b
2259; SSE41-NEXT:    andb %al, %r8b
2260; SSE41-NEXT:    movq %r12, %rax
2261; SSE41-NEXT:    mulq %r15
2262; SSE41-NEXT:    movq %rax, %rbp
2263; SSE41-NEXT:    seto %r10b
2264; SSE41-NEXT:    movq %r9, %rax
2265; SSE41-NEXT:    mulq %r11
2266; SSE41-NEXT:    movq %rax, %rbx
2267; SSE41-NEXT:    seto %r9b
2268; SSE41-NEXT:    orb %r10b, %r9b
2269; SSE41-NEXT:    addq %rbp, %rbx
2270; SSE41-NEXT:    movq %r11, %rax
2271; SSE41-NEXT:    mulq %r15
2272; SSE41-NEXT:    addq %rbx, %rdx
2273; SSE41-NEXT:    setb %bl
2274; SSE41-NEXT:    orb %r9b, %bl
2275; SSE41-NEXT:    orb %r8b, %bl
2276; SSE41-NEXT:    movzbl %bl, %ebp
2277; SSE41-NEXT:    negl %ebp
2278; SSE41-NEXT:    movzbl %cl, %ecx
2279; SSE41-NEXT:    negl %ecx
2280; SSE41-NEXT:    movd %ecx, %xmm0
2281; SSE41-NEXT:    pinsrd $1, %ebp, %xmm0
2282; SSE41-NEXT:    movq %rax, 16(%r14)
2283; SSE41-NEXT:    movq %rdi, (%r14)
2284; SSE41-NEXT:    movq %rdx, 24(%r14)
2285; SSE41-NEXT:    movq %rsi, 8(%r14)
2286; SSE41-NEXT:    popq %rbx
2287; SSE41-NEXT:    popq %r12
2288; SSE41-NEXT:    popq %r13
2289; SSE41-NEXT:    popq %r14
2290; SSE41-NEXT:    popq %r15
2291; SSE41-NEXT:    popq %rbp
2292; SSE41-NEXT:    retq
2293;
2294; AVX1-LABEL: umulo_v2i128:
2295; AVX1:       # %bb.0:
2296; AVX1-NEXT:    pushq %rbp
2297; AVX1-NEXT:    pushq %r15
2298; AVX1-NEXT:    pushq %r14
2299; AVX1-NEXT:    pushq %r13
2300; AVX1-NEXT:    pushq %r12
2301; AVX1-NEXT:    pushq %rbx
2302; AVX1-NEXT:    movq %r9, %r10
2303; AVX1-NEXT:    movq %rcx, %r12
2304; AVX1-NEXT:    movq %rdx, %r11
2305; AVX1-NEXT:    movq %rsi, %rax
2306; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2307; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2308; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2309; AVX1-NEXT:    testq %r10, %r10
2310; AVX1-NEXT:    setne %cl
2311; AVX1-NEXT:    testq %rsi, %rsi
2312; AVX1-NEXT:    setne %r13b
2313; AVX1-NEXT:    andb %cl, %r13b
2314; AVX1-NEXT:    mulq %r8
2315; AVX1-NEXT:    movq %rax, %rsi
2316; AVX1-NEXT:    seto %bpl
2317; AVX1-NEXT:    movq %r10, %rax
2318; AVX1-NEXT:    mulq %rdi
2319; AVX1-NEXT:    movq %rax, %rcx
2320; AVX1-NEXT:    seto %bl
2321; AVX1-NEXT:    orb %bpl, %bl
2322; AVX1-NEXT:    addq %rsi, %rcx
2323; AVX1-NEXT:    movq %rdi, %rax
2324; AVX1-NEXT:    mulq %r8
2325; AVX1-NEXT:    movq %rax, %rdi
2326; AVX1-NEXT:    movq %rdx, %rsi
2327; AVX1-NEXT:    addq %rcx, %rsi
2328; AVX1-NEXT:    setb %cl
2329; AVX1-NEXT:    orb %bl, %cl
2330; AVX1-NEXT:    orb %r13b, %cl
2331; AVX1-NEXT:    testq %r9, %r9
2332; AVX1-NEXT:    setne %al
2333; AVX1-NEXT:    testq %r12, %r12
2334; AVX1-NEXT:    setne %r8b
2335; AVX1-NEXT:    andb %al, %r8b
2336; AVX1-NEXT:    movq %r12, %rax
2337; AVX1-NEXT:    mulq %r15
2338; AVX1-NEXT:    movq %rax, %rbp
2339; AVX1-NEXT:    seto %r10b
2340; AVX1-NEXT:    movq %r9, %rax
2341; AVX1-NEXT:    mulq %r11
2342; AVX1-NEXT:    movq %rax, %rbx
2343; AVX1-NEXT:    seto %r9b
2344; AVX1-NEXT:    orb %r10b, %r9b
2345; AVX1-NEXT:    addq %rbp, %rbx
2346; AVX1-NEXT:    movq %r11, %rax
2347; AVX1-NEXT:    mulq %r15
2348; AVX1-NEXT:    addq %rbx, %rdx
2349; AVX1-NEXT:    setb %bl
2350; AVX1-NEXT:    orb %r9b, %bl
2351; AVX1-NEXT:    orb %r8b, %bl
2352; AVX1-NEXT:    movzbl %bl, %ebp
2353; AVX1-NEXT:    negl %ebp
2354; AVX1-NEXT:    movzbl %cl, %ecx
2355; AVX1-NEXT:    negl %ecx
2356; AVX1-NEXT:    vmovd %ecx, %xmm0
2357; AVX1-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
2358; AVX1-NEXT:    movq %rax, 16(%r14)
2359; AVX1-NEXT:    movq %rdi, (%r14)
2360; AVX1-NEXT:    movq %rdx, 24(%r14)
2361; AVX1-NEXT:    movq %rsi, 8(%r14)
2362; AVX1-NEXT:    popq %rbx
2363; AVX1-NEXT:    popq %r12
2364; AVX1-NEXT:    popq %r13
2365; AVX1-NEXT:    popq %r14
2366; AVX1-NEXT:    popq %r15
2367; AVX1-NEXT:    popq %rbp
2368; AVX1-NEXT:    retq
2369;
2370; AVX2-LABEL: umulo_v2i128:
2371; AVX2:       # %bb.0:
2372; AVX2-NEXT:    pushq %rbp
2373; AVX2-NEXT:    pushq %r15
2374; AVX2-NEXT:    pushq %r14
2375; AVX2-NEXT:    pushq %r13
2376; AVX2-NEXT:    pushq %r12
2377; AVX2-NEXT:    pushq %rbx
2378; AVX2-NEXT:    movq %r9, %r10
2379; AVX2-NEXT:    movq %rcx, %r12
2380; AVX2-NEXT:    movq %rdx, %r11
2381; AVX2-NEXT:    movq %rsi, %rax
2382; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2383; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2384; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
2385; AVX2-NEXT:    testq %r10, %r10
2386; AVX2-NEXT:    setne %cl
2387; AVX2-NEXT:    testq %rsi, %rsi
2388; AVX2-NEXT:    setne %r13b
2389; AVX2-NEXT:    andb %cl, %r13b
2390; AVX2-NEXT:    mulq %r8
2391; AVX2-NEXT:    movq %rax, %rsi
2392; AVX2-NEXT:    seto %bpl
2393; AVX2-NEXT:    movq %r10, %rax
2394; AVX2-NEXT:    mulq %rdi
2395; AVX2-NEXT:    movq %rax, %rcx
2396; AVX2-NEXT:    seto %bl
2397; AVX2-NEXT:    orb %bpl, %bl
2398; AVX2-NEXT:    addq %rsi, %rcx
2399; AVX2-NEXT:    movq %rdi, %rax
2400; AVX2-NEXT:    mulq %r8
2401; AVX2-NEXT:    movq %rax, %rdi
2402; AVX2-NEXT:    movq %rdx, %rsi
2403; AVX2-NEXT:    addq %rcx, %rsi
2404; AVX2-NEXT:    setb %cl
2405; AVX2-NEXT:    orb %bl, %cl
2406; AVX2-NEXT:    orb %r13b, %cl
2407; AVX2-NEXT:    testq %r9, %r9
2408; AVX2-NEXT:    setne %al
2409; AVX2-NEXT:    testq %r12, %r12
2410; AVX2-NEXT:    setne %r8b
2411; AVX2-NEXT:    andb %al, %r8b
2412; AVX2-NEXT:    movq %r12, %rax
2413; AVX2-NEXT:    mulq %r15
2414; AVX2-NEXT:    movq %rax, %rbp
2415; AVX2-NEXT:    seto %r10b
2416; AVX2-NEXT:    movq %r9, %rax
2417; AVX2-NEXT:    mulq %r11
2418; AVX2-NEXT:    movq %rax, %rbx
2419; AVX2-NEXT:    seto %r9b
2420; AVX2-NEXT:    orb %r10b, %r9b
2421; AVX2-NEXT:    addq %rbp, %rbx
2422; AVX2-NEXT:    movq %r11, %rax
2423; AVX2-NEXT:    mulq %r15
2424; AVX2-NEXT:    addq %rbx, %rdx
2425; AVX2-NEXT:    setb %bl
2426; AVX2-NEXT:    orb %r9b, %bl
2427; AVX2-NEXT:    orb %r8b, %bl
2428; AVX2-NEXT:    movzbl %bl, %ebp
2429; AVX2-NEXT:    negl %ebp
2430; AVX2-NEXT:    movzbl %cl, %ecx
2431; AVX2-NEXT:    negl %ecx
2432; AVX2-NEXT:    vmovd %ecx, %xmm0
2433; AVX2-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
2434; AVX2-NEXT:    movq %rax, 16(%r14)
2435; AVX2-NEXT:    movq %rdi, (%r14)
2436; AVX2-NEXT:    movq %rdx, 24(%r14)
2437; AVX2-NEXT:    movq %rsi, 8(%r14)
2438; AVX2-NEXT:    popq %rbx
2439; AVX2-NEXT:    popq %r12
2440; AVX2-NEXT:    popq %r13
2441; AVX2-NEXT:    popq %r14
2442; AVX2-NEXT:    popq %r15
2443; AVX2-NEXT:    popq %rbp
2444; AVX2-NEXT:    retq
2445;
2446; AVX512-LABEL: umulo_v2i128:
2447; AVX512:       # %bb.0:
2448; AVX512-NEXT:    pushq %rbp
2449; AVX512-NEXT:    pushq %r15
2450; AVX512-NEXT:    pushq %r14
2451; AVX512-NEXT:    pushq %r13
2452; AVX512-NEXT:    pushq %r12
2453; AVX512-NEXT:    pushq %rbx
2454; AVX512-NEXT:    movq %rcx, %rax
2455; AVX512-NEXT:    movq %rdx, %r12
2456; AVX512-NEXT:    movq %rdi, %r11
2457; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r14
2458; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r15
2459; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
2460; AVX512-NEXT:    testq %r10, %r10
2461; AVX512-NEXT:    setne %dl
2462; AVX512-NEXT:    testq %rcx, %rcx
2463; AVX512-NEXT:    setne %r13b
2464; AVX512-NEXT:    andb %dl, %r13b
2465; AVX512-NEXT:    mulq %r15
2466; AVX512-NEXT:    movq %rax, %rdi
2467; AVX512-NEXT:    seto %bpl
2468; AVX512-NEXT:    movq %r10, %rax
2469; AVX512-NEXT:    mulq %r12
2470; AVX512-NEXT:    movq %rax, %rbx
2471; AVX512-NEXT:    seto %cl
2472; AVX512-NEXT:    orb %bpl, %cl
2473; AVX512-NEXT:    addq %rdi, %rbx
2474; AVX512-NEXT:    movq %r12, %rax
2475; AVX512-NEXT:    mulq %r15
2476; AVX512-NEXT:    movq %rax, %r10
2477; AVX512-NEXT:    movq %rdx, %r15
2478; AVX512-NEXT:    addq %rbx, %r15
2479; AVX512-NEXT:    setb %al
2480; AVX512-NEXT:    orb %cl, %al
2481; AVX512-NEXT:    orb %r13b, %al
2482; AVX512-NEXT:    kmovd %eax, %k0
2483; AVX512-NEXT:    kshiftlw $1, %k0, %k0
2484; AVX512-NEXT:    testq %r9, %r9
2485; AVX512-NEXT:    setne %al
2486; AVX512-NEXT:    testq %rsi, %rsi
2487; AVX512-NEXT:    setne %cl
2488; AVX512-NEXT:    andb %al, %cl
2489; AVX512-NEXT:    movq %rsi, %rax
2490; AVX512-NEXT:    mulq %r8
2491; AVX512-NEXT:    movq %rax, %rsi
2492; AVX512-NEXT:    seto %bpl
2493; AVX512-NEXT:    movq %r9, %rax
2494; AVX512-NEXT:    mulq %r11
2495; AVX512-NEXT:    movq %rax, %rdi
2496; AVX512-NEXT:    seto %bl
2497; AVX512-NEXT:    orb %bpl, %bl
2498; AVX512-NEXT:    addq %rsi, %rdi
2499; AVX512-NEXT:    movq %r11, %rax
2500; AVX512-NEXT:    mulq %r8
2501; AVX512-NEXT:    addq %rdi, %rdx
2502; AVX512-NEXT:    setb %sil
2503; AVX512-NEXT:    orb %bl, %sil
2504; AVX512-NEXT:    orb %cl, %sil
2505; AVX512-NEXT:    andl $1, %esi
2506; AVX512-NEXT:    kmovw %esi, %k1
2507; AVX512-NEXT:    korw %k0, %k1, %k1
2508; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2509; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2510; AVX512-NEXT:    movq %r10, 16(%r14)
2511; AVX512-NEXT:    movq %rax, (%r14)
2512; AVX512-NEXT:    movq %r15, 24(%r14)
2513; AVX512-NEXT:    movq %rdx, 8(%r14)
2514; AVX512-NEXT:    popq %rbx
2515; AVX512-NEXT:    popq %r12
2516; AVX512-NEXT:    popq %r13
2517; AVX512-NEXT:    popq %r14
2518; AVX512-NEXT:    popq %r15
2519; AVX512-NEXT:    popq %rbp
2520; AVX512-NEXT:    retq
2521  %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
2522  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
2523  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
2524  %res = sext <2 x i1> %obit to <2 x i32>
2525  store <2 x i128> %val, <2 x i128>* %p2
2526  ret <2 x i32> %res
2527}
2528