1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
9
10declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
11declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
12declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
13declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
14declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
15declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
16declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
17
18declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
19declare {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
20declare {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
21declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
22declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
23
24declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
25declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
26declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
27
28define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
29; CHECK-LABEL: umulo_v1i32:
30; CHECK:       # %bb.0:
31; CHECK-NEXT:    movq %rdx, %rcx
32; CHECK-NEXT:    movl %edi, %eax
33; CHECK-NEXT:    xorl %edi, %edi
34; CHECK-NEXT:    mull %esi
35; CHECK-NEXT:    seto %dil
36; CHECK-NEXT:    negl %edi
37; CHECK-NEXT:    movl %eax, (%rcx)
38; CHECK-NEXT:    movl %edi, %eax
39; CHECK-NEXT:    retq
40  %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
41  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
42  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
43  %res = sext <1 x i1> %obit to <1 x i32>
44  store <1 x i32> %val, <1 x i32>* %p2
45  ret <1 x i32> %res
46}
47
48define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
49; SSE2-LABEL: umulo_v2i32:
50; SSE2:       # %bb.0:
51; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
52; SSE2-NEXT:    pmuludq %xmm1, %xmm0
53; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
54; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
55; SSE2-NEXT:    pmuludq %xmm2, %xmm4
56; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
57; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
58; SSE2-NEXT:    pxor %xmm2, %xmm2
59; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
60; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
61; SSE2-NEXT:    pxor %xmm2, %xmm1
62; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
63; SSE2-NEXT:    movq %xmm0, (%rdi)
64; SSE2-NEXT:    movdqa %xmm1, %xmm0
65; SSE2-NEXT:    retq
66;
67; SSSE3-LABEL: umulo_v2i32:
68; SSSE3:       # %bb.0:
69; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
70; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
71; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
72; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
73; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
74; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
75; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
76; SSSE3-NEXT:    pxor %xmm2, %xmm2
77; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
78; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
79; SSSE3-NEXT:    pxor %xmm2, %xmm1
80; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
81; SSSE3-NEXT:    movq %xmm0, (%rdi)
82; SSSE3-NEXT:    movdqa %xmm1, %xmm0
83; SSSE3-NEXT:    retq
84;
85; SSE41-LABEL: umulo_v2i32:
86; SSE41:       # %bb.0:
87; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
88; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
89; SSE41-NEXT:    pmuludq %xmm2, %xmm3
90; SSE41-NEXT:    movdqa %xmm0, %xmm2
91; SSE41-NEXT:    pmuludq %xmm1, %xmm2
92; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
93; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
94; SSE41-NEXT:    pxor %xmm3, %xmm3
95; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
96; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
97; SSE41-NEXT:    pxor %xmm3, %xmm2
98; SSE41-NEXT:    pmulld %xmm1, %xmm0
99; SSE41-NEXT:    movq %xmm0, (%rdi)
100; SSE41-NEXT:    movdqa %xmm2, %xmm0
101; SSE41-NEXT:    retq
102;
103; AVX1-LABEL: umulo_v2i32:
104; AVX1:       # %bb.0:
105; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
106; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
107; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
108; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
109; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
110; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
111; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
112; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
113; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
114; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
115; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
116; AVX1-NEXT:    vmovq %xmm0, (%rdi)
117; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
118; AVX1-NEXT:    retq
119;
120; AVX2-LABEL: umulo_v2i32:
121; AVX2:       # %bb.0:
122; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
123; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
124; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
125; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
126; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
127; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
128; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
129; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
130; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
131; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
132; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
133; AVX2-NEXT:    vmovq %xmm0, (%rdi)
134; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
135; AVX2-NEXT:    retq
136;
137; AVX512-LABEL: umulo_v2i32:
138; AVX512:       # %bb.0:
139; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
140; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
141; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
142; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
143; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
144; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
145; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
146; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
147; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
148; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
149; AVX512-NEXT:    vmovq %xmm1, (%rdi)
150; AVX512-NEXT:    retq
151  %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
152  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
153  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
154  %res = sext <2 x i1> %obit to <2 x i32>
155  store <2 x i32> %val, <2 x i32>* %p2
156  ret <2 x i32> %res
157}
158
159define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
160; SSE2-LABEL: umulo_v3i32:
161; SSE2:       # %bb.0:
162; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
163; SSE2-NEXT:    pmuludq %xmm1, %xmm0
164; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
165; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
166; SSE2-NEXT:    pmuludq %xmm2, %xmm4
167; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
168; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
169; SSE2-NEXT:    pxor %xmm2, %xmm2
170; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
171; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
172; SSE2-NEXT:    pxor %xmm2, %xmm1
173; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
174; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
175; SSE2-NEXT:    movd %xmm2, 8(%rdi)
176; SSE2-NEXT:    movq %xmm0, (%rdi)
177; SSE2-NEXT:    movdqa %xmm1, %xmm0
178; SSE2-NEXT:    retq
179;
180; SSSE3-LABEL: umulo_v3i32:
181; SSSE3:       # %bb.0:
182; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
183; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
184; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
185; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
186; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
187; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
188; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
189; SSSE3-NEXT:    pxor %xmm2, %xmm2
190; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
191; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
192; SSSE3-NEXT:    pxor %xmm2, %xmm1
193; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
194; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
195; SSSE3-NEXT:    movd %xmm2, 8(%rdi)
196; SSSE3-NEXT:    movq %xmm0, (%rdi)
197; SSSE3-NEXT:    movdqa %xmm1, %xmm0
198; SSSE3-NEXT:    retq
199;
200; SSE41-LABEL: umulo_v3i32:
201; SSE41:       # %bb.0:
202; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
203; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
204; SSE41-NEXT:    pmuludq %xmm2, %xmm3
205; SSE41-NEXT:    movdqa %xmm0, %xmm2
206; SSE41-NEXT:    pmuludq %xmm1, %xmm2
207; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
208; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
209; SSE41-NEXT:    pxor %xmm3, %xmm3
210; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
211; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
212; SSE41-NEXT:    pxor %xmm3, %xmm2
213; SSE41-NEXT:    pmulld %xmm1, %xmm0
214; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
215; SSE41-NEXT:    movq %xmm0, (%rdi)
216; SSE41-NEXT:    movdqa %xmm2, %xmm0
217; SSE41-NEXT:    retq
218;
219; AVX1-LABEL: umulo_v3i32:
220; AVX1:       # %bb.0:
221; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
222; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
223; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
224; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
225; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
226; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
227; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
228; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
229; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
230; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
231; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
232; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
233; AVX1-NEXT:    vmovq %xmm0, (%rdi)
234; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
235; AVX1-NEXT:    retq
236;
237; AVX2-LABEL: umulo_v3i32:
238; AVX2:       # %bb.0:
239; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
240; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
241; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
242; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
243; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
244; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
245; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
246; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
247; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
248; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
249; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
250; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
251; AVX2-NEXT:    vmovq %xmm0, (%rdi)
252; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
253; AVX2-NEXT:    retq
254;
255; AVX512-LABEL: umulo_v3i32:
256; AVX512:       # %bb.0:
257; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
258; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
259; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
260; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
261; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
262; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
263; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
264; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
265; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
266; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
267; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
268; AVX512-NEXT:    vmovq %xmm1, (%rdi)
269; AVX512-NEXT:    retq
270  %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
271  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
272  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
273  %res = sext <3 x i1> %obit to <3 x i32>
274  store <3 x i32> %val, <3 x i32>* %p2
275  ret <3 x i32> %res
276}
277
278define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
279; SSE2-LABEL: umulo_v4i32:
280; SSE2:       # %bb.0:
281; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
282; SSE2-NEXT:    pmuludq %xmm1, %xmm0
283; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
284; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
285; SSE2-NEXT:    pmuludq %xmm2, %xmm4
286; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
287; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
288; SSE2-NEXT:    pxor %xmm2, %xmm2
289; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
290; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
291; SSE2-NEXT:    pxor %xmm2, %xmm1
292; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
293; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
294; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
295; SSE2-NEXT:    movdqa %xmm0, (%rdi)
296; SSE2-NEXT:    movdqa %xmm1, %xmm0
297; SSE2-NEXT:    retq
298;
299; SSSE3-LABEL: umulo_v4i32:
300; SSSE3:       # %bb.0:
301; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
302; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
303; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
304; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
305; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
306; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
307; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
308; SSSE3-NEXT:    pxor %xmm2, %xmm2
309; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
310; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
311; SSSE3-NEXT:    pxor %xmm2, %xmm1
312; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
313; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
314; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
315; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
316; SSSE3-NEXT:    movdqa %xmm1, %xmm0
317; SSSE3-NEXT:    retq
318;
319; SSE41-LABEL: umulo_v4i32:
320; SSE41:       # %bb.0:
321; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
322; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
323; SSE41-NEXT:    pmuludq %xmm2, %xmm3
324; SSE41-NEXT:    movdqa %xmm0, %xmm2
325; SSE41-NEXT:    pmuludq %xmm1, %xmm2
326; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
327; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
328; SSE41-NEXT:    pxor %xmm3, %xmm3
329; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
330; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
331; SSE41-NEXT:    pxor %xmm3, %xmm2
332; SSE41-NEXT:    pmulld %xmm1, %xmm0
333; SSE41-NEXT:    movdqa %xmm0, (%rdi)
334; SSE41-NEXT:    movdqa %xmm2, %xmm0
335; SSE41-NEXT:    retq
336;
337; AVX1-LABEL: umulo_v4i32:
338; AVX1:       # %bb.0:
339; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
340; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
341; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
342; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
343; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
344; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
345; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
346; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
347; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
348; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
349; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
350; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
351; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
352; AVX1-NEXT:    retq
353;
354; AVX2-LABEL: umulo_v4i32:
355; AVX2:       # %bb.0:
356; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
357; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
358; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
359; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
360; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
361; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
362; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
363; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
364; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
365; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
366; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
367; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
368; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
369; AVX2-NEXT:    retq
370;
371; AVX512-LABEL: umulo_v4i32:
372; AVX512:       # %bb.0:
373; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
374; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
375; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
376; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
377; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
378; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
379; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
380; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
381; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
382; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
383; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
384; AVX512-NEXT:    retq
385  %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
386  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
387  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
388  %res = sext <4 x i1> %obit to <4 x i32>
389  store <4 x i32> %val, <4 x i32>* %p2
390  ret <4 x i32> %res
391}
392
393define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
394; SSE2-LABEL: umulo_v6i32:
395; SSE2:       # %bb.0:
396; SSE2-NEXT:    movq %rdi, %rax
397; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
398; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
399; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
400; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
401; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
402; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
403; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
404; SSE2-NEXT:    movd %r8d, %xmm0
405; SSE2-NEXT:    movd %ecx, %xmm1
406; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
407; SSE2-NEXT:    movd %edx, %xmm0
408; SSE2-NEXT:    movd %esi, %xmm3
409; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
410; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
411; SSE2-NEXT:    movd %r9d, %xmm1
412; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
413; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
414; SSE2-NEXT:    pmuludq %xmm1, %xmm0
415; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
416; SSE2-NEXT:    pmuludq %xmm2, %xmm3
417; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
418; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
419; SSE2-NEXT:    pmuludq %xmm4, %xmm2
420; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
421; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
422; SSE2-NEXT:    pxor %xmm4, %xmm4
423; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
424; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
425; SSE2-NEXT:    pxor %xmm5, %xmm1
426; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
427; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
428; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
429; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
430; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
431; SSE2-NEXT:    pmuludq %xmm2, %xmm6
432; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
433; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]
434; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
435; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
436; SSE2-NEXT:    pxor %xmm5, %xmm7
437; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
438; SSE2-NEXT:    movq %xmm0, 16(%rcx)
439; SSE2-NEXT:    movdqa %xmm3, (%rcx)
440; SSE2-NEXT:    movq %xmm7, 16(%rdi)
441; SSE2-NEXT:    movdqa %xmm1, (%rdi)
442; SSE2-NEXT:    retq
443;
444; SSSE3-LABEL: umulo_v6i32:
445; SSSE3:       # %bb.0:
446; SSSE3-NEXT:    movq %rdi, %rax
447; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
448; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
449; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
450; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
451; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
452; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
453; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
454; SSSE3-NEXT:    movd %r8d, %xmm0
455; SSSE3-NEXT:    movd %ecx, %xmm1
456; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
457; SSSE3-NEXT:    movd %edx, %xmm0
458; SSSE3-NEXT:    movd %esi, %xmm3
459; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
460; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
461; SSSE3-NEXT:    movd %r9d, %xmm1
462; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
463; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
464; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
465; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
466; SSSE3-NEXT:    pmuludq %xmm2, %xmm3
467; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
468; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
469; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
470; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
471; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
472; SSSE3-NEXT:    pxor %xmm4, %xmm4
473; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
474; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
475; SSSE3-NEXT:    pxor %xmm5, %xmm1
476; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
477; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
478; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
479; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
480; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
481; SSSE3-NEXT:    pmuludq %xmm2, %xmm6
482; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
483; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]
484; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
485; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
486; SSSE3-NEXT:    pxor %xmm5, %xmm7
487; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
488; SSSE3-NEXT:    movq %xmm0, 16(%rcx)
489; SSSE3-NEXT:    movdqa %xmm3, (%rcx)
490; SSSE3-NEXT:    movq %xmm7, 16(%rdi)
491; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
492; SSSE3-NEXT:    retq
493;
494; SSE41-LABEL: umulo_v6i32:
495; SSE41:       # %bb.0:
496; SSE41-NEXT:    movq %rdi, %rax
497; SSE41-NEXT:    movd %esi, %xmm2
498; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
499; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
500; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
501; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
502; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
503; SSE41-NEXT:    movdqa %xmm1, %xmm0
504; SSE41-NEXT:    pmuludq %xmm2, %xmm1
505; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
506; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
507; SSE41-NEXT:    movd %r9d, %xmm4
508; SSE41-NEXT:    movdqa %xmm4, %xmm5
509; SSE41-NEXT:    pmuludq %xmm3, %xmm4
510; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
511; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
512; SSE41-NEXT:    pmulld %xmm3, %xmm5
513; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
514; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
515; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
516; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
517; SSE41-NEXT:    pmuludq %xmm3, %xmm6
518; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
519; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
520; SSE41-NEXT:    pxor %xmm8, %xmm8
521; SSE41-NEXT:    pcmpeqd %xmm8, %xmm1
522; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
523; SSE41-NEXT:    pxor %xmm6, %xmm1
524; SSE41-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
525; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
526; SSE41-NEXT:    pmuludq %xmm7, %xmm3
527; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
528; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
529; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
530; SSE41-NEXT:    pxor %xmm6, %xmm4
531; SSE41-NEXT:    pmulld %xmm2, %xmm0
532; SSE41-NEXT:    movq %xmm5, 16(%rcx)
533; SSE41-NEXT:    movdqa %xmm0, (%rcx)
534; SSE41-NEXT:    movq %xmm4, 16(%rdi)
535; SSE41-NEXT:    movdqa %xmm1, (%rdi)
536; SSE41-NEXT:    retq
537;
538; AVX1-LABEL: umulo_v6i32:
539; AVX1:       # %bb.0:
540; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
541; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
542; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
543; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
544; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
545; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
546; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
547; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
548; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
549; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
550; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
551; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
552; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
553; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
554; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
555; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
556; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
557; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
558; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
559; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
560; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
561; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
562; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
563; AVX1-NEXT:    vmovq %xmm1, 16(%rdi)
564; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
565; AVX1-NEXT:    vmovaps %ymm2, %ymm0
566; AVX1-NEXT:    retq
567;
568; AVX2-LABEL: umulo_v6i32:
569; AVX2:       # %bb.0:
570; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
571; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
572; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
573; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
574; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
575; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
576; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
577; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
578; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
579; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
580; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
581; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
582; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
583; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
584; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
585; AVX2-NEXT:    retq
586;
587; AVX512-LABEL: umulo_v6i32:
588; AVX512:       # %bb.0:
589; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
590; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
591; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
592; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
593; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
594; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
595; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
596; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
597; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
598; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
599; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
600; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
601; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
602; AVX512-NEXT:    retq
603  %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
604  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
605  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
606  %res = sext <6 x i1> %obit to <6 x i32>
607  store <6 x i32> %val, <6 x i32>* %p2
608  ret <6 x i32> %res
609}
610
611define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
612; SSE2-LABEL: umulo_v8i32:
613; SSE2:       # %bb.0:
614; SSE2-NEXT:    movdqa %xmm0, %xmm4
615; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
616; SSE2-NEXT:    pmuludq %xmm2, %xmm4
617; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
618; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
619; SSE2-NEXT:    pmuludq %xmm5, %xmm6
620; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
621; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
622; SSE2-NEXT:    pxor %xmm8, %xmm8
623; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
624; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
625; SSE2-NEXT:    pxor %xmm7, %xmm0
626; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
627; SSE2-NEXT:    pmuludq %xmm3, %xmm1
628; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
629; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
630; SSE2-NEXT:    pmuludq %xmm5, %xmm3
631; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
632; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
633; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
634; SSE2-NEXT:    pxor %xmm7, %xmm2
635; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
636; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
637; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
638; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
639; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
640; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
641; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
642; SSE2-NEXT:    movdqa %xmm4, (%rdi)
643; SSE2-NEXT:    movdqa %xmm2, %xmm1
644; SSE2-NEXT:    retq
645;
646; SSSE3-LABEL: umulo_v8i32:
647; SSSE3:       # %bb.0:
648; SSSE3-NEXT:    movdqa %xmm0, %xmm4
649; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
650; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
651; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
652; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
653; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
654; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
655; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
656; SSSE3-NEXT:    pxor %xmm8, %xmm8
657; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
658; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
659; SSSE3-NEXT:    pxor %xmm7, %xmm0
660; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
661; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
662; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
663; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
664; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
665; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
666; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
667; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
668; SSSE3-NEXT:    pxor %xmm7, %xmm2
669; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
670; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
671; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
672; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
673; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
674; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
675; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
676; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
677; SSSE3-NEXT:    movdqa %xmm2, %xmm1
678; SSSE3-NEXT:    retq
679;
680; SSE41-LABEL: umulo_v8i32:
681; SSE41:       # %bb.0:
682; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
683; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
684; SSE41-NEXT:    pmuludq %xmm4, %xmm5
685; SSE41-NEXT:    movdqa %xmm0, %xmm4
686; SSE41-NEXT:    pmuludq %xmm2, %xmm4
687; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
688; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
689; SSE41-NEXT:    pxor %xmm8, %xmm8
690; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
691; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
692; SSE41-NEXT:    pxor %xmm7, %xmm4
693; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
694; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
695; SSE41-NEXT:    pmuludq %xmm5, %xmm6
696; SSE41-NEXT:    movdqa %xmm1, %xmm5
697; SSE41-NEXT:    pmuludq %xmm3, %xmm5
698; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
699; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
700; SSE41-NEXT:    pcmpeqd %xmm8, %xmm5
701; SSE41-NEXT:    pxor %xmm7, %xmm5
702; SSE41-NEXT:    pmulld %xmm2, %xmm0
703; SSE41-NEXT:    pmulld %xmm3, %xmm1
704; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
705; SSE41-NEXT:    movdqa %xmm0, (%rdi)
706; SSE41-NEXT:    movdqa %xmm4, %xmm0
707; SSE41-NEXT:    movdqa %xmm5, %xmm1
708; SSE41-NEXT:    retq
709;
710; AVX1-LABEL: umulo_v8i32:
711; AVX1:       # %bb.0:
712; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
713; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
714; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
715; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
716; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
717; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
718; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
719; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
720; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
721; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
722; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
723; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
724; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
725; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
726; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
727; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
728; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
729; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
730; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
731; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
732; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
733; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
734; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
735; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)
736; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
737; AVX1-NEXT:    vmovaps %ymm2, %ymm0
738; AVX1-NEXT:    retq
739;
740; AVX2-LABEL: umulo_v8i32:
741; AVX2:       # %bb.0:
742; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
743; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
744; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
745; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
746; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
747; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
748; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
749; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
750; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
751; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
752; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
753; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
754; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
755; AVX2-NEXT:    retq
756;
757; AVX512-LABEL: umulo_v8i32:
758; AVX512:       # %bb.0:
759; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
760; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
761; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
762; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
763; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
764; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
765; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
766; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
767; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
768; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
769; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
770; AVX512-NEXT:    retq
771  %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
772  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
773  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
774  %res = sext <8 x i1> %obit to <8 x i32>
775  store <8 x i32> %val, <8 x i32>* %p2
776  ret <8 x i32> %res
777}
778
779define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
780; SSE2-LABEL: umulo_v16i32:
781; SSE2:       # %bb.0:
782; SSE2-NEXT:    movdqa %xmm0, %xmm8
783; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
784; SSE2-NEXT:    pmuludq %xmm4, %xmm8
785; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
786; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
787; SSE2-NEXT:    pmuludq %xmm10, %xmm9
788; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
789; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
790; SSE2-NEXT:    pxor %xmm10, %xmm10
791; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
792; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
793; SSE2-NEXT:    pxor %xmm11, %xmm0
794; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
795; SSE2-NEXT:    pmuludq %xmm5, %xmm1
796; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
797; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
798; SSE2-NEXT:    pmuludq %xmm13, %xmm12
799; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
800; SSE2-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
801; SSE2-NEXT:    pcmpeqd %xmm10, %xmm15
802; SSE2-NEXT:    pxor %xmm11, %xmm15
803; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
804; SSE2-NEXT:    pmuludq %xmm6, %xmm2
805; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
806; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
807; SSE2-NEXT:    pmuludq %xmm14, %xmm13
808; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
809; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
810; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
811; SSE2-NEXT:    pxor %xmm11, %xmm5
812; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
813; SSE2-NEXT:    pmuludq %xmm7, %xmm3
814; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
815; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
816; SSE2-NEXT:    pmuludq %xmm14, %xmm7
817; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
818; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
819; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
820; SSE2-NEXT:    pxor %xmm11, %xmm6
821; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
822; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
823; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
824; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
825; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
826; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
827; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
828; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
829; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
830; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
831; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
832; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
833; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
834; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
835; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
836; SSE2-NEXT:    movdqa %xmm8, (%rdi)
837; SSE2-NEXT:    movdqa %xmm15, %xmm1
838; SSE2-NEXT:    movdqa %xmm5, %xmm2
839; SSE2-NEXT:    movdqa %xmm6, %xmm3
840; SSE2-NEXT:    retq
841;
842; SSSE3-LABEL: umulo_v16i32:
843; SSSE3:       # %bb.0:
844; SSSE3-NEXT:    movdqa %xmm0, %xmm8
845; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
846; SSSE3-NEXT:    pmuludq %xmm4, %xmm8
847; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
848; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
849; SSSE3-NEXT:    pmuludq %xmm10, %xmm9
850; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
851; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
852; SSSE3-NEXT:    pxor %xmm10, %xmm10
853; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
854; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm11
855; SSSE3-NEXT:    pxor %xmm11, %xmm0
856; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
857; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
858; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
859; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
860; SSSE3-NEXT:    pmuludq %xmm13, %xmm12
861; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
862; SSSE3-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
863; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm15
864; SSSE3-NEXT:    pxor %xmm11, %xmm15
865; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
866; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
867; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
868; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
869; SSSE3-NEXT:    pmuludq %xmm14, %xmm13
870; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
871; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
872; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm5
873; SSSE3-NEXT:    pxor %xmm11, %xmm5
874; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
875; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
876; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
877; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
878; SSSE3-NEXT:    pmuludq %xmm14, %xmm7
879; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
880; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
881; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm6
882; SSSE3-NEXT:    pxor %xmm11, %xmm6
883; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
884; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
885; SSSE3-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
886; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
887; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
888; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
889; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
890; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
891; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
892; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
893; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
894; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
895; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
896; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
897; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
898; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
899; SSSE3-NEXT:    movdqa %xmm15, %xmm1
900; SSSE3-NEXT:    movdqa %xmm5, %xmm2
901; SSSE3-NEXT:    movdqa %xmm6, %xmm3
902; SSSE3-NEXT:    retq
903;
904; SSE41-LABEL: umulo_v16i32:
905; SSE41:       # %bb.0:
906; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
907; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
908; SSE41-NEXT:    pmuludq %xmm8, %xmm9
909; SSE41-NEXT:    movdqa %xmm0, %xmm8
910; SSE41-NEXT:    pmuludq %xmm4, %xmm8
911; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
912; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
913; SSE41-NEXT:    pxor %xmm12, %xmm12
914; SSE41-NEXT:    pcmpeqd %xmm12, %xmm8
915; SSE41-NEXT:    pcmpeqd %xmm13, %xmm13
916; SSE41-NEXT:    pxor %xmm13, %xmm8
917; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
918; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
919; SSE41-NEXT:    pmuludq %xmm9, %xmm10
920; SSE41-NEXT:    movdqa %xmm1, %xmm9
921; SSE41-NEXT:    pmuludq %xmm5, %xmm9
922; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
923; SSE41-NEXT:    pblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3],xmm9[4,5],xmm10[6,7]
924; SSE41-NEXT:    pcmpeqd %xmm12, %xmm9
925; SSE41-NEXT:    pxor %xmm13, %xmm9
926; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
927; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]
928; SSE41-NEXT:    pmuludq %xmm10, %xmm11
929; SSE41-NEXT:    movdqa %xmm2, %xmm10
930; SSE41-NEXT:    pmuludq %xmm6, %xmm10
931; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
932; SSE41-NEXT:    pblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]
933; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
934; SSE41-NEXT:    pxor %xmm13, %xmm10
935; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
936; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
937; SSE41-NEXT:    pmuludq %xmm11, %xmm14
938; SSE41-NEXT:    movdqa %xmm3, %xmm11
939; SSE41-NEXT:    pmuludq %xmm7, %xmm11
940; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
941; SSE41-NEXT:    pblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3],xmm11[4,5],xmm14[6,7]
942; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
943; SSE41-NEXT:    pxor %xmm13, %xmm11
944; SSE41-NEXT:    pmulld %xmm4, %xmm0
945; SSE41-NEXT:    pmulld %xmm5, %xmm1
946; SSE41-NEXT:    pmulld %xmm6, %xmm2
947; SSE41-NEXT:    pmulld %xmm7, %xmm3
948; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
949; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
950; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
951; SSE41-NEXT:    movdqa %xmm0, (%rdi)
952; SSE41-NEXT:    movdqa %xmm8, %xmm0
953; SSE41-NEXT:    movdqa %xmm9, %xmm1
954; SSE41-NEXT:    movdqa %xmm10, %xmm2
955; SSE41-NEXT:    movdqa %xmm11, %xmm3
956; SSE41-NEXT:    retq
957;
958; AVX1-LABEL: umulo_v16i32:
959; AVX1:       # %bb.0:
960; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
961; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
962; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
963; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
964; AVX1-NEXT:    vpmuludq %xmm6, %xmm7, %xmm6
965; AVX1-NEXT:    vpmuludq %xmm10, %xmm12, %xmm7
966; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
967; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
968; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
969; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm8, %xmm7
970; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
971; AVX1-NEXT:    vpxor %xmm7, %xmm9, %xmm7
972; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
973; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
974; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
975; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm6
976; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
977; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5],xmm4[6,7]
978; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
979; AVX1-NEXT:    vpxor %xmm4, %xmm9, %xmm4
980; AVX1-NEXT:    vpackssdw %xmm7, %xmm4, %xmm11
981; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
982; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
983; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
984; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
985; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
986; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm7
987; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
988; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
989; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
990; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm13
991; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
992; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
993; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
994; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm7
995; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
996; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
997; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
998; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
999; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
1000; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm5
1001; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1002; AVX1-NEXT:    vpmulld %xmm6, %xmm4, %xmm4
1003; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1004; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
1005; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm0
1006; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
1007; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1008; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1009; AVX1-NEXT:    vpacksswb %xmm11, %xmm11, %xmm1
1010; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1011; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1012; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1013; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1014; AVX1-NEXT:    vmovdqa %xmm6, 48(%rdi)
1015; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1016; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1017; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1018; AVX1-NEXT:    retq
1019;
1020; AVX2-LABEL: umulo_v16i32:
1021; AVX2:       # %bb.0:
1022; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1023; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1024; AVX2-NEXT:    vpmuludq %ymm4, %ymm5, %ymm4
1025; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm5
1026; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1027; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1028; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1029; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
1030; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
1031; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
1032; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm7
1033; AVX2-NEXT:    vpackssdw %xmm7, %xmm4, %xmm4
1034; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[1,1,3,3,5,5,7,7]
1035; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm0[1,1,3,3,5,5,7,7]
1036; AVX2-NEXT:    vpmuludq %ymm7, %ymm8, %ymm7
1037; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm8
1038; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,1,3,3,5,5,7,7]
1039; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
1040; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
1041; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
1042; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1043; AVX2-NEXT:    vpackssdw %xmm6, %xmm5, %xmm5
1044; AVX2-NEXT:    vpacksswb %xmm5, %xmm5, %xmm5
1045; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1046; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1047; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm0
1048; AVX2-NEXT:    vpacksswb %xmm4, %xmm4, %xmm1
1049; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1050; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1051; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1052; AVX2-NEXT:    retq
1053;
1054; AVX512-LABEL: umulo_v16i32:
1055; AVX512:       # %bb.0:
1056; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm2
1057; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1058; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1059; AVX512-NEXT:    vpmuludq %zmm3, %zmm4, %zmm3
1060; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1061; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1062; AVX512-NEXT:    vptestmd %zmm4, %zmm4, %k1
1063; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1064; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1065; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1066; AVX512-NEXT:    retq
1067  %t = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1068  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1069  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1070  %res = sext <16 x i1> %obit to <16 x i32>
1071  store <16 x i32> %val, <16 x i32>* %p2
1072  ret <16 x i32> %res
1073}
1074
1075define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1076; SSE2-LABEL: umulo_v16i8:
1077; SSE2:       # %bb.0:
1078; SSE2-NEXT:    movdqa %xmm1, %xmm2
1079; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1080; SSE2-NEXT:    movdqa %xmm0, %xmm3
1081; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1082; SSE2-NEXT:    pmullw %xmm2, %xmm3
1083; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1084; SSE2-NEXT:    pand %xmm2, %xmm3
1085; SSE2-NEXT:    movdqa %xmm1, %xmm5
1086; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1087; SSE2-NEXT:    movdqa %xmm0, %xmm4
1088; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1089; SSE2-NEXT:    pmullw %xmm5, %xmm4
1090; SSE2-NEXT:    pand %xmm2, %xmm4
1091; SSE2-NEXT:    packuswb %xmm3, %xmm4
1092; SSE2-NEXT:    pxor %xmm2, %xmm2
1093; SSE2-NEXT:    movdqa %xmm1, %xmm3
1094; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1095; SSE2-NEXT:    movdqa %xmm0, %xmm5
1096; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1097; SSE2-NEXT:    pmullw %xmm3, %xmm5
1098; SSE2-NEXT:    psrlw $8, %xmm5
1099; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1100; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1101; SSE2-NEXT:    pmullw %xmm1, %xmm0
1102; SSE2-NEXT:    psrlw $8, %xmm0
1103; SSE2-NEXT:    packuswb %xmm5, %xmm0
1104; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
1105; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1106; SSE2-NEXT:    pxor %xmm0, %xmm3
1107; SSE2-NEXT:    movdqa %xmm3, %xmm1
1108; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1109; SSE2-NEXT:    movdqa %xmm1, %xmm0
1110; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1111; SSE2-NEXT:    pslld $31, %xmm0
1112; SSE2-NEXT:    psrad $31, %xmm0
1113; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1114; SSE2-NEXT:    pslld $31, %xmm1
1115; SSE2-NEXT:    psrad $31, %xmm1
1116; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1117; SSE2-NEXT:    movdqa %xmm3, %xmm2
1118; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1119; SSE2-NEXT:    pslld $31, %xmm2
1120; SSE2-NEXT:    psrad $31, %xmm2
1121; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1122; SSE2-NEXT:    pslld $31, %xmm3
1123; SSE2-NEXT:    psrad $31, %xmm3
1124; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1125; SSE2-NEXT:    retq
1126;
1127; SSSE3-LABEL: umulo_v16i8:
1128; SSSE3:       # %bb.0:
1129; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1130; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1131; SSSE3-NEXT:    movdqa %xmm0, %xmm3
1132; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1133; SSSE3-NEXT:    pmullw %xmm2, %xmm3
1134; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1135; SSSE3-NEXT:    pand %xmm2, %xmm3
1136; SSSE3-NEXT:    movdqa %xmm1, %xmm5
1137; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1138; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1139; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1140; SSSE3-NEXT:    pmullw %xmm5, %xmm4
1141; SSSE3-NEXT:    pand %xmm2, %xmm4
1142; SSSE3-NEXT:    packuswb %xmm3, %xmm4
1143; SSSE3-NEXT:    pxor %xmm2, %xmm2
1144; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1145; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1146; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1147; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1148; SSSE3-NEXT:    pmullw %xmm3, %xmm5
1149; SSSE3-NEXT:    psrlw $8, %xmm5
1150; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1151; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1152; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1153; SSSE3-NEXT:    psrlw $8, %xmm0
1154; SSSE3-NEXT:    packuswb %xmm5, %xmm0
1155; SSSE3-NEXT:    pcmpeqb %xmm2, %xmm0
1156; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1157; SSSE3-NEXT:    pxor %xmm0, %xmm3
1158; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1159; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1160; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1161; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1162; SSSE3-NEXT:    pslld $31, %xmm0
1163; SSSE3-NEXT:    psrad $31, %xmm0
1164; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1165; SSSE3-NEXT:    pslld $31, %xmm1
1166; SSSE3-NEXT:    psrad $31, %xmm1
1167; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1168; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1169; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1170; SSSE3-NEXT:    pslld $31, %xmm2
1171; SSSE3-NEXT:    psrad $31, %xmm2
1172; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1173; SSSE3-NEXT:    pslld $31, %xmm3
1174; SSSE3-NEXT:    psrad $31, %xmm3
1175; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1176; SSSE3-NEXT:    retq
1177;
1178; SSE41-LABEL: umulo_v16i8:
1179; SSE41:       # %bb.0:
1180; SSE41-NEXT:    movdqa %xmm1, %xmm2
1181; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1182; SSE41-NEXT:    movdqa %xmm0, %xmm3
1183; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1184; SSE41-NEXT:    pmullw %xmm2, %xmm3
1185; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
1186; SSE41-NEXT:    pand %xmm4, %xmm3
1187; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1188; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1189; SSE41-NEXT:    pmullw %xmm2, %xmm5
1190; SSE41-NEXT:    pand %xmm5, %xmm4
1191; SSE41-NEXT:    packuswb %xmm3, %xmm4
1192; SSE41-NEXT:    pxor %xmm2, %xmm2
1193; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1194; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1195; SSE41-NEXT:    pmullw %xmm1, %xmm0
1196; SSE41-NEXT:    psrlw $8, %xmm0
1197; SSE41-NEXT:    psrlw $8, %xmm5
1198; SSE41-NEXT:    packuswb %xmm0, %xmm5
1199; SSE41-NEXT:    pcmpeqb %xmm2, %xmm5
1200; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1201; SSE41-NEXT:    pxor %xmm5, %xmm3
1202; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1203; SSE41-NEXT:    pslld $31, %xmm0
1204; SSE41-NEXT:    psrad $31, %xmm0
1205; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
1206; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1207; SSE41-NEXT:    pslld $31, %xmm1
1208; SSE41-NEXT:    psrad $31, %xmm1
1209; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1210; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1211; SSE41-NEXT:    pslld $31, %xmm2
1212; SSE41-NEXT:    psrad $31, %xmm2
1213; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1214; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1215; SSE41-NEXT:    pslld $31, %xmm3
1216; SSE41-NEXT:    psrad $31, %xmm3
1217; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1218; SSE41-NEXT:    retq
1219;
1220; AVX1-LABEL: umulo_v16i8:
1221; AVX1:       # %bb.0:
1222; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1223; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1224; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1225; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1226; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1227; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1228; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1229; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1230; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1231; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1232; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1233; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1234; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1235; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1236; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1237; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm1
1238; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1239; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1240; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1241; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1242; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1243; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
1244; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1245; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1246; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1247; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1248; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1249; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1250; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1251; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1252; AVX1-NEXT:    retq
1253;
1254; AVX2-LABEL: umulo_v16i8:
1255; AVX2:       # %bb.0:
1256; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1257; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1258; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1259; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm1
1260; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
1261; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm2
1262; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1263; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1264; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1265; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1266; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
1267; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1268; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1269; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1270; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1271; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1272; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1273; AVX2-NEXT:    retq
1274;
1275; AVX512F-LABEL: umulo_v16i8:
1276; AVX512F:       # %bb.0:
1277; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1278; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1279; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1280; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm0
1281; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1282; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1283; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1284; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
1285; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1286; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1287; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
1288; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1289; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
1290; AVX512F-NEXT:    vpmovdb %zmm1, (%rdi)
1291; AVX512F-NEXT:    retq
1292;
1293; AVX512BW-LABEL: umulo_v16i8:
1294; AVX512BW:       # %bb.0:
1295; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1296; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1297; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1298; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm0
1299; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
1300; AVX512BW-NEXT:    vptestmb %xmm0, %xmm0, %k1
1301; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1302; AVX512BW-NEXT:    vpmovwb %ymm1, (%rdi)
1303; AVX512BW-NEXT:    retq
1304  %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1305  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1306  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1307  %res = sext <16 x i1> %obit to <16 x i32>
1308  store <16 x i8> %val, <16 x i8>* %p2
1309  ret <16 x i32> %res
1310}
1311
1312define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nounwind {
1313; SSE2-LABEL: umulo_v32i8:
1314; SSE2:       # %bb.0:
1315; SSE2-NEXT:    movq %rdi, %rax
1316; SSE2-NEXT:    movdqa %xmm2, %xmm4
1317; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1318; SSE2-NEXT:    movdqa %xmm0, %xmm5
1319; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1320; SSE2-NEXT:    pmullw %xmm4, %xmm5
1321; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1322; SSE2-NEXT:    pand %xmm8, %xmm5
1323; SSE2-NEXT:    movdqa %xmm2, %xmm7
1324; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1325; SSE2-NEXT:    movdqa %xmm0, %xmm9
1326; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1327; SSE2-NEXT:    pmullw %xmm7, %xmm9
1328; SSE2-NEXT:    pand %xmm8, %xmm9
1329; SSE2-NEXT:    packuswb %xmm5, %xmm9
1330; SSE2-NEXT:    movdqa %xmm3, %xmm5
1331; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1332; SSE2-NEXT:    movdqa %xmm1, %xmm7
1333; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1334; SSE2-NEXT:    pmullw %xmm5, %xmm7
1335; SSE2-NEXT:    pand %xmm8, %xmm7
1336; SSE2-NEXT:    movdqa %xmm3, %xmm6
1337; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1338; SSE2-NEXT:    movdqa %xmm1, %xmm10
1339; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1340; SSE2-NEXT:    pmullw %xmm6, %xmm10
1341; SSE2-NEXT:    pand %xmm8, %xmm10
1342; SSE2-NEXT:    packuswb %xmm7, %xmm10
1343; SSE2-NEXT:    pxor %xmm6, %xmm6
1344; SSE2-NEXT:    movdqa %xmm3, %xmm7
1345; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1346; SSE2-NEXT:    movdqa %xmm1, %xmm4
1347; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1348; SSE2-NEXT:    pmullw %xmm7, %xmm4
1349; SSE2-NEXT:    psrlw $8, %xmm4
1350; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
1351; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
1352; SSE2-NEXT:    pmullw %xmm3, %xmm1
1353; SSE2-NEXT:    psrlw $8, %xmm1
1354; SSE2-NEXT:    packuswb %xmm4, %xmm1
1355; SSE2-NEXT:    pcmpeqb %xmm6, %xmm1
1356; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1357; SSE2-NEXT:    pxor %xmm3, %xmm1
1358; SSE2-NEXT:    movdqa %xmm2, %xmm4
1359; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1360; SSE2-NEXT:    movdqa %xmm0, %xmm7
1361; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1362; SSE2-NEXT:    pmullw %xmm4, %xmm7
1363; SSE2-NEXT:    psrlw $8, %xmm7
1364; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
1365; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
1366; SSE2-NEXT:    pmullw %xmm2, %xmm0
1367; SSE2-NEXT:    psrlw $8, %xmm0
1368; SSE2-NEXT:    packuswb %xmm7, %xmm0
1369; SSE2-NEXT:    pcmpeqb %xmm6, %xmm0
1370; SSE2-NEXT:    pxor %xmm3, %xmm0
1371; SSE2-NEXT:    movdqa %xmm0, %xmm2
1372; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1373; SSE2-NEXT:    movdqa %xmm2, %xmm3
1374; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1375; SSE2-NEXT:    pslld $31, %xmm3
1376; SSE2-NEXT:    psrad $31, %xmm3
1377; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1378; SSE2-NEXT:    pslld $31, %xmm2
1379; SSE2-NEXT:    psrad $31, %xmm2
1380; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1381; SSE2-NEXT:    movdqa %xmm0, %xmm6
1382; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1383; SSE2-NEXT:    pslld $31, %xmm6
1384; SSE2-NEXT:    psrad $31, %xmm6
1385; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1386; SSE2-NEXT:    pslld $31, %xmm0
1387; SSE2-NEXT:    psrad $31, %xmm0
1388; SSE2-NEXT:    movdqa %xmm1, %xmm4
1389; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1390; SSE2-NEXT:    movdqa %xmm4, %xmm7
1391; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1392; SSE2-NEXT:    pslld $31, %xmm7
1393; SSE2-NEXT:    psrad $31, %xmm7
1394; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
1395; SSE2-NEXT:    pslld $31, %xmm4
1396; SSE2-NEXT:    psrad $31, %xmm4
1397; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1398; SSE2-NEXT:    movdqa %xmm1, %xmm5
1399; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1400; SSE2-NEXT:    pslld $31, %xmm5
1401; SSE2-NEXT:    psrad $31, %xmm5
1402; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1403; SSE2-NEXT:    pslld $31, %xmm1
1404; SSE2-NEXT:    psrad $31, %xmm1
1405; SSE2-NEXT:    movdqa %xmm10, 16(%rsi)
1406; SSE2-NEXT:    movdqa %xmm9, (%rsi)
1407; SSE2-NEXT:    movdqa %xmm1, 112(%rdi)
1408; SSE2-NEXT:    movdqa %xmm5, 96(%rdi)
1409; SSE2-NEXT:    movdqa %xmm4, 80(%rdi)
1410; SSE2-NEXT:    movdqa %xmm7, 64(%rdi)
1411; SSE2-NEXT:    movdqa %xmm0, 48(%rdi)
1412; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
1413; SSE2-NEXT:    movdqa %xmm2, 16(%rdi)
1414; SSE2-NEXT:    movdqa %xmm3, (%rdi)
1415; SSE2-NEXT:    retq
1416;
1417; SSSE3-LABEL: umulo_v32i8:
1418; SSSE3:       # %bb.0:
1419; SSSE3-NEXT:    movq %rdi, %rax
1420; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1421; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1422; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1423; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1424; SSSE3-NEXT:    pmullw %xmm4, %xmm5
1425; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1426; SSSE3-NEXT:    pand %xmm8, %xmm5
1427; SSSE3-NEXT:    movdqa %xmm2, %xmm7
1428; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1429; SSSE3-NEXT:    movdqa %xmm0, %xmm9
1430; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1431; SSSE3-NEXT:    pmullw %xmm7, %xmm9
1432; SSSE3-NEXT:    pand %xmm8, %xmm9
1433; SSSE3-NEXT:    packuswb %xmm5, %xmm9
1434; SSSE3-NEXT:    movdqa %xmm3, %xmm5
1435; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1436; SSSE3-NEXT:    movdqa %xmm1, %xmm7
1437; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1438; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1439; SSSE3-NEXT:    pand %xmm8, %xmm7
1440; SSSE3-NEXT:    movdqa %xmm3, %xmm6
1441; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1442; SSSE3-NEXT:    movdqa %xmm1, %xmm10
1443; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1444; SSSE3-NEXT:    pmullw %xmm6, %xmm10
1445; SSSE3-NEXT:    pand %xmm8, %xmm10
1446; SSSE3-NEXT:    packuswb %xmm7, %xmm10
1447; SSSE3-NEXT:    pxor %xmm6, %xmm6
1448; SSSE3-NEXT:    movdqa %xmm3, %xmm7
1449; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1450; SSSE3-NEXT:    movdqa %xmm1, %xmm4
1451; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1452; SSSE3-NEXT:    pmullw %xmm7, %xmm4
1453; SSSE3-NEXT:    psrlw $8, %xmm4
1454; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
1455; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
1456; SSSE3-NEXT:    pmullw %xmm3, %xmm1
1457; SSSE3-NEXT:    psrlw $8, %xmm1
1458; SSSE3-NEXT:    packuswb %xmm4, %xmm1
1459; SSSE3-NEXT:    pcmpeqb %xmm6, %xmm1
1460; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1461; SSSE3-NEXT:    pxor %xmm3, %xmm1
1462; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1463; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1464; SSSE3-NEXT:    movdqa %xmm0, %xmm7
1465; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1466; SSSE3-NEXT:    pmullw %xmm4, %xmm7
1467; SSSE3-NEXT:    psrlw $8, %xmm7
1468; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
1469; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
1470; SSSE3-NEXT:    pmullw %xmm2, %xmm0
1471; SSSE3-NEXT:    psrlw $8, %xmm0
1472; SSSE3-NEXT:    packuswb %xmm7, %xmm0
1473; SSSE3-NEXT:    pcmpeqb %xmm6, %xmm0
1474; SSSE3-NEXT:    pxor %xmm3, %xmm0
1475; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1476; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1477; SSSE3-NEXT:    movdqa %xmm2, %xmm3
1478; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1479; SSSE3-NEXT:    pslld $31, %xmm3
1480; SSSE3-NEXT:    psrad $31, %xmm3
1481; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1482; SSSE3-NEXT:    pslld $31, %xmm2
1483; SSSE3-NEXT:    psrad $31, %xmm2
1484; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1485; SSSE3-NEXT:    movdqa %xmm0, %xmm6
1486; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
1487; SSSE3-NEXT:    pslld $31, %xmm6
1488; SSSE3-NEXT:    psrad $31, %xmm6
1489; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1490; SSSE3-NEXT:    pslld $31, %xmm0
1491; SSSE3-NEXT:    psrad $31, %xmm0
1492; SSSE3-NEXT:    movdqa %xmm1, %xmm4
1493; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1494; SSSE3-NEXT:    movdqa %xmm4, %xmm7
1495; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
1496; SSSE3-NEXT:    pslld $31, %xmm7
1497; SSSE3-NEXT:    psrad $31, %xmm7
1498; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
1499; SSSE3-NEXT:    pslld $31, %xmm4
1500; SSSE3-NEXT:    psrad $31, %xmm4
1501; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1502; SSSE3-NEXT:    movdqa %xmm1, %xmm5
1503; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1504; SSSE3-NEXT:    pslld $31, %xmm5
1505; SSSE3-NEXT:    psrad $31, %xmm5
1506; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1507; SSSE3-NEXT:    pslld $31, %xmm1
1508; SSSE3-NEXT:    psrad $31, %xmm1
1509; SSSE3-NEXT:    movdqa %xmm10, 16(%rsi)
1510; SSSE3-NEXT:    movdqa %xmm9, (%rsi)
1511; SSSE3-NEXT:    movdqa %xmm1, 112(%rdi)
1512; SSSE3-NEXT:    movdqa %xmm5, 96(%rdi)
1513; SSSE3-NEXT:    movdqa %xmm4, 80(%rdi)
1514; SSSE3-NEXT:    movdqa %xmm7, 64(%rdi)
1515; SSSE3-NEXT:    movdqa %xmm0, 48(%rdi)
1516; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
1517; SSSE3-NEXT:    movdqa %xmm2, 16(%rdi)
1518; SSSE3-NEXT:    movdqa %xmm3, (%rdi)
1519; SSSE3-NEXT:    retq
1520;
1521; SSE41-LABEL: umulo_v32i8:
1522; SSE41:       # %bb.0:
1523; SSE41-NEXT:    movq %rdi, %rax
1524; SSE41-NEXT:    movdqa %xmm2, %xmm4
1525; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1526; SSE41-NEXT:    movdqa %xmm0, %xmm7
1527; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1528; SSE41-NEXT:    pmullw %xmm4, %xmm7
1529; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
1530; SSE41-NEXT:    pand %xmm9, %xmm7
1531; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1532; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1533; SSE41-NEXT:    pmullw %xmm5, %xmm6
1534; SSE41-NEXT:    movdqa %xmm6, %xmm8
1535; SSE41-NEXT:    pand %xmm9, %xmm8
1536; SSE41-NEXT:    packuswb %xmm7, %xmm8
1537; SSE41-NEXT:    movdqa %xmm3, %xmm7
1538; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1539; SSE41-NEXT:    movdqa %xmm1, %xmm5
1540; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1541; SSE41-NEXT:    pmullw %xmm7, %xmm5
1542; SSE41-NEXT:    pand %xmm9, %xmm5
1543; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1544; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1545; SSE41-NEXT:    pmullw %xmm4, %xmm7
1546; SSE41-NEXT:    pand %xmm7, %xmm9
1547; SSE41-NEXT:    packuswb %xmm5, %xmm9
1548; SSE41-NEXT:    pxor %xmm4, %xmm4
1549; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
1550; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
1551; SSE41-NEXT:    pmullw %xmm3, %xmm1
1552; SSE41-NEXT:    psrlw $8, %xmm1
1553; SSE41-NEXT:    psrlw $8, %xmm7
1554; SSE41-NEXT:    packuswb %xmm1, %xmm7
1555; SSE41-NEXT:    pcmpeqb %xmm4, %xmm7
1556; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
1557; SSE41-NEXT:    pxor %xmm1, %xmm7
1558; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
1559; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
1560; SSE41-NEXT:    pmullw %xmm2, %xmm0
1561; SSE41-NEXT:    psrlw $8, %xmm0
1562; SSE41-NEXT:    psrlw $8, %xmm6
1563; SSE41-NEXT:    packuswb %xmm0, %xmm6
1564; SSE41-NEXT:    pcmpeqb %xmm4, %xmm6
1565; SSE41-NEXT:    pxor %xmm1, %xmm6
1566; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
1567; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1568; SSE41-NEXT:    pslld $31, %xmm0
1569; SSE41-NEXT:    psrad $31, %xmm0
1570; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
1571; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1572; SSE41-NEXT:    pslld $31, %xmm1
1573; SSE41-NEXT:    psrad $31, %xmm1
1574; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
1575; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1576; SSE41-NEXT:    pslld $31, %xmm2
1577; SSE41-NEXT:    psrad $31, %xmm2
1578; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,1,1]
1579; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1580; SSE41-NEXT:    pslld $31, %xmm3
1581; SSE41-NEXT:    psrad $31, %xmm3
1582; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[2,3,2,3]
1583; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
1584; SSE41-NEXT:    pslld $31, %xmm4
1585; SSE41-NEXT:    psrad $31, %xmm4
1586; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[3,3,3,3]
1587; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
1588; SSE41-NEXT:    pslld $31, %xmm5
1589; SSE41-NEXT:    psrad $31, %xmm5
1590; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
1591; SSE41-NEXT:    pslld $31, %xmm6
1592; SSE41-NEXT:    psrad $31, %xmm6
1593; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
1594; SSE41-NEXT:    pslld $31, %xmm7
1595; SSE41-NEXT:    psrad $31, %xmm7
1596; SSE41-NEXT:    movdqa %xmm9, 16(%rsi)
1597; SSE41-NEXT:    movdqa %xmm8, (%rsi)
1598; SSE41-NEXT:    movdqa %xmm7, 64(%rdi)
1599; SSE41-NEXT:    movdqa %xmm6, (%rdi)
1600; SSE41-NEXT:    movdqa %xmm5, 112(%rdi)
1601; SSE41-NEXT:    movdqa %xmm4, 96(%rdi)
1602; SSE41-NEXT:    movdqa %xmm3, 80(%rdi)
1603; SSE41-NEXT:    movdqa %xmm2, 48(%rdi)
1604; SSE41-NEXT:    movdqa %xmm1, 32(%rdi)
1605; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
1606; SSE41-NEXT:    retq
1607;
1608; AVX1-LABEL: umulo_v32i8:
1609; AVX1:       # %bb.0:
1610; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1611; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1612; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1613; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1614; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1615; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1616; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1617; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm9
1618; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm4
1619; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm8
1620; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1621; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1622; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1623; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1624; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
1625; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm4
1626; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1627; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
1628; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm6
1629; AVX1-NEXT:    vpand %xmm3, %xmm6, %xmm3
1630; AVX1-NEXT:    vpackuswb %xmm4, %xmm3, %xmm5
1631; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1632; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
1633; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1634; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
1635; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1636; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm4
1637; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
1638; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
1639; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1640; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
1641; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1642; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1643; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1644; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1645; AVX1-NEXT:    vpsrlw $8, %xmm9, %xmm1
1646; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1647; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1648; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
1649; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1650; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1651; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
1652; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1653; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm2
1654; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
1655; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1656; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
1657; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1658; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1659; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1660; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1661; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1662; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
1663; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1664; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,3,3]
1665; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1666; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
1667; AVX1-NEXT:    vmovdqa %xmm5, 16(%rdi)
1668; AVX1-NEXT:    vmovdqa %xmm8, (%rdi)
1669; AVX1-NEXT:    retq
1670;
1671; AVX2-LABEL: umulo_v32i8:
1672; AVX2:       # %bb.0:
1673; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1674; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1675; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1676; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1677; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
1678; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1679; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1680; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1681; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
1682; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
1683; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1684; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
1685; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
1686; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
1687; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
1688; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
1689; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
1690; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1691; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1692; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
1693; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
1694; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
1695; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
1696; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1697; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1698; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
1699; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1700; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1701; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1702; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
1703; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
1704; AVX2-NEXT:    retq
1705;
1706; AVX512F-LABEL: umulo_v32i8:
1707; AVX512F:       # %bb.0:
1708; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
1709; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1710; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
1711; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
1712; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1713; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm3
1714; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1715; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1716; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
1717; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
1718; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
1719; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
1720; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k1
1721; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1722; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1723; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm3
1724; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm0
1725; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1726; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1727; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
1728; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1729; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1730; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
1731; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
1732; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1733; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1734; AVX512F-NEXT:    vpmovdb %zmm2, 16(%rdi)
1735; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1736; AVX512F-NEXT:    vpmovdb %zmm2, (%rdi)
1737; AVX512F-NEXT:    retq
1738;
1739; AVX512BW-LABEL: umulo_v32i8:
1740; AVX512BW:       # %bb.0:
1741; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
1742; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1743; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
1744; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm0
1745; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
1746; AVX512BW-NEXT:    vptestmb %ymm0, %ymm0, %k1
1747; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1748; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
1749; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1750; AVX512BW-NEXT:    vpmovwb %zmm2, (%rdi)
1751; AVX512BW-NEXT:    retq
1752  %t = call {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)
1753  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 0
1754  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 1
1755  %res = sext <32 x i1> %obit to <32 x i32>
1756  store <32 x i8> %val, <32 x i8>* %p2
1757  ret <32 x i32> %res
1758}
1759
1760define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nounwind {
1761; SSE2-LABEL: umulo_v64i8:
1762; SSE2:       # %bb.0:
1763; SSE2-NEXT:    movq %rdi, %rax
1764; SSE2-NEXT:    movdqa %xmm4, %xmm8
1765; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1766; SSE2-NEXT:    movdqa %xmm0, %xmm9
1767; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1768; SSE2-NEXT:    pmullw %xmm8, %xmm9
1769; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [255,255,255,255,255,255,255,255]
1770; SSE2-NEXT:    pand %xmm12, %xmm9
1771; SSE2-NEXT:    movdqa %xmm4, %xmm10
1772; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1773; SSE2-NEXT:    movdqa %xmm0, %xmm8
1774; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1775; SSE2-NEXT:    pmullw %xmm10, %xmm8
1776; SSE2-NEXT:    pand %xmm12, %xmm8
1777; SSE2-NEXT:    packuswb %xmm9, %xmm8
1778; SSE2-NEXT:    movdqa %xmm5, %xmm9
1779; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1780; SSE2-NEXT:    movdqa %xmm1, %xmm10
1781; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1782; SSE2-NEXT:    pmullw %xmm9, %xmm10
1783; SSE2-NEXT:    pand %xmm12, %xmm10
1784; SSE2-NEXT:    movdqa %xmm5, %xmm11
1785; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1786; SSE2-NEXT:    movdqa %xmm1, %xmm9
1787; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1788; SSE2-NEXT:    pmullw %xmm11, %xmm9
1789; SSE2-NEXT:    pand %xmm12, %xmm9
1790; SSE2-NEXT:    packuswb %xmm10, %xmm9
1791; SSE2-NEXT:    movdqa %xmm6, %xmm10
1792; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1793; SSE2-NEXT:    movdqa %xmm2, %xmm11
1794; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1795; SSE2-NEXT:    pmullw %xmm10, %xmm11
1796; SSE2-NEXT:    pand %xmm12, %xmm11
1797; SSE2-NEXT:    movdqa %xmm6, %xmm13
1798; SSE2-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1799; SSE2-NEXT:    movdqa %xmm2, %xmm10
1800; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1801; SSE2-NEXT:    pmullw %xmm13, %xmm10
1802; SSE2-NEXT:    pand %xmm12, %xmm10
1803; SSE2-NEXT:    packuswb %xmm11, %xmm10
1804; SSE2-NEXT:    movdqa %xmm7, %xmm11
1805; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1806; SSE2-NEXT:    movdqa %xmm3, %xmm13
1807; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1808; SSE2-NEXT:    pmullw %xmm11, %xmm13
1809; SSE2-NEXT:    pand %xmm12, %xmm13
1810; SSE2-NEXT:    movdqa %xmm7, %xmm14
1811; SSE2-NEXT:    punpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1812; SSE2-NEXT:    movdqa %xmm3, %xmm11
1813; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1814; SSE2-NEXT:    pmullw %xmm14, %xmm11
1815; SSE2-NEXT:    pand %xmm12, %xmm11
1816; SSE2-NEXT:    packuswb %xmm13, %xmm11
1817; SSE2-NEXT:    pxor %xmm12, %xmm12
1818; SSE2-NEXT:    movdqa %xmm7, %xmm13
1819; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
1820; SSE2-NEXT:    movdqa %xmm3, %xmm14
1821; SSE2-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm12[8],xmm14[9],xmm12[9],xmm14[10],xmm12[10],xmm14[11],xmm12[11],xmm14[12],xmm12[12],xmm14[13],xmm12[13],xmm14[14],xmm12[14],xmm14[15],xmm12[15]
1822; SSE2-NEXT:    pmullw %xmm13, %xmm14
1823; SSE2-NEXT:    psrlw $8, %xmm14
1824; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1],xmm7[2],xmm12[2],xmm7[3],xmm12[3],xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
1825; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
1826; SSE2-NEXT:    pmullw %xmm7, %xmm3
1827; SSE2-NEXT:    psrlw $8, %xmm3
1828; SSE2-NEXT:    packuswb %xmm14, %xmm3
1829; SSE2-NEXT:    movdqa %xmm6, %xmm13
1830; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
1831; SSE2-NEXT:    movdqa %xmm2, %xmm7
1832; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
1833; SSE2-NEXT:    pmullw %xmm13, %xmm7
1834; SSE2-NEXT:    psrlw $8, %xmm7
1835; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]
1836; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
1837; SSE2-NEXT:    pmullw %xmm6, %xmm2
1838; SSE2-NEXT:    psrlw $8, %xmm2
1839; SSE2-NEXT:    packuswb %xmm7, %xmm2
1840; SSE2-NEXT:    movdqa %xmm5, %xmm6
1841; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
1842; SSE2-NEXT:    movdqa %xmm1, %xmm7
1843; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
1844; SSE2-NEXT:    pmullw %xmm6, %xmm7
1845; SSE2-NEXT:    psrlw $8, %xmm7
1846; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3],xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
1847; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
1848; SSE2-NEXT:    pmullw %xmm5, %xmm1
1849; SSE2-NEXT:    psrlw $8, %xmm1
1850; SSE2-NEXT:    packuswb %xmm7, %xmm1
1851; SSE2-NEXT:    movdqa %xmm4, %xmm5
1852; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm12[8],xmm5[9],xmm12[9],xmm5[10],xmm12[10],xmm5[11],xmm12[11],xmm5[12],xmm12[12],xmm5[13],xmm12[13],xmm5[14],xmm12[14],xmm5[15],xmm12[15]
1853; SSE2-NEXT:    movdqa %xmm0, %xmm6
1854; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
1855; SSE2-NEXT:    pmullw %xmm5, %xmm6
1856; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
1857; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
1858; SSE2-NEXT:    pmullw %xmm4, %xmm0
1859; SSE2-NEXT:    psrlw $8, %xmm6
1860; SSE2-NEXT:    psrlw $8, %xmm0
1861; SSE2-NEXT:    packuswb %xmm6, %xmm0
1862; SSE2-NEXT:    pcmpeqb %xmm12, %xmm3
1863; SSE2-NEXT:    pcmpeqb %xmm12, %xmm2
1864; SSE2-NEXT:    pcmpeqb %xmm12, %xmm1
1865; SSE2-NEXT:    pcmpeqb %xmm12, %xmm0
1866; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
1867; SSE2-NEXT:    pxor %xmm4, %xmm3
1868; SSE2-NEXT:    pxor %xmm4, %xmm2
1869; SSE2-NEXT:    pxor %xmm4, %xmm1
1870; SSE2-NEXT:    pxor %xmm4, %xmm0
1871; SSE2-NEXT:    movdqa %xmm11, 48(%rsi)
1872; SSE2-NEXT:    movdqa %xmm10, 32(%rsi)
1873; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
1874; SSE2-NEXT:    movdqa %xmm3, %xmm4
1875; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1876; SSE2-NEXT:    movdqa %xmm8, (%rsi)
1877; SSE2-NEXT:    movdqa %xmm3, %xmm5
1878; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1879; SSE2-NEXT:    pslld $31, %xmm3
1880; SSE2-NEXT:    psrad $31, %xmm3
1881; SSE2-NEXT:    movdqa %xmm3, 224(%rdi)
1882; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1883; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1884; SSE2-NEXT:    pslld $31, %xmm5
1885; SSE2-NEXT:    psrad $31, %xmm5
1886; SSE2-NEXT:    movdqa %xmm5, 240(%rdi)
1887; SSE2-NEXT:    movdqa %xmm4, %xmm3
1888; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1889; SSE2-NEXT:    pslld $31, %xmm4
1890; SSE2-NEXT:    psrad $31, %xmm4
1891; SSE2-NEXT:    movdqa %xmm4, 192(%rdi)
1892; SSE2-NEXT:    movdqa %xmm2, %xmm4
1893; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1894; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1895; SSE2-NEXT:    pslld $31, %xmm3
1896; SSE2-NEXT:    psrad $31, %xmm3
1897; SSE2-NEXT:    movdqa %xmm3, 208(%rdi)
1898; SSE2-NEXT:    movdqa %xmm2, %xmm3
1899; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1900; SSE2-NEXT:    pslld $31, %xmm2
1901; SSE2-NEXT:    psrad $31, %xmm2
1902; SSE2-NEXT:    movdqa %xmm2, 160(%rdi)
1903; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1904; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1905; SSE2-NEXT:    pslld $31, %xmm3
1906; SSE2-NEXT:    psrad $31, %xmm3
1907; SSE2-NEXT:    movdqa %xmm3, 176(%rdi)
1908; SSE2-NEXT:    movdqa %xmm4, %xmm2
1909; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1910; SSE2-NEXT:    pslld $31, %xmm4
1911; SSE2-NEXT:    psrad $31, %xmm4
1912; SSE2-NEXT:    movdqa %xmm4, 128(%rdi)
1913; SSE2-NEXT:    movdqa %xmm1, %xmm3
1914; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1915; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1916; SSE2-NEXT:    pslld $31, %xmm2
1917; SSE2-NEXT:    psrad $31, %xmm2
1918; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
1919; SSE2-NEXT:    movdqa %xmm1, %xmm2
1920; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
1921; SSE2-NEXT:    pslld $31, %xmm1
1922; SSE2-NEXT:    psrad $31, %xmm1
1923; SSE2-NEXT:    movdqa %xmm1, 96(%rdi)
1924; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1925; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1926; SSE2-NEXT:    pslld $31, %xmm2
1927; SSE2-NEXT:    psrad $31, %xmm2
1928; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
1929; SSE2-NEXT:    movdqa %xmm3, %xmm1
1930; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1931; SSE2-NEXT:    pslld $31, %xmm3
1932; SSE2-NEXT:    psrad $31, %xmm3
1933; SSE2-NEXT:    movdqa %xmm3, 64(%rdi)
1934; SSE2-NEXT:    movdqa %xmm0, %xmm2
1935; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1936; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1937; SSE2-NEXT:    pslld $31, %xmm1
1938; SSE2-NEXT:    psrad $31, %xmm1
1939; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
1940; SSE2-NEXT:    movdqa %xmm0, %xmm1
1941; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1942; SSE2-NEXT:    pslld $31, %xmm0
1943; SSE2-NEXT:    psrad $31, %xmm0
1944; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
1945; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1946; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1947; SSE2-NEXT:    pslld $31, %xmm1
1948; SSE2-NEXT:    psrad $31, %xmm1
1949; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
1950; SSE2-NEXT:    movdqa %xmm2, %xmm0
1951; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1952; SSE2-NEXT:    pslld $31, %xmm2
1953; SSE2-NEXT:    psrad $31, %xmm2
1954; SSE2-NEXT:    movdqa %xmm2, (%rdi)
1955; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1956; SSE2-NEXT:    pslld $31, %xmm0
1957; SSE2-NEXT:    psrad $31, %xmm0
1958; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
1959; SSE2-NEXT:    retq
1960;
1961; SSSE3-LABEL: umulo_v64i8:
1962; SSSE3:       # %bb.0:
1963; SSSE3-NEXT:    movq %rdi, %rax
1964; SSSE3-NEXT:    movdqa %xmm4, %xmm8
1965; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1966; SSSE3-NEXT:    movdqa %xmm0, %xmm9
1967; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1968; SSSE3-NEXT:    pmullw %xmm8, %xmm9
1969; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [255,255,255,255,255,255,255,255]
1970; SSSE3-NEXT:    pand %xmm12, %xmm9
1971; SSSE3-NEXT:    movdqa %xmm4, %xmm10
1972; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1973; SSSE3-NEXT:    movdqa %xmm0, %xmm8
1974; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1975; SSSE3-NEXT:    pmullw %xmm10, %xmm8
1976; SSSE3-NEXT:    pand %xmm12, %xmm8
1977; SSSE3-NEXT:    packuswb %xmm9, %xmm8
1978; SSSE3-NEXT:    movdqa %xmm5, %xmm9
1979; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1980; SSSE3-NEXT:    movdqa %xmm1, %xmm10
1981; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1982; SSSE3-NEXT:    pmullw %xmm9, %xmm10
1983; SSSE3-NEXT:    pand %xmm12, %xmm10
1984; SSSE3-NEXT:    movdqa %xmm5, %xmm11
1985; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1986; SSSE3-NEXT:    movdqa %xmm1, %xmm9
1987; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1988; SSSE3-NEXT:    pmullw %xmm11, %xmm9
1989; SSSE3-NEXT:    pand %xmm12, %xmm9
1990; SSSE3-NEXT:    packuswb %xmm10, %xmm9
1991; SSSE3-NEXT:    movdqa %xmm6, %xmm10
1992; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1993; SSSE3-NEXT:    movdqa %xmm2, %xmm11
1994; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1995; SSSE3-NEXT:    pmullw %xmm10, %xmm11
1996; SSSE3-NEXT:    pand %xmm12, %xmm11
1997; SSSE3-NEXT:    movdqa %xmm6, %xmm13
1998; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1999; SSSE3-NEXT:    movdqa %xmm2, %xmm10
2000; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2001; SSSE3-NEXT:    pmullw %xmm13, %xmm10
2002; SSSE3-NEXT:    pand %xmm12, %xmm10
2003; SSSE3-NEXT:    packuswb %xmm11, %xmm10
2004; SSSE3-NEXT:    movdqa %xmm7, %xmm11
2005; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2006; SSSE3-NEXT:    movdqa %xmm3, %xmm13
2007; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2008; SSSE3-NEXT:    pmullw %xmm11, %xmm13
2009; SSSE3-NEXT:    pand %xmm12, %xmm13
2010; SSSE3-NEXT:    movdqa %xmm7, %xmm14
2011; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2012; SSSE3-NEXT:    movdqa %xmm3, %xmm11
2013; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2014; SSSE3-NEXT:    pmullw %xmm14, %xmm11
2015; SSSE3-NEXT:    pand %xmm12, %xmm11
2016; SSSE3-NEXT:    packuswb %xmm13, %xmm11
2017; SSSE3-NEXT:    pxor %xmm12, %xmm12
2018; SSSE3-NEXT:    movdqa %xmm7, %xmm13
2019; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
2020; SSSE3-NEXT:    movdqa %xmm3, %xmm14
2021; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm12[8],xmm14[9],xmm12[9],xmm14[10],xmm12[10],xmm14[11],xmm12[11],xmm14[12],xmm12[12],xmm14[13],xmm12[13],xmm14[14],xmm12[14],xmm14[15],xmm12[15]
2022; SSSE3-NEXT:    pmullw %xmm13, %xmm14
2023; SSSE3-NEXT:    psrlw $8, %xmm14
2024; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1],xmm7[2],xmm12[2],xmm7[3],xmm12[3],xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
2025; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
2026; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2027; SSSE3-NEXT:    psrlw $8, %xmm3
2028; SSSE3-NEXT:    packuswb %xmm14, %xmm3
2029; SSSE3-NEXT:    movdqa %xmm6, %xmm13
2030; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
2031; SSSE3-NEXT:    movdqa %xmm2, %xmm7
2032; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
2033; SSSE3-NEXT:    pmullw %xmm13, %xmm7
2034; SSSE3-NEXT:    psrlw $8, %xmm7
2035; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]
2036; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
2037; SSSE3-NEXT:    pmullw %xmm6, %xmm2
2038; SSSE3-NEXT:    psrlw $8, %xmm2
2039; SSSE3-NEXT:    packuswb %xmm7, %xmm2
2040; SSSE3-NEXT:    movdqa %xmm5, %xmm6
2041; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
2042; SSSE3-NEXT:    movdqa %xmm1, %xmm7
2043; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
2044; SSSE3-NEXT:    pmullw %xmm6, %xmm7
2045; SSSE3-NEXT:    psrlw $8, %xmm7
2046; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3],xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
2047; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
2048; SSSE3-NEXT:    pmullw %xmm5, %xmm1
2049; SSSE3-NEXT:    psrlw $8, %xmm1
2050; SSSE3-NEXT:    packuswb %xmm7, %xmm1
2051; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2052; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm12[8],xmm5[9],xmm12[9],xmm5[10],xmm12[10],xmm5[11],xmm12[11],xmm5[12],xmm12[12],xmm5[13],xmm12[13],xmm5[14],xmm12[14],xmm5[15],xmm12[15]
2053; SSSE3-NEXT:    movdqa %xmm0, %xmm6
2054; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
2055; SSSE3-NEXT:    pmullw %xmm5, %xmm6
2056; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
2057; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
2058; SSSE3-NEXT:    pmullw %xmm4, %xmm0
2059; SSSE3-NEXT:    psrlw $8, %xmm6
2060; SSSE3-NEXT:    psrlw $8, %xmm0
2061; SSSE3-NEXT:    packuswb %xmm6, %xmm0
2062; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm3
2063; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm2
2064; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm1
2065; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm0
2066; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
2067; SSSE3-NEXT:    pxor %xmm4, %xmm3
2068; SSSE3-NEXT:    pxor %xmm4, %xmm2
2069; SSSE3-NEXT:    pxor %xmm4, %xmm1
2070; SSSE3-NEXT:    pxor %xmm4, %xmm0
2071; SSSE3-NEXT:    movdqa %xmm11, 48(%rsi)
2072; SSSE3-NEXT:    movdqa %xmm10, 32(%rsi)
2073; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
2074; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2075; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2076; SSSE3-NEXT:    movdqa %xmm8, (%rsi)
2077; SSSE3-NEXT:    movdqa %xmm3, %xmm5
2078; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2079; SSSE3-NEXT:    pslld $31, %xmm3
2080; SSSE3-NEXT:    psrad $31, %xmm3
2081; SSSE3-NEXT:    movdqa %xmm3, 224(%rdi)
2082; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2083; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
2084; SSSE3-NEXT:    pslld $31, %xmm5
2085; SSSE3-NEXT:    psrad $31, %xmm5
2086; SSSE3-NEXT:    movdqa %xmm5, 240(%rdi)
2087; SSSE3-NEXT:    movdqa %xmm4, %xmm3
2088; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2089; SSSE3-NEXT:    pslld $31, %xmm4
2090; SSSE3-NEXT:    psrad $31, %xmm4
2091; SSSE3-NEXT:    movdqa %xmm4, 192(%rdi)
2092; SSSE3-NEXT:    movdqa %xmm2, %xmm4
2093; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2094; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
2095; SSSE3-NEXT:    pslld $31, %xmm3
2096; SSSE3-NEXT:    psrad $31, %xmm3
2097; SSSE3-NEXT:    movdqa %xmm3, 208(%rdi)
2098; SSSE3-NEXT:    movdqa %xmm2, %xmm3
2099; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2100; SSSE3-NEXT:    pslld $31, %xmm2
2101; SSSE3-NEXT:    psrad $31, %xmm2
2102; SSSE3-NEXT:    movdqa %xmm2, 160(%rdi)
2103; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2104; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
2105; SSSE3-NEXT:    pslld $31, %xmm3
2106; SSSE3-NEXT:    psrad $31, %xmm3
2107; SSSE3-NEXT:    movdqa %xmm3, 176(%rdi)
2108; SSSE3-NEXT:    movdqa %xmm4, %xmm2
2109; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2110; SSSE3-NEXT:    pslld $31, %xmm4
2111; SSSE3-NEXT:    psrad $31, %xmm4
2112; SSSE3-NEXT:    movdqa %xmm4, 128(%rdi)
2113; SSSE3-NEXT:    movdqa %xmm1, %xmm3
2114; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2115; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2116; SSSE3-NEXT:    pslld $31, %xmm2
2117; SSSE3-NEXT:    psrad $31, %xmm2
2118; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
2119; SSSE3-NEXT:    movdqa %xmm1, %xmm2
2120; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2121; SSSE3-NEXT:    pslld $31, %xmm1
2122; SSSE3-NEXT:    psrad $31, %xmm1
2123; SSSE3-NEXT:    movdqa %xmm1, 96(%rdi)
2124; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2125; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2126; SSSE3-NEXT:    pslld $31, %xmm2
2127; SSSE3-NEXT:    psrad $31, %xmm2
2128; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
2129; SSSE3-NEXT:    movdqa %xmm3, %xmm1
2130; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2131; SSSE3-NEXT:    pslld $31, %xmm3
2132; SSSE3-NEXT:    psrad $31, %xmm3
2133; SSSE3-NEXT:    movdqa %xmm3, 64(%rdi)
2134; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2135; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2136; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2137; SSSE3-NEXT:    pslld $31, %xmm1
2138; SSSE3-NEXT:    psrad $31, %xmm1
2139; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
2140; SSSE3-NEXT:    movdqa %xmm0, %xmm1
2141; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2142; SSSE3-NEXT:    pslld $31, %xmm0
2143; SSSE3-NEXT:    psrad $31, %xmm0
2144; SSSE3-NEXT:    movdqa %xmm0, 32(%rdi)
2145; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2146; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2147; SSSE3-NEXT:    pslld $31, %xmm1
2148; SSSE3-NEXT:    psrad $31, %xmm1
2149; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
2150; SSSE3-NEXT:    movdqa %xmm2, %xmm0
2151; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2152; SSSE3-NEXT:    pslld $31, %xmm2
2153; SSSE3-NEXT:    psrad $31, %xmm2
2154; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
2155; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2156; SSSE3-NEXT:    pslld $31, %xmm0
2157; SSSE3-NEXT:    psrad $31, %xmm0
2158; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
2159; SSSE3-NEXT:    retq
2160;
2161; SSE41-LABEL: umulo_v64i8:
2162; SSE41:       # %bb.0:
2163; SSE41-NEXT:    movdqa %xmm5, %xmm11
2164; SSE41-NEXT:    movdqa %xmm4, %xmm13
2165; SSE41-NEXT:    movdqa %xmm3, %xmm4
2166; SSE41-NEXT:    movdqa %xmm0, %xmm5
2167; SSE41-NEXT:    movq %rdi, %rax
2168; SSE41-NEXT:    movdqa %xmm13, %xmm8
2169; SSE41-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2170; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2171; SSE41-NEXT:    pmullw %xmm8, %xmm0
2172; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2173; SSE41-NEXT:    pand %xmm10, %xmm0
2174; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
2175; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2176; SSE41-NEXT:    pmullw %xmm9, %xmm8
2177; SSE41-NEXT:    movdqa %xmm8, %xmm3
2178; SSE41-NEXT:    pand %xmm10, %xmm3
2179; SSE41-NEXT:    packuswb %xmm0, %xmm3
2180; SSE41-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2181; SSE41-NEXT:    movdqa %xmm11, %xmm9
2182; SSE41-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2183; SSE41-NEXT:    movdqa %xmm1, %xmm0
2184; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2185; SSE41-NEXT:    pmullw %xmm9, %xmm0
2186; SSE41-NEXT:    pand %xmm10, %xmm0
2187; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2188; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2189; SSE41-NEXT:    pmullw %xmm12, %xmm9
2190; SSE41-NEXT:    movdqa %xmm9, %xmm3
2191; SSE41-NEXT:    pand %xmm10, %xmm3
2192; SSE41-NEXT:    packuswb %xmm0, %xmm3
2193; SSE41-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2194; SSE41-NEXT:    movdqa %xmm6, %xmm12
2195; SSE41-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2196; SSE41-NEXT:    movdqa %xmm2, %xmm0
2197; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2198; SSE41-NEXT:    pmullw %xmm12, %xmm0
2199; SSE41-NEXT:    pand %xmm10, %xmm0
2200; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2201; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2202; SSE41-NEXT:    pmullw %xmm14, %xmm12
2203; SSE41-NEXT:    movdqa %xmm12, %xmm15
2204; SSE41-NEXT:    pand %xmm10, %xmm15
2205; SSE41-NEXT:    packuswb %xmm0, %xmm15
2206; SSE41-NEXT:    movdqa %xmm7, %xmm0
2207; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2208; SSE41-NEXT:    movdqa %xmm4, %xmm3
2209; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2210; SSE41-NEXT:    pmullw %xmm0, %xmm3
2211; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2212; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2213; SSE41-NEXT:    pmullw %xmm0, %xmm14
2214; SSE41-NEXT:    pand %xmm10, %xmm3
2215; SSE41-NEXT:    pand %xmm14, %xmm10
2216; SSE41-NEXT:    packuswb %xmm3, %xmm10
2217; SSE41-NEXT:    pxor %xmm0, %xmm0
2218; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2219; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2220; SSE41-NEXT:    pmullw %xmm7, %xmm4
2221; SSE41-NEXT:    psrlw $8, %xmm4
2222; SSE41-NEXT:    psrlw $8, %xmm14
2223; SSE41-NEXT:    packuswb %xmm4, %xmm14
2224; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
2225; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2226; SSE41-NEXT:    pmullw %xmm6, %xmm2
2227; SSE41-NEXT:    psrlw $8, %xmm2
2228; SSE41-NEXT:    psrlw $8, %xmm12
2229; SSE41-NEXT:    packuswb %xmm2, %xmm12
2230; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2231; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2232; SSE41-NEXT:    pmullw %xmm11, %xmm1
2233; SSE41-NEXT:    psrlw $8, %xmm1
2234; SSE41-NEXT:    psrlw $8, %xmm9
2235; SSE41-NEXT:    packuswb %xmm1, %xmm9
2236; SSE41-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm0[8],xmm13[9],xmm0[9],xmm13[10],xmm0[10],xmm13[11],xmm0[11],xmm13[12],xmm0[12],xmm13[13],xmm0[13],xmm13[14],xmm0[14],xmm13[15],xmm0[15]
2237; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
2238; SSE41-NEXT:    pmullw %xmm13, %xmm5
2239; SSE41-NEXT:    psrlw $8, %xmm5
2240; SSE41-NEXT:    psrlw $8, %xmm8
2241; SSE41-NEXT:    packuswb %xmm5, %xmm8
2242; SSE41-NEXT:    pcmpeqb %xmm0, %xmm14
2243; SSE41-NEXT:    pcmpeqb %xmm0, %xmm12
2244; SSE41-NEXT:    pcmpeqb %xmm0, %xmm9
2245; SSE41-NEXT:    pcmpeqb %xmm0, %xmm8
2246; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2247; SSE41-NEXT:    pxor %xmm0, %xmm14
2248; SSE41-NEXT:    pxor %xmm0, %xmm12
2249; SSE41-NEXT:    pxor %xmm0, %xmm9
2250; SSE41-NEXT:    pxor %xmm0, %xmm8
2251; SSE41-NEXT:    movdqa %xmm10, 48(%rsi)
2252; SSE41-NEXT:    movdqa %xmm15, 32(%rsi)
2253; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2254; SSE41-NEXT:    movaps %xmm0, 16(%rsi)
2255; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2256; SSE41-NEXT:    movaps %xmm0, (%rsi)
2257; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero
2258; SSE41-NEXT:    pslld $31, %xmm0
2259; SSE41-NEXT:    psrad $31, %xmm0
2260; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
2261; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero,xmm12[2],zero,zero,zero,xmm12[3],zero,zero,zero
2262; SSE41-NEXT:    pslld $31, %xmm0
2263; SSE41-NEXT:    psrad $31, %xmm0
2264; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
2265; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero
2266; SSE41-NEXT:    pslld $31, %xmm0
2267; SSE41-NEXT:    psrad $31, %xmm0
2268; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
2269; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
2270; SSE41-NEXT:    pslld $31, %xmm0
2271; SSE41-NEXT:    psrad $31, %xmm0
2272; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2273; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[2,3,2,3]
2274; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2275; SSE41-NEXT:    pslld $31, %xmm0
2276; SSE41-NEXT:    psrad $31, %xmm0
2277; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
2278; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[3,3,3,3]
2279; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2280; SSE41-NEXT:    pslld $31, %xmm0
2281; SSE41-NEXT:    psrad $31, %xmm0
2282; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
2283; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[1,1,1,1]
2284; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2285; SSE41-NEXT:    pslld $31, %xmm0
2286; SSE41-NEXT:    psrad $31, %xmm0
2287; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
2288; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[2,3,2,3]
2289; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2290; SSE41-NEXT:    pslld $31, %xmm0
2291; SSE41-NEXT:    psrad $31, %xmm0
2292; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
2293; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[3,3,3,3]
2294; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2295; SSE41-NEXT:    pslld $31, %xmm0
2296; SSE41-NEXT:    psrad $31, %xmm0
2297; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
2298; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,1,1]
2299; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2300; SSE41-NEXT:    pslld $31, %xmm0
2301; SSE41-NEXT:    psrad $31, %xmm0
2302; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
2303; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
2304; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2305; SSE41-NEXT:    pslld $31, %xmm0
2306; SSE41-NEXT:    psrad $31, %xmm0
2307; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
2308; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[3,3,3,3]
2309; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2310; SSE41-NEXT:    pslld $31, %xmm0
2311; SSE41-NEXT:    psrad $31, %xmm0
2312; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
2313; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,1,1]
2314; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2315; SSE41-NEXT:    pslld $31, %xmm0
2316; SSE41-NEXT:    psrad $31, %xmm0
2317; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
2318; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
2319; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2320; SSE41-NEXT:    pslld $31, %xmm0
2321; SSE41-NEXT:    psrad $31, %xmm0
2322; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
2323; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
2324; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2325; SSE41-NEXT:    pslld $31, %xmm0
2326; SSE41-NEXT:    psrad $31, %xmm0
2327; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
2328; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
2329; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2330; SSE41-NEXT:    pslld $31, %xmm0
2331; SSE41-NEXT:    psrad $31, %xmm0
2332; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
2333; SSE41-NEXT:    retq
2334;
2335; AVX1-LABEL: umulo_v64i8:
2336; AVX1:       # %bb.0:
2337; AVX1-NEXT:    vmovdqa %ymm2, %ymm10
2338; AVX1-NEXT:    vmovdqa %ymm0, %ymm2
2339; AVX1-NEXT:    movq %rdi, %rax
2340; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2341; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2342; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2343; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
2344; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
2345; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
2346; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2347; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm12
2348; AVX1-NEXT:    vpand %xmm7, %xmm12, %xmm5
2349; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm0
2350; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2351; AVX1-NEXT:    vextractf128 $1, %ymm10, %xmm13
2352; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2353; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm14
2354; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm14[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2355; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2356; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm5
2357; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
2358; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
2359; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm15
2360; AVX1-NEXT:    vpand %xmm7, %xmm15, %xmm6
2361; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm0
2362; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2363; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2364; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2365; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2366; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
2367; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2368; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2369; AVX1-NEXT:    vpmullw %xmm6, %xmm4, %xmm0
2370; AVX1-NEXT:    vpand %xmm7, %xmm0, %xmm6
2371; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm4
2372; AVX1-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2373; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
2374; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2375; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
2376; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2377; AVX1-NEXT:    vpmullw %xmm6, %xmm8, %xmm8
2378; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2379; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm9 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2380; AVX1-NEXT:    vpmullw %xmm6, %xmm9, %xmm6
2381; AVX1-NEXT:    vpand %xmm7, %xmm8, %xmm8
2382; AVX1-NEXT:    vpand %xmm7, %xmm6, %xmm7
2383; AVX1-NEXT:    vpackuswb %xmm8, %xmm7, %xmm8
2384; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
2385; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
2386; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
2387; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
2388; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2389; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm5
2390; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
2391; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm7[8],xmm3[9],xmm7[9],xmm3[10],xmm7[10],xmm3[11],xmm7[11],xmm3[12],xmm7[12],xmm3[13],xmm7[13],xmm3[14],xmm7[14],xmm3[15],xmm7[15]
2392; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
2393; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2394; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2395; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
2396; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
2397; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm7[8],xmm13[9],xmm7[9],xmm13[10],xmm7[10],xmm13[11],xmm7[11],xmm13[12],xmm7[12],xmm13[13],xmm7[13],xmm13[14],xmm7[14],xmm13[15],xmm7[15]
2398; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm7[8],xmm14[9],xmm7[9],xmm14[10],xmm7[10],xmm14[11],xmm7[11],xmm14[12],xmm7[12],xmm14[13],xmm7[13],xmm14[14],xmm7[14],xmm14[15],xmm7[15]
2399; AVX1-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
2400; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2401; AVX1-NEXT:    vpsrlw $8, %xmm15, %xmm3
2402; AVX1-NEXT:    vpackuswb %xmm1, %xmm3, %xmm1
2403; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2404; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]
2405; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
2406; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
2407; AVX1-NEXT:    vpsrlw $8, %xmm12, %xmm3
2408; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
2409; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm3
2410; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm0, %xmm0
2411; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm1, %xmm1
2412; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm2, %xmm4
2413; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
2414; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
2415; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm2
2416; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
2417; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm0
2418; AVX1-NEXT:    vmovdqa %xmm8, 48(%rsi)
2419; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2420; AVX1-NEXT:    vmovaps %xmm4, 32(%rsi)
2421; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2422; AVX1-NEXT:    vmovaps %xmm4, 16(%rsi)
2423; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2424; AVX1-NEXT:    vmovaps %xmm4, (%rsi)
2425; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm4
2426; AVX1-NEXT:    vmovdqa %xmm4, 192(%rdi)
2427; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm4
2428; AVX1-NEXT:    vmovdqa %xmm4, 128(%rdi)
2429; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
2430; AVX1-NEXT:    vmovdqa %xmm4, 64(%rdi)
2431; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
2432; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)
2433; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
2434; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
2435; AVX1-NEXT:    vmovdqa %xmm4, 224(%rdi)
2436; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
2437; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
2438; AVX1-NEXT:    vmovdqa %xmm4, 240(%rdi)
2439; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
2440; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2441; AVX1-NEXT:    vmovdqa %xmm3, 208(%rdi)
2442; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
2443; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2444; AVX1-NEXT:    vmovdqa %xmm3, 160(%rdi)
2445; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
2446; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2447; AVX1-NEXT:    vmovdqa %xmm3, 176(%rdi)
2448; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
2449; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2450; AVX1-NEXT:    vmovdqa %xmm2, 144(%rdi)
2451; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2452; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2453; AVX1-NEXT:    vmovdqa %xmm2, 96(%rdi)
2454; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
2455; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2456; AVX1-NEXT:    vmovdqa %xmm2, 112(%rdi)
2457; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
2458; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2459; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
2460; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
2461; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2462; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
2463; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2464; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2465; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
2466; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
2467; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
2468; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
2469; AVX1-NEXT:    vzeroupper
2470; AVX1-NEXT:    retq
2471;
2472; AVX2-LABEL: umulo_v64i8:
2473; AVX2:       # %bb.0:
2474; AVX2-NEXT:    movq %rdi, %rax
2475; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2476; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2477; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2478; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2479; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
2480; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2481; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2482; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2483; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2484; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm9
2485; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2486; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2487; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2488; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2489; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2490; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2491; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2492; AVX2-NEXT:    vpand %ymm5, %ymm7, %ymm5
2493; AVX2-NEXT:    vpackuswb %ymm6, %ymm5, %ymm10
2494; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
2495; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm3[8],ymm6[8],ymm3[9],ymm6[9],ymm3[10],ymm6[10],ymm3[11],ymm6[11],ymm3[12],ymm6[12],ymm3[13],ymm6[13],ymm3[14],ymm6[14],ymm3[15],ymm6[15],ymm3[24],ymm6[24],ymm3[25],ymm6[25],ymm3[26],ymm6[26],ymm3[27],ymm6[27],ymm3[28],ymm6[28],ymm3[29],ymm6[29],ymm3[30],ymm6[30],ymm3[31],ymm6[31]
2496; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm1[8],ymm6[8],ymm1[9],ymm6[9],ymm1[10],ymm6[10],ymm1[11],ymm6[11],ymm1[12],ymm6[12],ymm1[13],ymm6[13],ymm1[14],ymm6[14],ymm1[15],ymm6[15],ymm1[24],ymm6[24],ymm1[25],ymm6[25],ymm1[26],ymm6[26],ymm1[27],ymm6[27],ymm1[28],ymm6[28],ymm1[29],ymm6[29],ymm1[30],ymm6[30],ymm1[31],ymm6[31]
2497; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2498; AVX2-NEXT:    vpsrlw $8, %ymm7, %ymm7
2499; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[2],ymm6[2],ymm3[3],ymm6[3],ymm3[4],ymm6[4],ymm3[5],ymm6[5],ymm3[6],ymm6[6],ymm3[7],ymm6[7],ymm3[16],ymm6[16],ymm3[17],ymm6[17],ymm3[18],ymm6[18],ymm3[19],ymm6[19],ymm3[20],ymm6[20],ymm3[21],ymm6[21],ymm3[22],ymm6[22],ymm3[23],ymm6[23]
2500; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[1],ymm6[1],ymm1[2],ymm6[2],ymm1[3],ymm6[3],ymm1[4],ymm6[4],ymm1[5],ymm6[5],ymm1[6],ymm6[6],ymm1[7],ymm6[7],ymm1[16],ymm6[16],ymm1[17],ymm6[17],ymm1[18],ymm6[18],ymm1[19],ymm6[19],ymm1[20],ymm6[20],ymm1[21],ymm6[21],ymm1[22],ymm6[22],ymm1[23],ymm6[23]
2501; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2502; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
2503; AVX2-NEXT:    vpackuswb %ymm7, %ymm1, %ymm1
2504; AVX2-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
2505; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
2506; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
2507; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm2[8],ymm6[8],ymm2[9],ymm6[9],ymm2[10],ymm6[10],ymm2[11],ymm6[11],ymm2[12],ymm6[12],ymm2[13],ymm6[13],ymm2[14],ymm6[14],ymm2[15],ymm6[15],ymm2[24],ymm6[24],ymm2[25],ymm6[25],ymm2[26],ymm6[26],ymm2[27],ymm6[27],ymm2[28],ymm6[28],ymm2[29],ymm6[29],ymm2[30],ymm6[30],ymm2[31],ymm6[31]
2508; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8],ymm6[8],ymm0[9],ymm6[9],ymm0[10],ymm6[10],ymm0[11],ymm6[11],ymm0[12],ymm6[12],ymm0[13],ymm6[13],ymm0[14],ymm6[14],ymm0[15],ymm6[15],ymm0[24],ymm6[24],ymm0[25],ymm6[25],ymm0[26],ymm6[26],ymm0[27],ymm6[27],ymm0[28],ymm6[28],ymm0[29],ymm6[29],ymm0[30],ymm6[30],ymm0[31],ymm6[31]
2509; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2510; AVX2-NEXT:    vpsrlw $8, %ymm7, %ymm7
2511; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[4],ymm6[4],ymm2[5],ymm6[5],ymm2[6],ymm6[6],ymm2[7],ymm6[7],ymm2[16],ymm6[16],ymm2[17],ymm6[17],ymm2[18],ymm6[18],ymm2[19],ymm6[19],ymm2[20],ymm6[20],ymm2[21],ymm6[21],ymm2[22],ymm6[22],ymm2[23],ymm6[23]
2512; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[4],ymm6[4],ymm0[5],ymm6[5],ymm0[6],ymm6[6],ymm0[7],ymm6[7],ymm0[16],ymm6[16],ymm0[17],ymm6[17],ymm0[18],ymm6[18],ymm0[19],ymm6[19],ymm0[20],ymm6[20],ymm0[21],ymm6[21],ymm0[22],ymm6[22],ymm0[23],ymm6[23]
2513; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
2514; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
2515; AVX2-NEXT:    vpackuswb %ymm7, %ymm0, %ymm0
2516; AVX2-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
2517; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
2518; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2519; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
2520; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
2521; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
2522; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
2523; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
2524; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
2525; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
2526; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
2527; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
2528; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
2529; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
2530; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
2531; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
2532; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
2533; AVX2-NEXT:    vmovdqa %ymm9, (%rsi)
2534; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
2535; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
2536; AVX2-NEXT:    vmovdqa %ymm3, 64(%rdi)
2537; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
2538; AVX2-NEXT:    vmovdqa %ymm5, 224(%rdi)
2539; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
2540; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
2541; AVX2-NEXT:    vmovdqa %ymm2, 32(%rdi)
2542; AVX2-NEXT:    vzeroupper
2543; AVX2-NEXT:    retq
2544;
2545; AVX512F-LABEL: umulo_v64i8:
2546; AVX512F:       # %bb.0:
2547; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
2548; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm2
2549; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2550; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
2551; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm4
2552; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2553; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm4
2554; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm2
2555; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2556; AVX512F-NEXT:    vpmovdb %zmm2, %xmm6
2557; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2558; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm6, %xmm6
2559; AVX512F-NEXT:    vpternlogq $15, %xmm6, %xmm6, %xmm6
2560; AVX512F-NEXT:    vpmovsxbd %xmm6, %zmm6
2561; AVX512F-NEXT:    vptestmd %zmm6, %zmm6, %k1
2562; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2563; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
2564; AVX512F-NEXT:    vpmullw %ymm3, %ymm5, %ymm5
2565; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm3
2566; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2567; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2568; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
2569; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
2570; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
2571; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k2
2572; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
2573; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2574; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm6
2575; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
2576; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm6
2577; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm3
2578; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2579; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2580; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
2581; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
2582; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
2583; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k3
2584; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2585; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2586; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm7
2587; AVX512F-NEXT:    vpsrlw $8, %ymm7, %ymm0
2588; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
2589; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2590; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2591; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2592; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
2593; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k4
2594; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
2595; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
2596; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
2597; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2598; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
2599; AVX512F-NEXT:    vpmovdb %zmm4, 48(%rdi)
2600; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
2601; AVX512F-NEXT:    vpmovdb %zmm4, 32(%rdi)
2602; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm6[0],zero,ymm6[1],zero,ymm6[2],zero,ymm6[3],zero,ymm6[4],zero,ymm6[5],zero,ymm6[6],zero,ymm6[7],zero,ymm6[8],zero,ymm6[9],zero,ymm6[10],zero,ymm6[11],zero,ymm6[12],zero,ymm6[13],zero,ymm6[14],zero,ymm6[15],zero
2603; AVX512F-NEXT:    vpmovdb %zmm4, 16(%rdi)
2604; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm7[0],zero,ymm7[1],zero,ymm7[2],zero,ymm7[3],zero,ymm7[4],zero,ymm7[5],zero,ymm7[6],zero,ymm7[7],zero,ymm7[8],zero,ymm7[9],zero,ymm7[10],zero,ymm7[11],zero,ymm7[12],zero,ymm7[13],zero,ymm7[14],zero,ymm7[15],zero
2605; AVX512F-NEXT:    vpmovdb %zmm4, (%rdi)
2606; AVX512F-NEXT:    retq
2607;
2608; AVX512BW-LABEL: umulo_v64i8:
2609; AVX512BW:       # %bb.0:
2610; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2611; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2612; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2613; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2614; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
2615; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2616; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm5 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2617; AVX512BW-NEXT:    vpmullw %zmm4, %zmm5, %zmm4
2618; AVX512BW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
2619; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm4
2620; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2621; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
2622; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm5 = zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[10],zmm2[10],zmm0[11],zmm2[11],zmm0[12],zmm2[12],zmm0[13],zmm2[13],zmm0[14],zmm2[14],zmm0[15],zmm2[15],zmm0[24],zmm2[24],zmm0[25],zmm2[25],zmm0[26],zmm2[26],zmm0[27],zmm2[27],zmm0[28],zmm2[28],zmm0[29],zmm2[29],zmm0[30],zmm2[30],zmm0[31],zmm2[31],zmm0[40],zmm2[40],zmm0[41],zmm2[41],zmm0[42],zmm2[42],zmm0[43],zmm2[43],zmm0[44],zmm2[44],zmm0[45],zmm2[45],zmm0[46],zmm2[46],zmm0[47],zmm2[47],zmm0[56],zmm2[56],zmm0[57],zmm2[57],zmm0[58],zmm2[58],zmm0[59],zmm2[59],zmm0[60],zmm2[60],zmm0[61],zmm2[61],zmm0[62],zmm2[62],zmm0[63],zmm2[63]
2623; AVX512BW-NEXT:    vpmullw %zmm3, %zmm5, %zmm3
2624; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
2625; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]
2626; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[2],zmm2[2],zmm0[3],zmm2[3],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[6],zmm2[6],zmm0[7],zmm2[7],zmm0[16],zmm2[16],zmm0[17],zmm2[17],zmm0[18],zmm2[18],zmm0[19],zmm2[19],zmm0[20],zmm2[20],zmm0[21],zmm2[21],zmm0[22],zmm2[22],zmm0[23],zmm2[23],zmm0[32],zmm2[32],zmm0[33],zmm2[33],zmm0[34],zmm2[34],zmm0[35],zmm2[35],zmm0[36],zmm2[36],zmm0[37],zmm2[37],zmm0[38],zmm2[38],zmm0[39],zmm2[39],zmm0[48],zmm2[48],zmm0[49],zmm2[49],zmm0[50],zmm2[50],zmm0[51],zmm2[51],zmm0[52],zmm2[52],zmm0[53],zmm2[53],zmm0[54],zmm2[54],zmm0[55],zmm2[55]
2627; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2628; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
2629; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
2630; AVX512BW-NEXT:    vptestmb %zmm0, %zmm0, %k1
2631; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2632; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
2633; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2634; AVX512BW-NEXT:    kshiftrq $32, %k1, %k1
2635; AVX512BW-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
2636; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2637; AVX512BW-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2638; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdi)
2639; AVX512BW-NEXT:    retq
2640  %t = call {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)
2641  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 0
2642  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 1
2643  %res = sext <64 x i1> %obit to <64 x i32>
2644  store <64 x i8> %val, <64 x i8>* %p2
2645  ret <64 x i32> %res
2646}
2647
2648define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
2649; SSE2-LABEL: umulo_v8i16:
2650; SSE2:       # %bb.0:
2651; SSE2-NEXT:    movdqa %xmm0, %xmm2
2652; SSE2-NEXT:    pmullw %xmm1, %xmm2
2653; SSE2-NEXT:    pmulhuw %xmm1, %xmm0
2654; SSE2-NEXT:    pxor %xmm3, %xmm3
2655; SSE2-NEXT:    pcmpeqw %xmm0, %xmm3
2656; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
2657; SSE2-NEXT:    pxor %xmm3, %xmm1
2658; SSE2-NEXT:    movdqa %xmm1, %xmm0
2659; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2660; SSE2-NEXT:    pslld $31, %xmm0
2661; SSE2-NEXT:    psrad $31, %xmm0
2662; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2663; SSE2-NEXT:    pslld $31, %xmm1
2664; SSE2-NEXT:    psrad $31, %xmm1
2665; SSE2-NEXT:    movdqa %xmm2, (%rdi)
2666; SSE2-NEXT:    retq
2667;
2668; SSSE3-LABEL: umulo_v8i16:
2669; SSSE3:       # %bb.0:
2670; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2671; SSSE3-NEXT:    pmullw %xmm1, %xmm2
2672; SSSE3-NEXT:    pmulhuw %xmm1, %xmm0
2673; SSSE3-NEXT:    pxor %xmm3, %xmm3
2674; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm3
2675; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
2676; SSSE3-NEXT:    pxor %xmm3, %xmm1
2677; SSSE3-NEXT:    movdqa %xmm1, %xmm0
2678; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2679; SSSE3-NEXT:    pslld $31, %xmm0
2680; SSSE3-NEXT:    psrad $31, %xmm0
2681; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2682; SSSE3-NEXT:    pslld $31, %xmm1
2683; SSSE3-NEXT:    psrad $31, %xmm1
2684; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
2685; SSSE3-NEXT:    retq
2686;
2687; SSE41-LABEL: umulo_v8i16:
2688; SSE41:       # %bb.0:
2689; SSE41-NEXT:    movdqa %xmm0, %xmm2
2690; SSE41-NEXT:    pmullw %xmm1, %xmm2
2691; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
2692; SSE41-NEXT:    pxor %xmm3, %xmm3
2693; SSE41-NEXT:    pcmpeqw %xmm0, %xmm3
2694; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
2695; SSE41-NEXT:    pxor %xmm3, %xmm1
2696; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2697; SSE41-NEXT:    pslld $31, %xmm0
2698; SSE41-NEXT:    psrad $31, %xmm0
2699; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2700; SSE41-NEXT:    pslld $31, %xmm1
2701; SSE41-NEXT:    psrad $31, %xmm1
2702; SSE41-NEXT:    movdqa %xmm2, (%rdi)
2703; SSE41-NEXT:    retq
2704;
2705; AVX1-LABEL: umulo_v8i16:
2706; AVX1:       # %bb.0:
2707; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2708; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2709; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2710; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2711; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
2712; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
2713; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
2714; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2715; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
2716; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2717; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
2718; AVX1-NEXT:    retq
2719;
2720; AVX2-LABEL: umulo_v8i16:
2721; AVX2:       # %bb.0:
2722; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2723; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2724; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2725; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2726; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
2727; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
2728; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
2729; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
2730; AVX2-NEXT:    retq
2731;
2732; AVX512F-LABEL: umulo_v8i16:
2733; AVX512F:       # %bb.0:
2734; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2735; AVX512F-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2736; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2737; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2738; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2739; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
2740; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
2741; AVX512F-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
2742; AVX512F-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
2743; AVX512F-NEXT:    vmovdqa %xmm2, (%rdi)
2744; AVX512F-NEXT:    retq
2745;
2746; AVX512BW-LABEL: umulo_v8i16:
2747; AVX512BW:       # %bb.0:
2748; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2749; AVX512BW-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2750; AVX512BW-NEXT:    vptestmw %xmm0, %xmm0, %k1
2751; AVX512BW-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
2752; AVX512BW-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
2753; AVX512BW-NEXT:    vmovdqa %xmm2, (%rdi)
2754; AVX512BW-NEXT:    retq
2755  %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
2756  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
2757  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
2758  %res = sext <8 x i1> %obit to <8 x i32>
2759  store <8 x i16> %val, <8 x i16>* %p2
2760  ret <8 x i32> %res
2761}
2762
2763define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
2764; SSE2-LABEL: umulo_v2i64:
2765; SSE2:       # %bb.0:
2766; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2767; SSE2-NEXT:    movq %xmm2, %r8
2768; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2769; SSE2-NEXT:    movq %xmm2, %r10
2770; SSE2-NEXT:    movq %xmm0, %rax
2771; SSE2-NEXT:    movq %xmm1, %rdx
2772; SSE2-NEXT:    xorl %ecx, %ecx
2773; SSE2-NEXT:    mulq %rdx
2774; SSE2-NEXT:    movq $-1, %r9
2775; SSE2-NEXT:    movl $0, %esi
2776; SSE2-NEXT:    cmovoq %r9, %rsi
2777; SSE2-NEXT:    movq %rax, %xmm1
2778; SSE2-NEXT:    movq %r8, %rax
2779; SSE2-NEXT:    mulq %r10
2780; SSE2-NEXT:    movq %rax, %xmm0
2781; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2782; SSE2-NEXT:    movq %rsi, %xmm0
2783; SSE2-NEXT:    cmovoq %r9, %rcx
2784; SSE2-NEXT:    movq %rcx, %xmm2
2785; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2786; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2787; SSE2-NEXT:    movdqa %xmm1, (%rdi)
2788; SSE2-NEXT:    retq
2789;
2790; SSSE3-LABEL: umulo_v2i64:
2791; SSSE3:       # %bb.0:
2792; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2793; SSSE3-NEXT:    movq %xmm2, %r8
2794; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2795; SSSE3-NEXT:    movq %xmm2, %r10
2796; SSSE3-NEXT:    movq %xmm0, %rax
2797; SSSE3-NEXT:    movq %xmm1, %rdx
2798; SSSE3-NEXT:    xorl %ecx, %ecx
2799; SSSE3-NEXT:    mulq %rdx
2800; SSSE3-NEXT:    movq $-1, %r9
2801; SSSE3-NEXT:    movl $0, %esi
2802; SSSE3-NEXT:    cmovoq %r9, %rsi
2803; SSSE3-NEXT:    movq %rax, %xmm1
2804; SSSE3-NEXT:    movq %r8, %rax
2805; SSSE3-NEXT:    mulq %r10
2806; SSSE3-NEXT:    movq %rax, %xmm0
2807; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2808; SSSE3-NEXT:    movq %rsi, %xmm0
2809; SSSE3-NEXT:    cmovoq %r9, %rcx
2810; SSSE3-NEXT:    movq %rcx, %xmm2
2811; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2812; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2813; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
2814; SSSE3-NEXT:    retq
2815;
2816; SSE41-LABEL: umulo_v2i64:
2817; SSE41:       # %bb.0:
2818; SSE41-NEXT:    movq %xmm0, %r10
2819; SSE41-NEXT:    movq %xmm1, %r8
2820; SSE41-NEXT:    pextrq $1, %xmm0, %rax
2821; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
2822; SSE41-NEXT:    xorl %esi, %esi
2823; SSE41-NEXT:    mulq %rdx
2824; SSE41-NEXT:    movq $-1, %r9
2825; SSE41-NEXT:    movl $0, %ecx
2826; SSE41-NEXT:    cmovoq %r9, %rcx
2827; SSE41-NEXT:    movq %rax, %xmm0
2828; SSE41-NEXT:    movq %r10, %rax
2829; SSE41-NEXT:    mulq %r8
2830; SSE41-NEXT:    movq %rax, %xmm1
2831; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2832; SSE41-NEXT:    movq %rcx, %xmm0
2833; SSE41-NEXT:    cmovoq %r9, %rsi
2834; SSE41-NEXT:    movq %rsi, %xmm2
2835; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
2836; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2837; SSE41-NEXT:    movdqa %xmm1, (%rdi)
2838; SSE41-NEXT:    retq
2839;
2840; AVX-LABEL: umulo_v2i64:
2841; AVX:       # %bb.0:
2842; AVX-NEXT:    vmovq %xmm0, %r10
2843; AVX-NEXT:    vmovq %xmm1, %r8
2844; AVX-NEXT:    vpextrq $1, %xmm0, %rax
2845; AVX-NEXT:    vpextrq $1, %xmm1, %rdx
2846; AVX-NEXT:    xorl %esi, %esi
2847; AVX-NEXT:    mulq %rdx
2848; AVX-NEXT:    movq $-1, %r9
2849; AVX-NEXT:    movl $0, %ecx
2850; AVX-NEXT:    cmovoq %r9, %rcx
2851; AVX-NEXT:    vmovq %rax, %xmm0
2852; AVX-NEXT:    movq %r10, %rax
2853; AVX-NEXT:    mulq %r8
2854; AVX-NEXT:    vmovq %rax, %xmm1
2855; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2856; AVX-NEXT:    vmovq %rcx, %xmm0
2857; AVX-NEXT:    cmovoq %r9, %rsi
2858; AVX-NEXT:    vmovq %rsi, %xmm2
2859; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
2860; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2861; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
2862; AVX-NEXT:    retq
2863;
2864; AVX512F-LABEL: umulo_v2i64:
2865; AVX512F:       # %bb.0:
2866; AVX512F-NEXT:    vmovq %xmm0, %rcx
2867; AVX512F-NEXT:    vmovq %xmm1, %rsi
2868; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
2869; AVX512F-NEXT:    vpextrq $1, %xmm1, %rdx
2870; AVX512F-NEXT:    mulq %rdx
2871; AVX512F-NEXT:    seto %r8b
2872; AVX512F-NEXT:    vmovq %rax, %xmm0
2873; AVX512F-NEXT:    movq %rcx, %rax
2874; AVX512F-NEXT:    mulq %rsi
2875; AVX512F-NEXT:    vmovq %rax, %xmm1
2876; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2877; AVX512F-NEXT:    seto %al
2878; AVX512F-NEXT:    movw $-3, %cx
2879; AVX512F-NEXT:    kmovw %ecx, %k0
2880; AVX512F-NEXT:    kmovw %eax, %k1
2881; AVX512F-NEXT:    kandw %k0, %k1, %k0
2882; AVX512F-NEXT:    kmovw %r8d, %k1
2883; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
2884; AVX512F-NEXT:    kshiftrw $14, %k1, %k1
2885; AVX512F-NEXT:    korw %k1, %k0, %k1
2886; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2887; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2888; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
2889; AVX512F-NEXT:    retq
2890;
2891; AVX512BW-LABEL: umulo_v2i64:
2892; AVX512BW:       # %bb.0:
2893; AVX512BW-NEXT:    vmovq %xmm0, %rcx
2894; AVX512BW-NEXT:    vmovq %xmm1, %rsi
2895; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
2896; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rdx
2897; AVX512BW-NEXT:    mulq %rdx
2898; AVX512BW-NEXT:    seto %r8b
2899; AVX512BW-NEXT:    vmovq %rax, %xmm0
2900; AVX512BW-NEXT:    movq %rcx, %rax
2901; AVX512BW-NEXT:    mulq %rsi
2902; AVX512BW-NEXT:    vmovq %rax, %xmm1
2903; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2904; AVX512BW-NEXT:    seto %al
2905; AVX512BW-NEXT:    movw $-3, %cx
2906; AVX512BW-NEXT:    kmovd %ecx, %k0
2907; AVX512BW-NEXT:    kmovd %eax, %k1
2908; AVX512BW-NEXT:    kandw %k0, %k1, %k0
2909; AVX512BW-NEXT:    kmovd %r8d, %k1
2910; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
2911; AVX512BW-NEXT:    kshiftrw $14, %k1, %k1
2912; AVX512BW-NEXT:    korw %k1, %k0, %k1
2913; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2914; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2915; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
2916; AVX512BW-NEXT:    retq
2917  %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
2918  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
2919  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
2920  %res = sext <2 x i1> %obit to <2 x i32>
2921  store <2 x i64> %val, <2 x i64>* %p2
2922  ret <2 x i32> %res
2923}
2924
2925define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
2926; SSE2-LABEL: umulo_v4i24:
2927; SSE2:       # %bb.0:
2928; SSE2-NEXT:    movdqa %xmm0, %xmm2
2929; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
2930; SSE2-NEXT:    pand %xmm0, %xmm1
2931; SSE2-NEXT:    pand %xmm0, %xmm2
2932; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
2933; SSE2-NEXT:    pmuludq %xmm1, %xmm2
2934; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
2935; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2936; SSE2-NEXT:    pmuludq %xmm0, %xmm1
2937; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
2938; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
2939; SSE2-NEXT:    pxor %xmm4, %xmm4
2940; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
2941; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
2942; SSE2-NEXT:    pxor %xmm5, %xmm3
2943; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2944; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
2945; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
2946; SSE2-NEXT:    psrld $24, %xmm0
2947; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
2948; SSE2-NEXT:    pxor %xmm5, %xmm0
2949; SSE2-NEXT:    por %xmm3, %xmm0
2950; SSE2-NEXT:    movd %xmm2, %eax
2951; SSE2-NEXT:    movw %ax, (%rdi)
2952; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2953; SSE2-NEXT:    movd %xmm2, %ecx
2954; SSE2-NEXT:    movw %cx, 6(%rdi)
2955; SSE2-NEXT:    movd %xmm1, %edx
2956; SSE2-NEXT:    movw %dx, 3(%rdi)
2957; SSE2-NEXT:    shrl $16, %eax
2958; SSE2-NEXT:    movb %al, 2(%rdi)
2959; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
2960; SSE2-NEXT:    movd %xmm1, %eax
2961; SSE2-NEXT:    movw %ax, 9(%rdi)
2962; SSE2-NEXT:    shrl $16, %ecx
2963; SSE2-NEXT:    movb %cl, 8(%rdi)
2964; SSE2-NEXT:    shrl $16, %edx
2965; SSE2-NEXT:    movb %dl, 5(%rdi)
2966; SSE2-NEXT:    shrl $16, %eax
2967; SSE2-NEXT:    movb %al, 11(%rdi)
2968; SSE2-NEXT:    retq
2969;
2970; SSSE3-LABEL: umulo_v4i24:
2971; SSSE3:       # %bb.0:
2972; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2973; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
2974; SSSE3-NEXT:    pand %xmm0, %xmm1
2975; SSSE3-NEXT:    pand %xmm0, %xmm2
2976; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
2977; SSSE3-NEXT:    pmuludq %xmm1, %xmm2
2978; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
2979; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2980; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
2981; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
2982; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
2983; SSSE3-NEXT:    pxor %xmm4, %xmm4
2984; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
2985; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
2986; SSSE3-NEXT:    pxor %xmm5, %xmm3
2987; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2988; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
2989; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
2990; SSSE3-NEXT:    psrld $24, %xmm0
2991; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
2992; SSSE3-NEXT:    pxor %xmm5, %xmm0
2993; SSSE3-NEXT:    por %xmm3, %xmm0
2994; SSSE3-NEXT:    movd %xmm2, %eax
2995; SSSE3-NEXT:    movw %ax, (%rdi)
2996; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2997; SSSE3-NEXT:    movd %xmm2, %ecx
2998; SSSE3-NEXT:    movw %cx, 6(%rdi)
2999; SSSE3-NEXT:    movd %xmm1, %edx
3000; SSSE3-NEXT:    movw %dx, 3(%rdi)
3001; SSSE3-NEXT:    shrl $16, %eax
3002; SSSE3-NEXT:    movb %al, 2(%rdi)
3003; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
3004; SSSE3-NEXT:    movd %xmm1, %eax
3005; SSSE3-NEXT:    movw %ax, 9(%rdi)
3006; SSSE3-NEXT:    shrl $16, %ecx
3007; SSSE3-NEXT:    movb %cl, 8(%rdi)
3008; SSSE3-NEXT:    shrl $16, %edx
3009; SSSE3-NEXT:    movb %dl, 5(%rdi)
3010; SSSE3-NEXT:    shrl $16, %eax
3011; SSSE3-NEXT:    movb %al, 11(%rdi)
3012; SSSE3-NEXT:    retq
3013;
3014; SSE41-LABEL: umulo_v4i24:
3015; SSE41:       # %bb.0:
3016; SSE41-NEXT:    movdqa %xmm0, %xmm2
3017; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
3018; SSE41-NEXT:    pand %xmm0, %xmm2
3019; SSE41-NEXT:    pand %xmm0, %xmm1
3020; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
3021; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
3022; SSE41-NEXT:    pmuludq %xmm0, %xmm3
3023; SSE41-NEXT:    movdqa %xmm2, %xmm0
3024; SSE41-NEXT:    pmuludq %xmm1, %xmm0
3025; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3026; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
3027; SSE41-NEXT:    pxor %xmm0, %xmm0
3028; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
3029; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
3030; SSE41-NEXT:    pxor %xmm3, %xmm4
3031; SSE41-NEXT:    pmulld %xmm2, %xmm1
3032; SSE41-NEXT:    pextrd $3, %xmm1, %eax
3033; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
3034; SSE41-NEXT:    pextrd $1, %xmm1, %edx
3035; SSE41-NEXT:    movd %xmm1, %esi
3036; SSE41-NEXT:    psrld $24, %xmm1
3037; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
3038; SSE41-NEXT:    pxor %xmm3, %xmm0
3039; SSE41-NEXT:    por %xmm4, %xmm0
3040; SSE41-NEXT:    movw %ax, 9(%rdi)
3041; SSE41-NEXT:    movw %cx, 6(%rdi)
3042; SSE41-NEXT:    movw %dx, 3(%rdi)
3043; SSE41-NEXT:    movw %si, (%rdi)
3044; SSE41-NEXT:    shrl $16, %eax
3045; SSE41-NEXT:    movb %al, 11(%rdi)
3046; SSE41-NEXT:    shrl $16, %ecx
3047; SSE41-NEXT:    movb %cl, 8(%rdi)
3048; SSE41-NEXT:    shrl $16, %edx
3049; SSE41-NEXT:    movb %dl, 5(%rdi)
3050; SSE41-NEXT:    shrl $16, %esi
3051; SSE41-NEXT:    movb %sil, 2(%rdi)
3052; SSE41-NEXT:    retq
3053;
3054; AVX1-LABEL: umulo_v4i24:
3055; AVX1:       # %bb.0:
3056; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3057; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
3058; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
3059; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[1,1,3,3]
3060; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[1,1,3,3]
3061; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
3062; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
3063; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3064; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3065; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
3066; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
3067; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
3068; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
3069; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3070; AVX1-NEXT:    vpsrld $24, %xmm1, %xmm0
3071; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
3072; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
3073; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
3074; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
3075; AVX1-NEXT:    movw %ax, 9(%rdi)
3076; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
3077; AVX1-NEXT:    movw %cx, 6(%rdi)
3078; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
3079; AVX1-NEXT:    movw %dx, 3(%rdi)
3080; AVX1-NEXT:    vmovd %xmm1, %esi
3081; AVX1-NEXT:    movw %si, (%rdi)
3082; AVX1-NEXT:    shrl $16, %eax
3083; AVX1-NEXT:    movb %al, 11(%rdi)
3084; AVX1-NEXT:    shrl $16, %ecx
3085; AVX1-NEXT:    movb %cl, 8(%rdi)
3086; AVX1-NEXT:    shrl $16, %edx
3087; AVX1-NEXT:    movb %dl, 5(%rdi)
3088; AVX1-NEXT:    shrl $16, %esi
3089; AVX1-NEXT:    movb %sil, 2(%rdi)
3090; AVX1-NEXT:    retq
3091;
3092; AVX2-LABEL: umulo_v4i24:
3093; AVX2:       # %bb.0:
3094; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3095; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
3096; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
3097; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3098; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3099; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
3100; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
3101; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3102; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3103; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
3104; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
3105; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
3106; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
3107; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3108; AVX2-NEXT:    vpsrld $24, %xmm1, %xmm0
3109; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
3110; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
3111; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
3112; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
3113; AVX2-NEXT:    movw %ax, 9(%rdi)
3114; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
3115; AVX2-NEXT:    movw %cx, 6(%rdi)
3116; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
3117; AVX2-NEXT:    movw %dx, 3(%rdi)
3118; AVX2-NEXT:    vmovd %xmm1, %esi
3119; AVX2-NEXT:    movw %si, (%rdi)
3120; AVX2-NEXT:    shrl $16, %eax
3121; AVX2-NEXT:    movb %al, 11(%rdi)
3122; AVX2-NEXT:    shrl $16, %ecx
3123; AVX2-NEXT:    movb %cl, 8(%rdi)
3124; AVX2-NEXT:    shrl $16, %edx
3125; AVX2-NEXT:    movb %dl, 5(%rdi)
3126; AVX2-NEXT:    shrl $16, %esi
3127; AVX2-NEXT:    movb %sil, 2(%rdi)
3128; AVX2-NEXT:    retq
3129;
3130; AVX512-LABEL: umulo_v4i24:
3131; AVX512:       # %bb.0:
3132; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3133; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
3134; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
3135; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
3136; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
3137; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3138; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
3139; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
3140; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
3141; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3142; AVX512-NEXT:    vpsrld $24, %xmm1, %xmm0
3143; AVX512-NEXT:    vpor %xmm4, %xmm0, %xmm0
3144; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
3145; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3146; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3147; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
3148; AVX512-NEXT:    movw %ax, 9(%rdi)
3149; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
3150; AVX512-NEXT:    movw %cx, 6(%rdi)
3151; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
3152; AVX512-NEXT:    movw %dx, 3(%rdi)
3153; AVX512-NEXT:    vmovd %xmm1, %esi
3154; AVX512-NEXT:    movw %si, (%rdi)
3155; AVX512-NEXT:    shrl $16, %eax
3156; AVX512-NEXT:    movb %al, 11(%rdi)
3157; AVX512-NEXT:    shrl $16, %ecx
3158; AVX512-NEXT:    movb %cl, 8(%rdi)
3159; AVX512-NEXT:    shrl $16, %edx
3160; AVX512-NEXT:    movb %dl, 5(%rdi)
3161; AVX512-NEXT:    shrl $16, %esi
3162; AVX512-NEXT:    movb %sil, 2(%rdi)
3163; AVX512-NEXT:    retq
3164  %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
3165  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
3166  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
3167  %res = sext <4 x i1> %obit to <4 x i32>
3168  store <4 x i24> %val, <4 x i24>* %p2
3169  ret <4 x i32> %res
3170}
3171
3172define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
3173; SSE-LABEL: umulo_v4i1:
3174; SSE:       # %bb.0:
3175; SSE-NEXT:    pand %xmm1, %xmm0
3176; SSE-NEXT:    pslld $31, %xmm0
3177; SSE-NEXT:    movmskps %xmm0, %eax
3178; SSE-NEXT:    movb %al, (%rdi)
3179; SSE-NEXT:    xorps %xmm0, %xmm0
3180; SSE-NEXT:    retq
3181;
3182; AVX-LABEL: umulo_v4i1:
3183; AVX:       # %bb.0:
3184; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
3185; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
3186; AVX-NEXT:    vmovmskps %xmm0, %eax
3187; AVX-NEXT:    movb %al, (%rdi)
3188; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
3189; AVX-NEXT:    retq
3190;
3191; AVX512F-LABEL: umulo_v4i1:
3192; AVX512F:       # %bb.0:
3193; AVX512F-NEXT:    vpand %xmm1, %xmm0, %xmm0
3194; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
3195; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k0
3196; AVX512F-NEXT:    kmovw %k0, %eax
3197; AVX512F-NEXT:    movb %al, (%rdi)
3198; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
3199; AVX512F-NEXT:    retq
3200;
3201; AVX512BW-LABEL: umulo_v4i1:
3202; AVX512BW:       # %bb.0:
3203; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
3204; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
3205; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k0
3206; AVX512BW-NEXT:    kmovd %k0, %eax
3207; AVX512BW-NEXT:    movb %al, (%rdi)
3208; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
3209; AVX512BW-NEXT:    retq
3210  %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
3211  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
3212  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
3213  %res = sext <4 x i1> %obit to <4 x i32>
3214  store <4 x i1> %val, <4 x i1>* %p2
3215  ret <4 x i32> %res
3216}
3217
3218define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
3219; SSE2-LABEL: umulo_v2i128:
3220; SSE2:       # %bb.0:
3221; SSE2-NEXT:    pushq %rbp
3222; SSE2-NEXT:    pushq %r15
3223; SSE2-NEXT:    pushq %r14
3224; SSE2-NEXT:    pushq %r13
3225; SSE2-NEXT:    pushq %r12
3226; SSE2-NEXT:    pushq %rbx
3227; SSE2-NEXT:    movq %r9, %r10
3228; SSE2-NEXT:    movq %rcx, %r12
3229; SSE2-NEXT:    movq %rdx, %r11
3230; SSE2-NEXT:    movq %rsi, %rax
3231; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3232; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3233; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3234; SSE2-NEXT:    testq %r10, %r10
3235; SSE2-NEXT:    setne %cl
3236; SSE2-NEXT:    testq %rsi, %rsi
3237; SSE2-NEXT:    setne %r13b
3238; SSE2-NEXT:    andb %cl, %r13b
3239; SSE2-NEXT:    mulq %r8
3240; SSE2-NEXT:    movq %rax, %rsi
3241; SSE2-NEXT:    seto %bpl
3242; SSE2-NEXT:    movq %r10, %rax
3243; SSE2-NEXT:    mulq %rdi
3244; SSE2-NEXT:    movq %rax, %rcx
3245; SSE2-NEXT:    seto %bl
3246; SSE2-NEXT:    orb %bpl, %bl
3247; SSE2-NEXT:    addq %rsi, %rcx
3248; SSE2-NEXT:    movq %rdi, %rax
3249; SSE2-NEXT:    mulq %r8
3250; SSE2-NEXT:    movq %rax, %rdi
3251; SSE2-NEXT:    movq %rdx, %rsi
3252; SSE2-NEXT:    addq %rcx, %rsi
3253; SSE2-NEXT:    setb %cl
3254; SSE2-NEXT:    orb %bl, %cl
3255; SSE2-NEXT:    orb %r13b, %cl
3256; SSE2-NEXT:    testq %r9, %r9
3257; SSE2-NEXT:    setne %al
3258; SSE2-NEXT:    testq %r12, %r12
3259; SSE2-NEXT:    setne %r8b
3260; SSE2-NEXT:    andb %al, %r8b
3261; SSE2-NEXT:    movq %r12, %rax
3262; SSE2-NEXT:    mulq %r15
3263; SSE2-NEXT:    movq %rax, %rbp
3264; SSE2-NEXT:    seto %r10b
3265; SSE2-NEXT:    movq %r9, %rax
3266; SSE2-NEXT:    mulq %r11
3267; SSE2-NEXT:    movq %rax, %rbx
3268; SSE2-NEXT:    seto %r9b
3269; SSE2-NEXT:    orb %r10b, %r9b
3270; SSE2-NEXT:    addq %rbp, %rbx
3271; SSE2-NEXT:    movq %r11, %rax
3272; SSE2-NEXT:    mulq %r15
3273; SSE2-NEXT:    addq %rbx, %rdx
3274; SSE2-NEXT:    setb %bl
3275; SSE2-NEXT:    orb %r9b, %bl
3276; SSE2-NEXT:    orb %r8b, %bl
3277; SSE2-NEXT:    movzbl %bl, %ebp
3278; SSE2-NEXT:    negl %ebp
3279; SSE2-NEXT:    movd %ebp, %xmm1
3280; SSE2-NEXT:    movzbl %cl, %ecx
3281; SSE2-NEXT:    negl %ecx
3282; SSE2-NEXT:    movd %ecx, %xmm0
3283; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3284; SSE2-NEXT:    movq %rax, 16(%r14)
3285; SSE2-NEXT:    movq %rdi, (%r14)
3286; SSE2-NEXT:    movq %rdx, 24(%r14)
3287; SSE2-NEXT:    movq %rsi, 8(%r14)
3288; SSE2-NEXT:    popq %rbx
3289; SSE2-NEXT:    popq %r12
3290; SSE2-NEXT:    popq %r13
3291; SSE2-NEXT:    popq %r14
3292; SSE2-NEXT:    popq %r15
3293; SSE2-NEXT:    popq %rbp
3294; SSE2-NEXT:    retq
3295;
3296; SSSE3-LABEL: umulo_v2i128:
3297; SSSE3:       # %bb.0:
3298; SSSE3-NEXT:    pushq %rbp
3299; SSSE3-NEXT:    pushq %r15
3300; SSSE3-NEXT:    pushq %r14
3301; SSSE3-NEXT:    pushq %r13
3302; SSSE3-NEXT:    pushq %r12
3303; SSSE3-NEXT:    pushq %rbx
3304; SSSE3-NEXT:    movq %r9, %r10
3305; SSSE3-NEXT:    movq %rcx, %r12
3306; SSSE3-NEXT:    movq %rdx, %r11
3307; SSSE3-NEXT:    movq %rsi, %rax
3308; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3309; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3310; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3311; SSSE3-NEXT:    testq %r10, %r10
3312; SSSE3-NEXT:    setne %cl
3313; SSSE3-NEXT:    testq %rsi, %rsi
3314; SSSE3-NEXT:    setne %r13b
3315; SSSE3-NEXT:    andb %cl, %r13b
3316; SSSE3-NEXT:    mulq %r8
3317; SSSE3-NEXT:    movq %rax, %rsi
3318; SSSE3-NEXT:    seto %bpl
3319; SSSE3-NEXT:    movq %r10, %rax
3320; SSSE3-NEXT:    mulq %rdi
3321; SSSE3-NEXT:    movq %rax, %rcx
3322; SSSE3-NEXT:    seto %bl
3323; SSSE3-NEXT:    orb %bpl, %bl
3324; SSSE3-NEXT:    addq %rsi, %rcx
3325; SSSE3-NEXT:    movq %rdi, %rax
3326; SSSE3-NEXT:    mulq %r8
3327; SSSE3-NEXT:    movq %rax, %rdi
3328; SSSE3-NEXT:    movq %rdx, %rsi
3329; SSSE3-NEXT:    addq %rcx, %rsi
3330; SSSE3-NEXT:    setb %cl
3331; SSSE3-NEXT:    orb %bl, %cl
3332; SSSE3-NEXT:    orb %r13b, %cl
3333; SSSE3-NEXT:    testq %r9, %r9
3334; SSSE3-NEXT:    setne %al
3335; SSSE3-NEXT:    testq %r12, %r12
3336; SSSE3-NEXT:    setne %r8b
3337; SSSE3-NEXT:    andb %al, %r8b
3338; SSSE3-NEXT:    movq %r12, %rax
3339; SSSE3-NEXT:    mulq %r15
3340; SSSE3-NEXT:    movq %rax, %rbp
3341; SSSE3-NEXT:    seto %r10b
3342; SSSE3-NEXT:    movq %r9, %rax
3343; SSSE3-NEXT:    mulq %r11
3344; SSSE3-NEXT:    movq %rax, %rbx
3345; SSSE3-NEXT:    seto %r9b
3346; SSSE3-NEXT:    orb %r10b, %r9b
3347; SSSE3-NEXT:    addq %rbp, %rbx
3348; SSSE3-NEXT:    movq %r11, %rax
3349; SSSE3-NEXT:    mulq %r15
3350; SSSE3-NEXT:    addq %rbx, %rdx
3351; SSSE3-NEXT:    setb %bl
3352; SSSE3-NEXT:    orb %r9b, %bl
3353; SSSE3-NEXT:    orb %r8b, %bl
3354; SSSE3-NEXT:    movzbl %bl, %ebp
3355; SSSE3-NEXT:    negl %ebp
3356; SSSE3-NEXT:    movd %ebp, %xmm1
3357; SSSE3-NEXT:    movzbl %cl, %ecx
3358; SSSE3-NEXT:    negl %ecx
3359; SSSE3-NEXT:    movd %ecx, %xmm0
3360; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3361; SSSE3-NEXT:    movq %rax, 16(%r14)
3362; SSSE3-NEXT:    movq %rdi, (%r14)
3363; SSSE3-NEXT:    movq %rdx, 24(%r14)
3364; SSSE3-NEXT:    movq %rsi, 8(%r14)
3365; SSSE3-NEXT:    popq %rbx
3366; SSSE3-NEXT:    popq %r12
3367; SSSE3-NEXT:    popq %r13
3368; SSSE3-NEXT:    popq %r14
3369; SSSE3-NEXT:    popq %r15
3370; SSSE3-NEXT:    popq %rbp
3371; SSSE3-NEXT:    retq
3372;
3373; SSE41-LABEL: umulo_v2i128:
3374; SSE41:       # %bb.0:
3375; SSE41-NEXT:    pushq %rbp
3376; SSE41-NEXT:    pushq %r15
3377; SSE41-NEXT:    pushq %r14
3378; SSE41-NEXT:    pushq %r13
3379; SSE41-NEXT:    pushq %r12
3380; SSE41-NEXT:    pushq %rbx
3381; SSE41-NEXT:    movq %r9, %r10
3382; SSE41-NEXT:    movq %rcx, %r12
3383; SSE41-NEXT:    movq %rdx, %r11
3384; SSE41-NEXT:    movq %rsi, %rax
3385; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3386; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3387; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3388; SSE41-NEXT:    testq %r10, %r10
3389; SSE41-NEXT:    setne %cl
3390; SSE41-NEXT:    testq %rsi, %rsi
3391; SSE41-NEXT:    setne %r13b
3392; SSE41-NEXT:    andb %cl, %r13b
3393; SSE41-NEXT:    mulq %r8
3394; SSE41-NEXT:    movq %rax, %rsi
3395; SSE41-NEXT:    seto %bpl
3396; SSE41-NEXT:    movq %r10, %rax
3397; SSE41-NEXT:    mulq %rdi
3398; SSE41-NEXT:    movq %rax, %rcx
3399; SSE41-NEXT:    seto %bl
3400; SSE41-NEXT:    orb %bpl, %bl
3401; SSE41-NEXT:    addq %rsi, %rcx
3402; SSE41-NEXT:    movq %rdi, %rax
3403; SSE41-NEXT:    mulq %r8
3404; SSE41-NEXT:    movq %rax, %rdi
3405; SSE41-NEXT:    movq %rdx, %rsi
3406; SSE41-NEXT:    addq %rcx, %rsi
3407; SSE41-NEXT:    setb %cl
3408; SSE41-NEXT:    orb %bl, %cl
3409; SSE41-NEXT:    orb %r13b, %cl
3410; SSE41-NEXT:    testq %r9, %r9
3411; SSE41-NEXT:    setne %al
3412; SSE41-NEXT:    testq %r12, %r12
3413; SSE41-NEXT:    setne %r8b
3414; SSE41-NEXT:    andb %al, %r8b
3415; SSE41-NEXT:    movq %r12, %rax
3416; SSE41-NEXT:    mulq %r15
3417; SSE41-NEXT:    movq %rax, %rbp
3418; SSE41-NEXT:    seto %r10b
3419; SSE41-NEXT:    movq %r9, %rax
3420; SSE41-NEXT:    mulq %r11
3421; SSE41-NEXT:    movq %rax, %rbx
3422; SSE41-NEXT:    seto %r9b
3423; SSE41-NEXT:    orb %r10b, %r9b
3424; SSE41-NEXT:    addq %rbp, %rbx
3425; SSE41-NEXT:    movq %r11, %rax
3426; SSE41-NEXT:    mulq %r15
3427; SSE41-NEXT:    addq %rbx, %rdx
3428; SSE41-NEXT:    setb %bl
3429; SSE41-NEXT:    orb %r9b, %bl
3430; SSE41-NEXT:    orb %r8b, %bl
3431; SSE41-NEXT:    movzbl %bl, %ebp
3432; SSE41-NEXT:    negl %ebp
3433; SSE41-NEXT:    movzbl %cl, %ecx
3434; SSE41-NEXT:    negl %ecx
3435; SSE41-NEXT:    movd %ecx, %xmm0
3436; SSE41-NEXT:    pinsrd $1, %ebp, %xmm0
3437; SSE41-NEXT:    movq %rax, 16(%r14)
3438; SSE41-NEXT:    movq %rdi, (%r14)
3439; SSE41-NEXT:    movq %rdx, 24(%r14)
3440; SSE41-NEXT:    movq %rsi, 8(%r14)
3441; SSE41-NEXT:    popq %rbx
3442; SSE41-NEXT:    popq %r12
3443; SSE41-NEXT:    popq %r13
3444; SSE41-NEXT:    popq %r14
3445; SSE41-NEXT:    popq %r15
3446; SSE41-NEXT:    popq %rbp
3447; SSE41-NEXT:    retq
3448;
3449; AVX-LABEL: umulo_v2i128:
3450; AVX:       # %bb.0:
3451; AVX-NEXT:    pushq %rbp
3452; AVX-NEXT:    pushq %r15
3453; AVX-NEXT:    pushq %r14
3454; AVX-NEXT:    pushq %r13
3455; AVX-NEXT:    pushq %r12
3456; AVX-NEXT:    pushq %rbx
3457; AVX-NEXT:    movq %r9, %r10
3458; AVX-NEXT:    movq %rcx, %r12
3459; AVX-NEXT:    movq %rdx, %r11
3460; AVX-NEXT:    movq %rsi, %rax
3461; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3462; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3463; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3464; AVX-NEXT:    testq %r10, %r10
3465; AVX-NEXT:    setne %cl
3466; AVX-NEXT:    testq %rsi, %rsi
3467; AVX-NEXT:    setne %r13b
3468; AVX-NEXT:    andb %cl, %r13b
3469; AVX-NEXT:    mulq %r8
3470; AVX-NEXT:    movq %rax, %rsi
3471; AVX-NEXT:    seto %bpl
3472; AVX-NEXT:    movq %r10, %rax
3473; AVX-NEXT:    mulq %rdi
3474; AVX-NEXT:    movq %rax, %rcx
3475; AVX-NEXT:    seto %bl
3476; AVX-NEXT:    orb %bpl, %bl
3477; AVX-NEXT:    addq %rsi, %rcx
3478; AVX-NEXT:    movq %rdi, %rax
3479; AVX-NEXT:    mulq %r8
3480; AVX-NEXT:    movq %rax, %rdi
3481; AVX-NEXT:    movq %rdx, %rsi
3482; AVX-NEXT:    addq %rcx, %rsi
3483; AVX-NEXT:    setb %cl
3484; AVX-NEXT:    orb %bl, %cl
3485; AVX-NEXT:    orb %r13b, %cl
3486; AVX-NEXT:    testq %r9, %r9
3487; AVX-NEXT:    setne %al
3488; AVX-NEXT:    testq %r12, %r12
3489; AVX-NEXT:    setne %r8b
3490; AVX-NEXT:    andb %al, %r8b
3491; AVX-NEXT:    movq %r12, %rax
3492; AVX-NEXT:    mulq %r15
3493; AVX-NEXT:    movq %rax, %rbp
3494; AVX-NEXT:    seto %r10b
3495; AVX-NEXT:    movq %r9, %rax
3496; AVX-NEXT:    mulq %r11
3497; AVX-NEXT:    movq %rax, %rbx
3498; AVX-NEXT:    seto %r9b
3499; AVX-NEXT:    orb %r10b, %r9b
3500; AVX-NEXT:    addq %rbp, %rbx
3501; AVX-NEXT:    movq %r11, %rax
3502; AVX-NEXT:    mulq %r15
3503; AVX-NEXT:    addq %rbx, %rdx
3504; AVX-NEXT:    setb %bl
3505; AVX-NEXT:    orb %r9b, %bl
3506; AVX-NEXT:    orb %r8b, %bl
3507; AVX-NEXT:    movzbl %bl, %ebp
3508; AVX-NEXT:    negl %ebp
3509; AVX-NEXT:    movzbl %cl, %ecx
3510; AVX-NEXT:    negl %ecx
3511; AVX-NEXT:    vmovd %ecx, %xmm0
3512; AVX-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
3513; AVX-NEXT:    movq %rax, 16(%r14)
3514; AVX-NEXT:    movq %rdi, (%r14)
3515; AVX-NEXT:    movq %rdx, 24(%r14)
3516; AVX-NEXT:    movq %rsi, 8(%r14)
3517; AVX-NEXT:    popq %rbx
3518; AVX-NEXT:    popq %r12
3519; AVX-NEXT:    popq %r13
3520; AVX-NEXT:    popq %r14
3521; AVX-NEXT:    popq %r15
3522; AVX-NEXT:    popq %rbp
3523; AVX-NEXT:    retq
3524;
3525; AVX512F-LABEL: umulo_v2i128:
3526; AVX512F:       # %bb.0:
3527; AVX512F-NEXT:    pushq %rbp
3528; AVX512F-NEXT:    pushq %r15
3529; AVX512F-NEXT:    pushq %r14
3530; AVX512F-NEXT:    pushq %r13
3531; AVX512F-NEXT:    pushq %r12
3532; AVX512F-NEXT:    pushq %rbx
3533; AVX512F-NEXT:    movq %rcx, %rax
3534; AVX512F-NEXT:    movq %rdx, %r12
3535; AVX512F-NEXT:    movq %rdi, %r11
3536; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3537; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3538; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
3539; AVX512F-NEXT:    testq %r10, %r10
3540; AVX512F-NEXT:    setne %dl
3541; AVX512F-NEXT:    testq %rcx, %rcx
3542; AVX512F-NEXT:    setne %r13b
3543; AVX512F-NEXT:    andb %dl, %r13b
3544; AVX512F-NEXT:    mulq %r15
3545; AVX512F-NEXT:    movq %rax, %rdi
3546; AVX512F-NEXT:    seto %bpl
3547; AVX512F-NEXT:    movq %r10, %rax
3548; AVX512F-NEXT:    mulq %r12
3549; AVX512F-NEXT:    movq %rax, %rbx
3550; AVX512F-NEXT:    seto %cl
3551; AVX512F-NEXT:    orb %bpl, %cl
3552; AVX512F-NEXT:    addq %rdi, %rbx
3553; AVX512F-NEXT:    movq %r12, %rax
3554; AVX512F-NEXT:    mulq %r15
3555; AVX512F-NEXT:    movq %rax, %r10
3556; AVX512F-NEXT:    movq %rdx, %r15
3557; AVX512F-NEXT:    addq %rbx, %r15
3558; AVX512F-NEXT:    setb %al
3559; AVX512F-NEXT:    orb %cl, %al
3560; AVX512F-NEXT:    orb %r13b, %al
3561; AVX512F-NEXT:    kmovw %eax, %k0
3562; AVX512F-NEXT:    testq %r9, %r9
3563; AVX512F-NEXT:    setne %al
3564; AVX512F-NEXT:    testq %rsi, %rsi
3565; AVX512F-NEXT:    setne %cl
3566; AVX512F-NEXT:    andb %al, %cl
3567; AVX512F-NEXT:    movq %rsi, %rax
3568; AVX512F-NEXT:    mulq %r8
3569; AVX512F-NEXT:    movq %rax, %rsi
3570; AVX512F-NEXT:    seto %bpl
3571; AVX512F-NEXT:    movq %r9, %rax
3572; AVX512F-NEXT:    mulq %r11
3573; AVX512F-NEXT:    movq %rax, %rdi
3574; AVX512F-NEXT:    seto %bl
3575; AVX512F-NEXT:    orb %bpl, %bl
3576; AVX512F-NEXT:    addq %rsi, %rdi
3577; AVX512F-NEXT:    movq %r11, %rax
3578; AVX512F-NEXT:    mulq %r8
3579; AVX512F-NEXT:    addq %rdi, %rdx
3580; AVX512F-NEXT:    setb %sil
3581; AVX512F-NEXT:    orb %bl, %sil
3582; AVX512F-NEXT:    orb %cl, %sil
3583; AVX512F-NEXT:    andl $1, %esi
3584; AVX512F-NEXT:    kmovw %esi, %k1
3585; AVX512F-NEXT:    kshiftlw $1, %k0, %k0
3586; AVX512F-NEXT:    korw %k0, %k1, %k1
3587; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3588; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3589; AVX512F-NEXT:    movq %r10, 16(%r14)
3590; AVX512F-NEXT:    movq %rax, (%r14)
3591; AVX512F-NEXT:    movq %r15, 24(%r14)
3592; AVX512F-NEXT:    movq %rdx, 8(%r14)
3593; AVX512F-NEXT:    popq %rbx
3594; AVX512F-NEXT:    popq %r12
3595; AVX512F-NEXT:    popq %r13
3596; AVX512F-NEXT:    popq %r14
3597; AVX512F-NEXT:    popq %r15
3598; AVX512F-NEXT:    popq %rbp
3599; AVX512F-NEXT:    retq
3600;
3601; AVX512BW-LABEL: umulo_v2i128:
3602; AVX512BW:       # %bb.0:
3603; AVX512BW-NEXT:    pushq %rbp
3604; AVX512BW-NEXT:    pushq %r15
3605; AVX512BW-NEXT:    pushq %r14
3606; AVX512BW-NEXT:    pushq %r13
3607; AVX512BW-NEXT:    pushq %r12
3608; AVX512BW-NEXT:    pushq %rbx
3609; AVX512BW-NEXT:    movq %rcx, %rax
3610; AVX512BW-NEXT:    movq %rdx, %r12
3611; AVX512BW-NEXT:    movq %rdi, %r11
3612; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3613; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3614; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
3615; AVX512BW-NEXT:    testq %r10, %r10
3616; AVX512BW-NEXT:    setne %dl
3617; AVX512BW-NEXT:    testq %rcx, %rcx
3618; AVX512BW-NEXT:    setne %r13b
3619; AVX512BW-NEXT:    andb %dl, %r13b
3620; AVX512BW-NEXT:    mulq %r15
3621; AVX512BW-NEXT:    movq %rax, %rdi
3622; AVX512BW-NEXT:    seto %bpl
3623; AVX512BW-NEXT:    movq %r10, %rax
3624; AVX512BW-NEXT:    mulq %r12
3625; AVX512BW-NEXT:    movq %rax, %rbx
3626; AVX512BW-NEXT:    seto %cl
3627; AVX512BW-NEXT:    orb %bpl, %cl
3628; AVX512BW-NEXT:    addq %rdi, %rbx
3629; AVX512BW-NEXT:    movq %r12, %rax
3630; AVX512BW-NEXT:    mulq %r15
3631; AVX512BW-NEXT:    movq %rax, %r10
3632; AVX512BW-NEXT:    movq %rdx, %r15
3633; AVX512BW-NEXT:    addq %rbx, %r15
3634; AVX512BW-NEXT:    setb %al
3635; AVX512BW-NEXT:    orb %cl, %al
3636; AVX512BW-NEXT:    orb %r13b, %al
3637; AVX512BW-NEXT:    kmovd %eax, %k0
3638; AVX512BW-NEXT:    testq %r9, %r9
3639; AVX512BW-NEXT:    setne %al
3640; AVX512BW-NEXT:    testq %rsi, %rsi
3641; AVX512BW-NEXT:    setne %cl
3642; AVX512BW-NEXT:    andb %al, %cl
3643; AVX512BW-NEXT:    movq %rsi, %rax
3644; AVX512BW-NEXT:    mulq %r8
3645; AVX512BW-NEXT:    movq %rax, %rsi
3646; AVX512BW-NEXT:    seto %bpl
3647; AVX512BW-NEXT:    movq %r9, %rax
3648; AVX512BW-NEXT:    mulq %r11
3649; AVX512BW-NEXT:    movq %rax, %rdi
3650; AVX512BW-NEXT:    seto %bl
3651; AVX512BW-NEXT:    orb %bpl, %bl
3652; AVX512BW-NEXT:    addq %rsi, %rdi
3653; AVX512BW-NEXT:    movq %r11, %rax
3654; AVX512BW-NEXT:    mulq %r8
3655; AVX512BW-NEXT:    addq %rdi, %rdx
3656; AVX512BW-NEXT:    setb %sil
3657; AVX512BW-NEXT:    orb %bl, %sil
3658; AVX512BW-NEXT:    orb %cl, %sil
3659; AVX512BW-NEXT:    andl $1, %esi
3660; AVX512BW-NEXT:    kmovw %esi, %k1
3661; AVX512BW-NEXT:    kshiftlw $1, %k0, %k0
3662; AVX512BW-NEXT:    korw %k0, %k1, %k1
3663; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3664; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3665; AVX512BW-NEXT:    movq %r10, 16(%r14)
3666; AVX512BW-NEXT:    movq %rax, (%r14)
3667; AVX512BW-NEXT:    movq %r15, 24(%r14)
3668; AVX512BW-NEXT:    movq %rdx, 8(%r14)
3669; AVX512BW-NEXT:    popq %rbx
3670; AVX512BW-NEXT:    popq %r12
3671; AVX512BW-NEXT:    popq %r13
3672; AVX512BW-NEXT:    popq %r14
3673; AVX512BW-NEXT:    popq %r15
3674; AVX512BW-NEXT:    popq %rbp
3675; AVX512BW-NEXT:    retq
3676  %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
3677  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
3678  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
3679  %res = sext <2 x i1> %obit to <2 x i32>
3680  store <2 x i128> %val, <2 x i128>* %p2
3681  ret <2 x i32> %res
3682}
3683