1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
9
10declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
11declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
12declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
13declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
14declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
15declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
16declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
17
18declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
19declare {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
20declare {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
21declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
22declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
23
24declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
25declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
26declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
27
28define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
29; CHECK-LABEL: umulo_v1i32:
30; CHECK:       # %bb.0:
31; CHECK-NEXT:    movq %rdx, %rcx
32; CHECK-NEXT:    movl %edi, %eax
33; CHECK-NEXT:    xorl %edi, %edi
34; CHECK-NEXT:    mull %esi
35; CHECK-NEXT:    seto %dil
36; CHECK-NEXT:    negl %edi
37; CHECK-NEXT:    movl %eax, (%rcx)
38; CHECK-NEXT:    movl %edi, %eax
39; CHECK-NEXT:    retq
40  %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
41  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
42  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
43  %res = sext <1 x i1> %obit to <1 x i32>
44  store <1 x i32> %val, <1 x i32>* %p2
45  ret <1 x i32> %res
46}
47
48define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
49; SSE2-LABEL: umulo_v2i32:
50; SSE2:       # %bb.0:
51; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
52; SSE2-NEXT:    pmuludq %xmm1, %xmm0
53; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
54; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
55; SSE2-NEXT:    pmuludq %xmm2, %xmm4
56; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
57; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
58; SSE2-NEXT:    pxor %xmm2, %xmm2
59; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
60; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
61; SSE2-NEXT:    pxor %xmm2, %xmm1
62; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
63; SSE2-NEXT:    movq %xmm0, (%rdi)
64; SSE2-NEXT:    movdqa %xmm1, %xmm0
65; SSE2-NEXT:    retq
66;
67; SSSE3-LABEL: umulo_v2i32:
68; SSSE3:       # %bb.0:
69; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
70; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
71; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
72; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
73; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
74; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
75; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
76; SSSE3-NEXT:    pxor %xmm2, %xmm2
77; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
78; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
79; SSSE3-NEXT:    pxor %xmm2, %xmm1
80; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
81; SSSE3-NEXT:    movq %xmm0, (%rdi)
82; SSSE3-NEXT:    movdqa %xmm1, %xmm0
83; SSSE3-NEXT:    retq
84;
85; SSE41-LABEL: umulo_v2i32:
86; SSE41:       # %bb.0:
87; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
88; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
89; SSE41-NEXT:    pmuludq %xmm2, %xmm3
90; SSE41-NEXT:    movdqa %xmm0, %xmm2
91; SSE41-NEXT:    pmuludq %xmm1, %xmm2
92; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
93; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
94; SSE41-NEXT:    pxor %xmm3, %xmm3
95; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
96; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
97; SSE41-NEXT:    pxor %xmm3, %xmm2
98; SSE41-NEXT:    pmulld %xmm1, %xmm0
99; SSE41-NEXT:    movq %xmm0, (%rdi)
100; SSE41-NEXT:    movdqa %xmm2, %xmm0
101; SSE41-NEXT:    retq
102;
103; AVX1-LABEL: umulo_v2i32:
104; AVX1:       # %bb.0:
105; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
106; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
107; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
108; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
109; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
110; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
111; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
112; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
113; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
114; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
115; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
116; AVX1-NEXT:    vmovq %xmm0, (%rdi)
117; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
118; AVX1-NEXT:    retq
119;
120; AVX2-LABEL: umulo_v2i32:
121; AVX2:       # %bb.0:
122; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
123; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
124; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
125; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
126; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
127; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
128; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
129; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
130; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
131; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
132; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
133; AVX2-NEXT:    vmovq %xmm0, (%rdi)
134; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
135; AVX2-NEXT:    retq
136;
137; AVX512-LABEL: umulo_v2i32:
138; AVX512:       # %bb.0:
139; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
140; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
141; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
142; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
143; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
144; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
145; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
146; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
147; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
148; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
149; AVX512-NEXT:    vmovq %xmm1, (%rdi)
150; AVX512-NEXT:    retq
151  %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
152  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
153  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
154  %res = sext <2 x i1> %obit to <2 x i32>
155  store <2 x i32> %val, <2 x i32>* %p2
156  ret <2 x i32> %res
157}
158
159define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
160; SSE2-LABEL: umulo_v3i32:
161; SSE2:       # %bb.0:
162; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
163; SSE2-NEXT:    pmuludq %xmm1, %xmm0
164; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
165; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
166; SSE2-NEXT:    pmuludq %xmm2, %xmm4
167; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
168; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
169; SSE2-NEXT:    pxor %xmm2, %xmm2
170; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
171; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
172; SSE2-NEXT:    pxor %xmm2, %xmm1
173; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
174; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
175; SSE2-NEXT:    movd %xmm2, 8(%rdi)
176; SSE2-NEXT:    movq %xmm0, (%rdi)
177; SSE2-NEXT:    movdqa %xmm1, %xmm0
178; SSE2-NEXT:    retq
179;
180; SSSE3-LABEL: umulo_v3i32:
181; SSSE3:       # %bb.0:
182; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
183; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
184; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
185; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
186; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
187; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
188; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
189; SSSE3-NEXT:    pxor %xmm2, %xmm2
190; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
191; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
192; SSSE3-NEXT:    pxor %xmm2, %xmm1
193; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
194; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
195; SSSE3-NEXT:    movd %xmm2, 8(%rdi)
196; SSSE3-NEXT:    movq %xmm0, (%rdi)
197; SSSE3-NEXT:    movdqa %xmm1, %xmm0
198; SSSE3-NEXT:    retq
199;
200; SSE41-LABEL: umulo_v3i32:
201; SSE41:       # %bb.0:
202; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
203; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
204; SSE41-NEXT:    pmuludq %xmm2, %xmm3
205; SSE41-NEXT:    movdqa %xmm0, %xmm2
206; SSE41-NEXT:    pmuludq %xmm1, %xmm2
207; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
208; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
209; SSE41-NEXT:    pxor %xmm3, %xmm3
210; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
211; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
212; SSE41-NEXT:    pxor %xmm3, %xmm2
213; SSE41-NEXT:    pmulld %xmm1, %xmm0
214; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
215; SSE41-NEXT:    movq %xmm0, (%rdi)
216; SSE41-NEXT:    movdqa %xmm2, %xmm0
217; SSE41-NEXT:    retq
218;
219; AVX1-LABEL: umulo_v3i32:
220; AVX1:       # %bb.0:
221; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
222; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
223; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
224; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
225; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
226; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
227; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
228; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
229; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
230; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
231; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
232; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
233; AVX1-NEXT:    vmovq %xmm0, (%rdi)
234; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
235; AVX1-NEXT:    retq
236;
237; AVX2-LABEL: umulo_v3i32:
238; AVX2:       # %bb.0:
239; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
240; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
241; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
242; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
243; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
244; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
245; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
246; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
247; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
248; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
249; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
250; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
251; AVX2-NEXT:    vmovq %xmm0, (%rdi)
252; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
253; AVX2-NEXT:    retq
254;
255; AVX512-LABEL: umulo_v3i32:
256; AVX512:       # %bb.0:
257; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
258; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
259; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
260; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
261; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
262; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
263; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
264; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
265; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
266; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
267; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
268; AVX512-NEXT:    vmovq %xmm1, (%rdi)
269; AVX512-NEXT:    retq
270  %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
271  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
272  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
273  %res = sext <3 x i1> %obit to <3 x i32>
274  store <3 x i32> %val, <3 x i32>* %p2
275  ret <3 x i32> %res
276}
277
278define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
279; SSE2-LABEL: umulo_v4i32:
280; SSE2:       # %bb.0:
281; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
282; SSE2-NEXT:    pmuludq %xmm1, %xmm0
283; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
284; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
285; SSE2-NEXT:    pmuludq %xmm2, %xmm4
286; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
287; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
288; SSE2-NEXT:    pxor %xmm2, %xmm2
289; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
290; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
291; SSE2-NEXT:    pxor %xmm2, %xmm1
292; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
293; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
294; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
295; SSE2-NEXT:    movdqa %xmm0, (%rdi)
296; SSE2-NEXT:    movdqa %xmm1, %xmm0
297; SSE2-NEXT:    retq
298;
299; SSSE3-LABEL: umulo_v4i32:
300; SSSE3:       # %bb.0:
301; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
302; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
303; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
304; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
305; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
306; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
307; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
308; SSSE3-NEXT:    pxor %xmm2, %xmm2
309; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
310; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
311; SSSE3-NEXT:    pxor %xmm2, %xmm1
312; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
313; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
314; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
315; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
316; SSSE3-NEXT:    movdqa %xmm1, %xmm0
317; SSSE3-NEXT:    retq
318;
319; SSE41-LABEL: umulo_v4i32:
320; SSE41:       # %bb.0:
321; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
322; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
323; SSE41-NEXT:    pmuludq %xmm2, %xmm3
324; SSE41-NEXT:    movdqa %xmm0, %xmm2
325; SSE41-NEXT:    pmuludq %xmm1, %xmm2
326; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
327; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
328; SSE41-NEXT:    pxor %xmm3, %xmm3
329; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
330; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
331; SSE41-NEXT:    pxor %xmm3, %xmm2
332; SSE41-NEXT:    pmulld %xmm1, %xmm0
333; SSE41-NEXT:    movdqa %xmm0, (%rdi)
334; SSE41-NEXT:    movdqa %xmm2, %xmm0
335; SSE41-NEXT:    retq
336;
337; AVX1-LABEL: umulo_v4i32:
338; AVX1:       # %bb.0:
339; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
340; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
341; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
342; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
343; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
344; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
345; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
346; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
347; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
348; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
349; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
350; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
351; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
352; AVX1-NEXT:    retq
353;
354; AVX2-LABEL: umulo_v4i32:
355; AVX2:       # %bb.0:
356; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
357; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
358; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
359; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
360; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
361; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
362; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
363; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
364; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
365; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
366; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
367; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
368; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
369; AVX2-NEXT:    retq
370;
371; AVX512-LABEL: umulo_v4i32:
372; AVX512:       # %bb.0:
373; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
374; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
375; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
376; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
377; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
378; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
379; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
380; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
381; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
382; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
383; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
384; AVX512-NEXT:    retq
385  %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
386  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
387  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
388  %res = sext <4 x i1> %obit to <4 x i32>
389  store <4 x i32> %val, <4 x i32>* %p2
390  ret <4 x i32> %res
391}
392
393define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
394; SSE2-LABEL: umulo_v6i32:
395; SSE2:       # %bb.0:
396; SSE2-NEXT:    movq %rdi, %rax
397; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
398; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
399; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
400; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
401; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
402; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
403; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
404; SSE2-NEXT:    movd %r8d, %xmm0
405; SSE2-NEXT:    movd %ecx, %xmm1
406; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
407; SSE2-NEXT:    movd %edx, %xmm0
408; SSE2-NEXT:    movd %esi, %xmm3
409; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
410; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
411; SSE2-NEXT:    movd %r9d, %xmm1
412; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
413; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
414; SSE2-NEXT:    pmuludq %xmm1, %xmm0
415; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
416; SSE2-NEXT:    pmuludq %xmm2, %xmm3
417; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
418; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
419; SSE2-NEXT:    pmuludq %xmm4, %xmm2
420; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
421; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
422; SSE2-NEXT:    pxor %xmm4, %xmm4
423; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
424; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
425; SSE2-NEXT:    pxor %xmm5, %xmm1
426; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
427; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
428; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
429; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
430; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
431; SSE2-NEXT:    pmuludq %xmm2, %xmm6
432; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
433; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]
434; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
435; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
436; SSE2-NEXT:    pxor %xmm5, %xmm7
437; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
438; SSE2-NEXT:    movq %xmm0, 16(%rcx)
439; SSE2-NEXT:    movdqa %xmm3, (%rcx)
440; SSE2-NEXT:    movq %xmm7, 16(%rdi)
441; SSE2-NEXT:    movdqa %xmm1, (%rdi)
442; SSE2-NEXT:    retq
443;
444; SSSE3-LABEL: umulo_v6i32:
445; SSSE3:       # %bb.0:
446; SSSE3-NEXT:    movq %rdi, %rax
447; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
448; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
449; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
450; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
451; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
452; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
453; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
454; SSSE3-NEXT:    movd %r8d, %xmm0
455; SSSE3-NEXT:    movd %ecx, %xmm1
456; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
457; SSSE3-NEXT:    movd %edx, %xmm0
458; SSSE3-NEXT:    movd %esi, %xmm3
459; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
460; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
461; SSSE3-NEXT:    movd %r9d, %xmm1
462; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
463; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
464; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
465; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
466; SSSE3-NEXT:    pmuludq %xmm2, %xmm3
467; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
468; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
469; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
470; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
471; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
472; SSSE3-NEXT:    pxor %xmm4, %xmm4
473; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
474; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
475; SSSE3-NEXT:    pxor %xmm5, %xmm1
476; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
477; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
478; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
479; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
480; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
481; SSSE3-NEXT:    pmuludq %xmm2, %xmm6
482; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
483; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]
484; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
485; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
486; SSSE3-NEXT:    pxor %xmm5, %xmm7
487; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
488; SSSE3-NEXT:    movq %xmm0, 16(%rcx)
489; SSSE3-NEXT:    movdqa %xmm3, (%rcx)
490; SSSE3-NEXT:    movq %xmm7, 16(%rdi)
491; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
492; SSSE3-NEXT:    retq
493;
494; SSE41-LABEL: umulo_v6i32:
495; SSE41:       # %bb.0:
496; SSE41-NEXT:    movq %rdi, %rax
497; SSE41-NEXT:    movd %esi, %xmm2
498; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
499; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
500; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
501; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
502; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
503; SSE41-NEXT:    movdqa %xmm1, %xmm0
504; SSE41-NEXT:    pmuludq %xmm2, %xmm1
505; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
506; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
507; SSE41-NEXT:    movd %r9d, %xmm4
508; SSE41-NEXT:    movdqa %xmm4, %xmm5
509; SSE41-NEXT:    pmuludq %xmm3, %xmm4
510; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
511; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
512; SSE41-NEXT:    pmulld %xmm3, %xmm5
513; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
514; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
515; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
516; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
517; SSE41-NEXT:    pmuludq %xmm3, %xmm6
518; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
519; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
520; SSE41-NEXT:    pxor %xmm8, %xmm8
521; SSE41-NEXT:    pcmpeqd %xmm8, %xmm1
522; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
523; SSE41-NEXT:    pxor %xmm6, %xmm1
524; SSE41-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
525; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
526; SSE41-NEXT:    pmuludq %xmm7, %xmm3
527; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
528; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
529; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
530; SSE41-NEXT:    pxor %xmm6, %xmm4
531; SSE41-NEXT:    pmulld %xmm2, %xmm0
532; SSE41-NEXT:    movq %xmm5, 16(%rcx)
533; SSE41-NEXT:    movdqa %xmm0, (%rcx)
534; SSE41-NEXT:    movq %xmm4, 16(%rdi)
535; SSE41-NEXT:    movdqa %xmm1, (%rdi)
536; SSE41-NEXT:    retq
537;
538; AVX1-LABEL: umulo_v6i32:
539; AVX1:       # %bb.0:
540; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
541; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
542; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
543; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
544; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
545; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
546; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
547; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
548; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
549; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
550; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
551; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
552; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
553; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
554; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
555; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
556; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
557; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
558; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
559; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
560; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
561; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
562; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
563; AVX1-NEXT:    vmovq %xmm1, 16(%rdi)
564; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
565; AVX1-NEXT:    vmovaps %ymm2, %ymm0
566; AVX1-NEXT:    retq
567;
568; AVX2-LABEL: umulo_v6i32:
569; AVX2:       # %bb.0:
570; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
571; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
572; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
573; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
574; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
575; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
576; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
577; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
578; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
579; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
580; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
581; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
582; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
583; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
584; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
585; AVX2-NEXT:    retq
586;
587; AVX512-LABEL: umulo_v6i32:
588; AVX512:       # %bb.0:
589; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
590; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
591; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
592; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
593; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
594; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
595; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
596; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
597; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
598; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
599; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
600; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
601; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
602; AVX512-NEXT:    retq
603  %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
604  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
605  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
606  %res = sext <6 x i1> %obit to <6 x i32>
607  store <6 x i32> %val, <6 x i32>* %p2
608  ret <6 x i32> %res
609}
610
611define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
612; SSE2-LABEL: umulo_v8i32:
613; SSE2:       # %bb.0:
614; SSE2-NEXT:    movdqa %xmm0, %xmm4
615; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
616; SSE2-NEXT:    pmuludq %xmm2, %xmm4
617; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
618; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
619; SSE2-NEXT:    pmuludq %xmm5, %xmm6
620; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
621; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
622; SSE2-NEXT:    pxor %xmm8, %xmm8
623; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
624; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
625; SSE2-NEXT:    pxor %xmm7, %xmm0
626; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
627; SSE2-NEXT:    pmuludq %xmm3, %xmm1
628; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
629; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
630; SSE2-NEXT:    pmuludq %xmm5, %xmm3
631; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
632; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
633; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
634; SSE2-NEXT:    pxor %xmm7, %xmm2
635; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
636; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
637; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
638; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
639; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
640; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
641; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
642; SSE2-NEXT:    movdqa %xmm4, (%rdi)
643; SSE2-NEXT:    movdqa %xmm2, %xmm1
644; SSE2-NEXT:    retq
645;
646; SSSE3-LABEL: umulo_v8i32:
647; SSSE3:       # %bb.0:
648; SSSE3-NEXT:    movdqa %xmm0, %xmm4
649; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
650; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
651; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
652; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
653; SSSE3-NEXT:    pmuludq %xmm5, %xmm6
654; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
655; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
656; SSSE3-NEXT:    pxor %xmm8, %xmm8
657; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
658; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
659; SSSE3-NEXT:    pxor %xmm7, %xmm0
660; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
661; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
662; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
663; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
664; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
665; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
666; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
667; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
668; SSSE3-NEXT:    pxor %xmm7, %xmm2
669; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
670; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
671; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
672; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
673; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
674; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
675; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
676; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
677; SSSE3-NEXT:    movdqa %xmm2, %xmm1
678; SSSE3-NEXT:    retq
679;
680; SSE41-LABEL: umulo_v8i32:
681; SSE41:       # %bb.0:
682; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
683; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
684; SSE41-NEXT:    pmuludq %xmm4, %xmm5
685; SSE41-NEXT:    movdqa %xmm0, %xmm4
686; SSE41-NEXT:    pmuludq %xmm2, %xmm4
687; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
688; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
689; SSE41-NEXT:    pxor %xmm8, %xmm8
690; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
691; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
692; SSE41-NEXT:    pxor %xmm7, %xmm4
693; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
694; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
695; SSE41-NEXT:    pmuludq %xmm5, %xmm6
696; SSE41-NEXT:    movdqa %xmm1, %xmm5
697; SSE41-NEXT:    pmuludq %xmm3, %xmm5
698; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
699; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
700; SSE41-NEXT:    pcmpeqd %xmm8, %xmm5
701; SSE41-NEXT:    pxor %xmm7, %xmm5
702; SSE41-NEXT:    pmulld %xmm2, %xmm0
703; SSE41-NEXT:    pmulld %xmm3, %xmm1
704; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
705; SSE41-NEXT:    movdqa %xmm0, (%rdi)
706; SSE41-NEXT:    movdqa %xmm4, %xmm0
707; SSE41-NEXT:    movdqa %xmm5, %xmm1
708; SSE41-NEXT:    retq
709;
710; AVX1-LABEL: umulo_v8i32:
711; AVX1:       # %bb.0:
712; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
713; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
714; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
715; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
716; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2
717; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5
718; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
719; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
720; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
721; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
722; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
723; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
724; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
725; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
726; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
727; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
728; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
729; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
730; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
731; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
732; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
733; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
734; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1
735; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)
736; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
737; AVX1-NEXT:    vmovaps %ymm2, %ymm0
738; AVX1-NEXT:    retq
739;
740; AVX2-LABEL: umulo_v8i32:
741; AVX2:       # %bb.0:
742; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
743; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]
744; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2
745; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3
746; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]
747; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]
748; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
749; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
750; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
751; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
752; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
753; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
754; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
755; AVX2-NEXT:    retq
756;
757; AVX512-LABEL: umulo_v8i32:
758; AVX512:       # %bb.0:
759; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
760; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
761; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]
762; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3
763; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]
764; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4
765; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1
766; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1
767; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
768; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
769; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
770; AVX512-NEXT:    retq
771  %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
772  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
773  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
774  %res = sext <8 x i1> %obit to <8 x i32>
775  store <8 x i32> %val, <8 x i32>* %p2
776  ret <8 x i32> %res
777}
778
779define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
780; SSE2-LABEL: umulo_v16i32:
781; SSE2:       # %bb.0:
782; SSE2-NEXT:    movdqa %xmm0, %xmm8
783; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
784; SSE2-NEXT:    pmuludq %xmm4, %xmm8
785; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
786; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
787; SSE2-NEXT:    pmuludq %xmm10, %xmm9
788; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
789; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
790; SSE2-NEXT:    pxor %xmm10, %xmm10
791; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
792; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
793; SSE2-NEXT:    pxor %xmm11, %xmm0
794; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
795; SSE2-NEXT:    pmuludq %xmm5, %xmm1
796; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
797; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
798; SSE2-NEXT:    pmuludq %xmm13, %xmm12
799; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
800; SSE2-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
801; SSE2-NEXT:    pcmpeqd %xmm10, %xmm15
802; SSE2-NEXT:    pxor %xmm11, %xmm15
803; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
804; SSE2-NEXT:    pmuludq %xmm6, %xmm2
805; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
806; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
807; SSE2-NEXT:    pmuludq %xmm14, %xmm13
808; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
809; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
810; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
811; SSE2-NEXT:    pxor %xmm11, %xmm5
812; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
813; SSE2-NEXT:    pmuludq %xmm7, %xmm3
814; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
815; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
816; SSE2-NEXT:    pmuludq %xmm14, %xmm7
817; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
818; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
819; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
820; SSE2-NEXT:    pxor %xmm11, %xmm6
821; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
822; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
823; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
824; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
825; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
826; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
827; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
828; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
829; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
830; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
831; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
832; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
833; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
834; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
835; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
836; SSE2-NEXT:    movdqa %xmm8, (%rdi)
837; SSE2-NEXT:    movdqa %xmm15, %xmm1
838; SSE2-NEXT:    movdqa %xmm5, %xmm2
839; SSE2-NEXT:    movdqa %xmm6, %xmm3
840; SSE2-NEXT:    retq
841;
842; SSSE3-LABEL: umulo_v16i32:
843; SSSE3:       # %bb.0:
844; SSSE3-NEXT:    movdqa %xmm0, %xmm8
845; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
846; SSSE3-NEXT:    pmuludq %xmm4, %xmm8
847; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,3,2,3]
848; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
849; SSSE3-NEXT:    pmuludq %xmm10, %xmm9
850; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,3,2,3]
851; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
852; SSSE3-NEXT:    pxor %xmm10, %xmm10
853; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
854; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm11
855; SSSE3-NEXT:    pxor %xmm11, %xmm0
856; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
857; SSSE3-NEXT:    pmuludq %xmm5, %xmm1
858; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[1,3,2,3]
859; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[1,1,3,3]
860; SSSE3-NEXT:    pmuludq %xmm13, %xmm12
861; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,3,2,3]
862; SSSE3-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
863; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm15
864; SSSE3-NEXT:    pxor %xmm11, %xmm15
865; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]
866; SSSE3-NEXT:    pmuludq %xmm6, %xmm2
867; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,3,2,3]
868; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
869; SSSE3-NEXT:    pmuludq %xmm14, %xmm13
870; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]
871; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
872; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm5
873; SSSE3-NEXT:    pxor %xmm11, %xmm5
874; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
875; SSSE3-NEXT:    pmuludq %xmm7, %xmm3
876; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
877; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
878; SSSE3-NEXT:    pmuludq %xmm14, %xmm7
879; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,3,2,3]
880; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
881; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm6
882; SSSE3-NEXT:    pxor %xmm11, %xmm6
883; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
884; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
885; SSSE3-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
886; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
887; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
888; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
889; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
890; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
891; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
892; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
893; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
894; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
895; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
896; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
897; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
898; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
899; SSSE3-NEXT:    movdqa %xmm15, %xmm1
900; SSSE3-NEXT:    movdqa %xmm5, %xmm2
901; SSSE3-NEXT:    movdqa %xmm6, %xmm3
902; SSSE3-NEXT:    retq
903;
904; SSE41-LABEL: umulo_v16i32:
905; SSE41:       # %bb.0:
906; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
907; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
908; SSE41-NEXT:    pmuludq %xmm8, %xmm9
909; SSE41-NEXT:    movdqa %xmm0, %xmm8
910; SSE41-NEXT:    pmuludq %xmm4, %xmm8
911; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
912; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]
913; SSE41-NEXT:    pxor %xmm12, %xmm12
914; SSE41-NEXT:    pcmpeqd %xmm12, %xmm8
915; SSE41-NEXT:    pcmpeqd %xmm13, %xmm13
916; SSE41-NEXT:    pxor %xmm13, %xmm8
917; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]
918; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
919; SSE41-NEXT:    pmuludq %xmm9, %xmm10
920; SSE41-NEXT:    movdqa %xmm1, %xmm9
921; SSE41-NEXT:    pmuludq %xmm5, %xmm9
922; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
923; SSE41-NEXT:    pblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3],xmm9[4,5],xmm10[6,7]
924; SSE41-NEXT:    pcmpeqd %xmm12, %xmm9
925; SSE41-NEXT:    pxor %xmm13, %xmm9
926; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
927; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]
928; SSE41-NEXT:    pmuludq %xmm10, %xmm11
929; SSE41-NEXT:    movdqa %xmm2, %xmm10
930; SSE41-NEXT:    pmuludq %xmm6, %xmm10
931; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
932; SSE41-NEXT:    pblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]
933; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
934; SSE41-NEXT:    pxor %xmm13, %xmm10
935; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
936; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]
937; SSE41-NEXT:    pmuludq %xmm11, %xmm14
938; SSE41-NEXT:    movdqa %xmm3, %xmm11
939; SSE41-NEXT:    pmuludq %xmm7, %xmm11
940; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
941; SSE41-NEXT:    pblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3],xmm11[4,5],xmm14[6,7]
942; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
943; SSE41-NEXT:    pxor %xmm13, %xmm11
944; SSE41-NEXT:    pmulld %xmm4, %xmm0
945; SSE41-NEXT:    pmulld %xmm5, %xmm1
946; SSE41-NEXT:    pmulld %xmm6, %xmm2
947; SSE41-NEXT:    pmulld %xmm7, %xmm3
948; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)
949; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
950; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)
951; SSE41-NEXT:    movdqa %xmm0, (%rdi)
952; SSE41-NEXT:    movdqa %xmm8, %xmm0
953; SSE41-NEXT:    movdqa %xmm9, %xmm1
954; SSE41-NEXT:    movdqa %xmm10, %xmm2
955; SSE41-NEXT:    movdqa %xmm11, %xmm3
956; SSE41-NEXT:    retq
957;
958; AVX1-LABEL: umulo_v16i32:
959; AVX1:       # %bb.0:
960; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
961; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
962; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
963; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
964; AVX1-NEXT:    vpmuludq %xmm6, %xmm7, %xmm6
965; AVX1-NEXT:    vpmuludq %xmm10, %xmm12, %xmm7
966; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
967; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
968; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
969; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm8, %xmm7
970; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
971; AVX1-NEXT:    vpxor %xmm7, %xmm9, %xmm7
972; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
973; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
974; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
975; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm6
976; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
977; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5],xmm4[6,7]
978; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
979; AVX1-NEXT:    vpxor %xmm4, %xmm9, %xmm4
980; AVX1-NEXT:    vpackssdw %xmm7, %xmm4, %xmm11
981; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
982; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
983; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
984; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
985; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
986; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm7
987; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
988; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
989; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
990; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm13
991; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
992; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
993; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
994; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm7
995; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
996; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
997; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
998; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
999; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
1000; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm5
1001; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
1002; AVX1-NEXT:    vpmulld %xmm6, %xmm4, %xmm4
1003; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
1004; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
1005; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm0
1006; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
1007; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1008; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1009; AVX1-NEXT:    vpacksswb %xmm11, %xmm11, %xmm1
1010; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
1011; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1012; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1013; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
1014; AVX1-NEXT:    vmovdqa %xmm6, 48(%rdi)
1015; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
1016; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
1017; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1018; AVX1-NEXT:    retq
1019;
1020; AVX2-LABEL: umulo_v16i32:
1021; AVX2:       # %bb.0:
1022; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]
1023; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]
1024; AVX2-NEXT:    vpmuludq %ymm4, %ymm5, %ymm4
1025; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm5
1026; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]
1027; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]
1028; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
1029; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
1030; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
1031; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
1032; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm7
1033; AVX2-NEXT:    vpackssdw %xmm7, %xmm4, %xmm4
1034; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[1,1,3,3,5,5,7,7]
1035; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm0[1,1,3,3,5,5,7,7]
1036; AVX2-NEXT:    vpmuludq %ymm7, %ymm8, %ymm7
1037; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm8
1038; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,1,3,3,5,5,7,7]
1039; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
1040; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
1041; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
1042; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1043; AVX2-NEXT:    vpackssdw %xmm6, %xmm5, %xmm5
1044; AVX2-NEXT:    vpacksswb %xmm5, %xmm5, %xmm5
1045; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm2
1046; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm3
1047; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm0
1048; AVX2-NEXT:    vpacksswb %xmm4, %xmm4, %xmm1
1049; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1050; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
1051; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
1052; AVX2-NEXT:    retq
1053;
1054; AVX512-LABEL: umulo_v16i32:
1055; AVX512:       # %bb.0:
1056; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm2
1057; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1058; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
1059; AVX512-NEXT:    vpmuludq %zmm3, %zmm4, %zmm3
1060; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
1061; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
1062; AVX512-NEXT:    vptestmd %zmm4, %zmm4, %k1
1063; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm1
1064; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1065; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
1066; AVX512-NEXT:    retq
1067  %t = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
1068  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
1069  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
1070  %res = sext <16 x i1> %obit to <16 x i32>
1071  store <16 x i32> %val, <16 x i32>* %p2
1072  ret <16 x i32> %res
1073}
1074
1075define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
1076; SSE2-LABEL: umulo_v16i8:
1077; SSE2:       # %bb.0:
1078; SSE2-NEXT:    movdqa %xmm1, %xmm2
1079; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1080; SSE2-NEXT:    movdqa %xmm0, %xmm3
1081; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1082; SSE2-NEXT:    pmullw %xmm2, %xmm3
1083; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1084; SSE2-NEXT:    pand %xmm2, %xmm3
1085; SSE2-NEXT:    movdqa %xmm1, %xmm5
1086; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1087; SSE2-NEXT:    movdqa %xmm0, %xmm4
1088; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1089; SSE2-NEXT:    pmullw %xmm5, %xmm4
1090; SSE2-NEXT:    pand %xmm2, %xmm4
1091; SSE2-NEXT:    packuswb %xmm3, %xmm4
1092; SSE2-NEXT:    pxor %xmm2, %xmm2
1093; SSE2-NEXT:    movdqa %xmm1, %xmm3
1094; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1095; SSE2-NEXT:    movdqa %xmm0, %xmm5
1096; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1097; SSE2-NEXT:    pmullw %xmm3, %xmm5
1098; SSE2-NEXT:    psrlw $8, %xmm5
1099; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1100; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1101; SSE2-NEXT:    pmullw %xmm1, %xmm0
1102; SSE2-NEXT:    psrlw $8, %xmm0
1103; SSE2-NEXT:    packuswb %xmm5, %xmm0
1104; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
1105; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1106; SSE2-NEXT:    pxor %xmm0, %xmm3
1107; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
1108; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1109; SSE2-NEXT:    psrad $24, %xmm0
1110; SSE2-NEXT:    movdqa %xmm3, %xmm1
1111; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1112; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1113; SSE2-NEXT:    pslld $31, %xmm1
1114; SSE2-NEXT:    psrad $31, %xmm1
1115; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1116; SSE2-NEXT:    movdqa %xmm3, %xmm2
1117; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1118; SSE2-NEXT:    pslld $31, %xmm2
1119; SSE2-NEXT:    psrad $31, %xmm2
1120; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1121; SSE2-NEXT:    pslld $31, %xmm3
1122; SSE2-NEXT:    psrad $31, %xmm3
1123; SSE2-NEXT:    movdqa %xmm4, (%rdi)
1124; SSE2-NEXT:    retq
1125;
1126; SSSE3-LABEL: umulo_v16i8:
1127; SSSE3:       # %bb.0:
1128; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1129; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1130; SSSE3-NEXT:    movdqa %xmm0, %xmm3
1131; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1132; SSSE3-NEXT:    pmullw %xmm2, %xmm3
1133; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1134; SSSE3-NEXT:    pand %xmm2, %xmm3
1135; SSSE3-NEXT:    movdqa %xmm1, %xmm5
1136; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1137; SSSE3-NEXT:    movdqa %xmm0, %xmm4
1138; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1139; SSSE3-NEXT:    pmullw %xmm5, %xmm4
1140; SSSE3-NEXT:    pand %xmm2, %xmm4
1141; SSSE3-NEXT:    packuswb %xmm3, %xmm4
1142; SSSE3-NEXT:    pxor %xmm2, %xmm2
1143; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1144; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
1145; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1146; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
1147; SSSE3-NEXT:    pmullw %xmm3, %xmm5
1148; SSSE3-NEXT:    psrlw $8, %xmm5
1149; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1150; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1151; SSSE3-NEXT:    pmullw %xmm1, %xmm0
1152; SSSE3-NEXT:    psrlw $8, %xmm0
1153; SSSE3-NEXT:    packuswb %xmm5, %xmm0
1154; SSSE3-NEXT:    pcmpeqb %xmm2, %xmm0
1155; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1156; SSSE3-NEXT:    pxor %xmm0, %xmm3
1157; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
1158; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1159; SSSE3-NEXT:    psrad $24, %xmm0
1160; SSSE3-NEXT:    movdqa %xmm3, %xmm1
1161; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1162; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1163; SSSE3-NEXT:    pslld $31, %xmm1
1164; SSSE3-NEXT:    psrad $31, %xmm1
1165; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1166; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1167; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1168; SSSE3-NEXT:    pslld $31, %xmm2
1169; SSSE3-NEXT:    psrad $31, %xmm2
1170; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1171; SSSE3-NEXT:    pslld $31, %xmm3
1172; SSSE3-NEXT:    psrad $31, %xmm3
1173; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
1174; SSSE3-NEXT:    retq
1175;
1176; SSE41-LABEL: umulo_v16i8:
1177; SSE41:       # %bb.0:
1178; SSE41-NEXT:    movdqa %xmm1, %xmm2
1179; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1180; SSE41-NEXT:    movdqa %xmm0, %xmm3
1181; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1182; SSE41-NEXT:    pmullw %xmm2, %xmm3
1183; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
1184; SSE41-NEXT:    pand %xmm4, %xmm3
1185; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1186; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1187; SSE41-NEXT:    pmullw %xmm2, %xmm5
1188; SSE41-NEXT:    pand %xmm5, %xmm4
1189; SSE41-NEXT:    packuswb %xmm3, %xmm4
1190; SSE41-NEXT:    pxor %xmm2, %xmm2
1191; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1192; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1193; SSE41-NEXT:    pmullw %xmm1, %xmm0
1194; SSE41-NEXT:    psrlw $8, %xmm0
1195; SSE41-NEXT:    psrlw $8, %xmm5
1196; SSE41-NEXT:    packuswb %xmm0, %xmm5
1197; SSE41-NEXT:    pcmpeqb %xmm2, %xmm5
1198; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
1199; SSE41-NEXT:    pxor %xmm5, %xmm3
1200; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
1201; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
1202; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1203; SSE41-NEXT:    pslld $31, %xmm1
1204; SSE41-NEXT:    psrad $31, %xmm1
1205; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1206; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1207; SSE41-NEXT:    pslld $31, %xmm2
1208; SSE41-NEXT:    psrad $31, %xmm2
1209; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
1210; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1211; SSE41-NEXT:    pslld $31, %xmm3
1212; SSE41-NEXT:    psrad $31, %xmm3
1213; SSE41-NEXT:    movdqa %xmm4, (%rdi)
1214; SSE41-NEXT:    retq
1215;
1216; AVX1-LABEL: umulo_v16i8:
1217; AVX1:       # %bb.0:
1218; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1219; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1220; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1221; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1222; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1223; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1224; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1225; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
1226; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
1227; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
1228; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1229; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1230; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1231; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1232; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1233; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm1
1234; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1235; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1236; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1237; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1238; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1239; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
1240; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1241; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
1242; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1243; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1244; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1245; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1246; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1247; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
1248; AVX1-NEXT:    retq
1249;
1250; AVX2-LABEL: umulo_v16i8:
1251; AVX2:       # %bb.0:
1252; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1253; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1254; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1255; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm1
1256; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
1257; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm2
1258; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1259; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1260; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
1261; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1262; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
1263; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
1264; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
1265; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1266; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1267; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1268; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
1269; AVX2-NEXT:    retq
1270;
1271; AVX512F-LABEL: umulo_v16i8:
1272; AVX512F:       # %bb.0:
1273; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1274; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1275; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1276; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm0
1277; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1278; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1279; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1280; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
1281; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1282; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1283; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
1284; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1285; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
1286; AVX512F-NEXT:    vpmovdb %zmm1, (%rdi)
1287; AVX512F-NEXT:    retq
1288;
1289; AVX512BW-LABEL: umulo_v16i8:
1290; AVX512BW:       # %bb.0:
1291; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1292; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1293; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm1
1294; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm0
1295; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
1296; AVX512BW-NEXT:    vptestmb %xmm0, %xmm0, %k1
1297; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1298; AVX512BW-NEXT:    vpmovwb %ymm1, (%rdi)
1299; AVX512BW-NEXT:    retq
1300  %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
1301  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
1302  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
1303  %res = sext <16 x i1> %obit to <16 x i32>
1304  store <16 x i8> %val, <16 x i8>* %p2
1305  ret <16 x i32> %res
1306}
1307
1308define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nounwind {
1309; SSE2-LABEL: umulo_v32i8:
1310; SSE2:       # %bb.0:
1311; SSE2-NEXT:    movq %rdi, %rax
1312; SSE2-NEXT:    movdqa %xmm2, %xmm4
1313; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1314; SSE2-NEXT:    movdqa %xmm0, %xmm5
1315; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1316; SSE2-NEXT:    pmullw %xmm4, %xmm5
1317; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1318; SSE2-NEXT:    pand %xmm8, %xmm5
1319; SSE2-NEXT:    movdqa %xmm2, %xmm7
1320; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1321; SSE2-NEXT:    movdqa %xmm0, %xmm9
1322; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1323; SSE2-NEXT:    pmullw %xmm7, %xmm9
1324; SSE2-NEXT:    pand %xmm8, %xmm9
1325; SSE2-NEXT:    packuswb %xmm5, %xmm9
1326; SSE2-NEXT:    movdqa %xmm3, %xmm5
1327; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1328; SSE2-NEXT:    movdqa %xmm1, %xmm7
1329; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1330; SSE2-NEXT:    pmullw %xmm5, %xmm7
1331; SSE2-NEXT:    pand %xmm8, %xmm7
1332; SSE2-NEXT:    movdqa %xmm3, %xmm6
1333; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1334; SSE2-NEXT:    movdqa %xmm1, %xmm11
1335; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1336; SSE2-NEXT:    pmullw %xmm6, %xmm11
1337; SSE2-NEXT:    pand %xmm8, %xmm11
1338; SSE2-NEXT:    packuswb %xmm7, %xmm11
1339; SSE2-NEXT:    pxor %xmm6, %xmm6
1340; SSE2-NEXT:    movdqa %xmm3, %xmm7
1341; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1342; SSE2-NEXT:    movdqa %xmm1, %xmm4
1343; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1344; SSE2-NEXT:    pmullw %xmm7, %xmm4
1345; SSE2-NEXT:    psrlw $8, %xmm4
1346; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
1347; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
1348; SSE2-NEXT:    pmullw %xmm3, %xmm1
1349; SSE2-NEXT:    psrlw $8, %xmm1
1350; SSE2-NEXT:    packuswb %xmm4, %xmm1
1351; SSE2-NEXT:    pcmpeqb %xmm6, %xmm1
1352; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
1353; SSE2-NEXT:    pxor %xmm3, %xmm1
1354; SSE2-NEXT:    movdqa %xmm2, %xmm4
1355; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1356; SSE2-NEXT:    movdqa %xmm0, %xmm7
1357; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1358; SSE2-NEXT:    pmullw %xmm4, %xmm7
1359; SSE2-NEXT:    psrlw $8, %xmm7
1360; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
1361; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
1362; SSE2-NEXT:    pmullw %xmm2, %xmm0
1363; SSE2-NEXT:    psrlw $8, %xmm0
1364; SSE2-NEXT:    packuswb %xmm7, %xmm0
1365; SSE2-NEXT:    pcmpeqb %xmm6, %xmm0
1366; SSE2-NEXT:    pxor %xmm3, %xmm0
1367; SSE2-NEXT:    movdqa %xmm0, %xmm2
1368; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]
1369; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1370; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1371; SSE2-NEXT:    pslld $31, %xmm0
1372; SSE2-NEXT:    psrad $31, %xmm0
1373; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1374; SSE2-NEXT:    movdqa %xmm2, %xmm4
1375; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1376; SSE2-NEXT:    pslld $31, %xmm4
1377; SSE2-NEXT:    psrad $31, %xmm4
1378; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1379; SSE2-NEXT:    pslld $31, %xmm2
1380; SSE2-NEXT:    psrad $31, %xmm2
1381; SSE2-NEXT:    movdqa %xmm1, %xmm6
1382; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1383; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
1384; SSE2-NEXT:    pslld $31, %xmm6
1385; SSE2-NEXT:    psrad $31, %xmm6
1386; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1],xmm10[2],xmm1[2],xmm10[3],xmm1[3],xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
1387; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1388; SSE2-NEXT:    movdqa %xmm1, %xmm3
1389; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1390; SSE2-NEXT:    pslld $31, %xmm3
1391; SSE2-NEXT:    psrad $31, %xmm3
1392; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1393; SSE2-NEXT:    pslld $31, %xmm1
1394; SSE2-NEXT:    psrad $31, %xmm1
1395; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1396; SSE2-NEXT:    psrad $24, %xmm7
1397; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
1398; SSE2-NEXT:    psrad $24, %xmm5
1399; SSE2-NEXT:    movdqa %xmm11, 16(%rsi)
1400; SSE2-NEXT:    movdqa %xmm9, (%rsi)
1401; SSE2-NEXT:    movdqa %xmm5, 64(%rdi)
1402; SSE2-NEXT:    movdqa %xmm7, (%rdi)
1403; SSE2-NEXT:    movdqa %xmm1, 112(%rdi)
1404; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
1405; SSE2-NEXT:    movdqa %xmm6, 80(%rdi)
1406; SSE2-NEXT:    movdqa %xmm2, 48(%rdi)
1407; SSE2-NEXT:    movdqa %xmm4, 32(%rdi)
1408; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
1409; SSE2-NEXT:    retq
1410;
1411; SSSE3-LABEL: umulo_v32i8:
1412; SSSE3:       # %bb.0:
1413; SSSE3-NEXT:    movq %rdi, %rax
1414; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1415; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1416; SSSE3-NEXT:    movdqa %xmm0, %xmm5
1417; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1418; SSSE3-NEXT:    pmullw %xmm4, %xmm5
1419; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1420; SSSE3-NEXT:    pand %xmm8, %xmm5
1421; SSSE3-NEXT:    movdqa %xmm2, %xmm7
1422; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1423; SSSE3-NEXT:    movdqa %xmm0, %xmm9
1424; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1425; SSSE3-NEXT:    pmullw %xmm7, %xmm9
1426; SSSE3-NEXT:    pand %xmm8, %xmm9
1427; SSSE3-NEXT:    packuswb %xmm5, %xmm9
1428; SSSE3-NEXT:    movdqa %xmm3, %xmm5
1429; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1430; SSSE3-NEXT:    movdqa %xmm1, %xmm7
1431; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1432; SSSE3-NEXT:    pmullw %xmm5, %xmm7
1433; SSSE3-NEXT:    pand %xmm8, %xmm7
1434; SSSE3-NEXT:    movdqa %xmm3, %xmm6
1435; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1436; SSSE3-NEXT:    movdqa %xmm1, %xmm11
1437; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1438; SSSE3-NEXT:    pmullw %xmm6, %xmm11
1439; SSSE3-NEXT:    pand %xmm8, %xmm11
1440; SSSE3-NEXT:    packuswb %xmm7, %xmm11
1441; SSSE3-NEXT:    pxor %xmm6, %xmm6
1442; SSSE3-NEXT:    movdqa %xmm3, %xmm7
1443; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1444; SSSE3-NEXT:    movdqa %xmm1, %xmm4
1445; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1446; SSSE3-NEXT:    pmullw %xmm7, %xmm4
1447; SSSE3-NEXT:    psrlw $8, %xmm4
1448; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
1449; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
1450; SSSE3-NEXT:    pmullw %xmm3, %xmm1
1451; SSSE3-NEXT:    psrlw $8, %xmm1
1452; SSSE3-NEXT:    packuswb %xmm4, %xmm1
1453; SSSE3-NEXT:    pcmpeqb %xmm6, %xmm1
1454; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
1455; SSSE3-NEXT:    pxor %xmm3, %xmm1
1456; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1457; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1458; SSSE3-NEXT:    movdqa %xmm0, %xmm7
1459; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
1460; SSSE3-NEXT:    pmullw %xmm4, %xmm7
1461; SSSE3-NEXT:    psrlw $8, %xmm7
1462; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
1463; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
1464; SSSE3-NEXT:    pmullw %xmm2, %xmm0
1465; SSSE3-NEXT:    psrlw $8, %xmm0
1466; SSSE3-NEXT:    packuswb %xmm7, %xmm0
1467; SSSE3-NEXT:    pcmpeqb %xmm6, %xmm0
1468; SSSE3-NEXT:    pxor %xmm3, %xmm0
1469; SSSE3-NEXT:    movdqa %xmm0, %xmm2
1470; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]
1471; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1472; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1473; SSSE3-NEXT:    pslld $31, %xmm0
1474; SSSE3-NEXT:    psrad $31, %xmm0
1475; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1476; SSSE3-NEXT:    movdqa %xmm2, %xmm4
1477; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1478; SSSE3-NEXT:    pslld $31, %xmm4
1479; SSSE3-NEXT:    psrad $31, %xmm4
1480; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1481; SSSE3-NEXT:    pslld $31, %xmm2
1482; SSSE3-NEXT:    psrad $31, %xmm2
1483; SSSE3-NEXT:    movdqa %xmm1, %xmm6
1484; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1485; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
1486; SSSE3-NEXT:    pslld $31, %xmm6
1487; SSSE3-NEXT:    psrad $31, %xmm6
1488; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1],xmm10[2],xmm1[2],xmm10[3],xmm1[3],xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
1489; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1490; SSSE3-NEXT:    movdqa %xmm1, %xmm3
1491; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1492; SSSE3-NEXT:    pslld $31, %xmm3
1493; SSSE3-NEXT:    psrad $31, %xmm3
1494; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1495; SSSE3-NEXT:    pslld $31, %xmm1
1496; SSSE3-NEXT:    psrad $31, %xmm1
1497; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1498; SSSE3-NEXT:    psrad $24, %xmm7
1499; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
1500; SSSE3-NEXT:    psrad $24, %xmm5
1501; SSSE3-NEXT:    movdqa %xmm11, 16(%rsi)
1502; SSSE3-NEXT:    movdqa %xmm9, (%rsi)
1503; SSSE3-NEXT:    movdqa %xmm5, 64(%rdi)
1504; SSSE3-NEXT:    movdqa %xmm7, (%rdi)
1505; SSSE3-NEXT:    movdqa %xmm1, 112(%rdi)
1506; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
1507; SSSE3-NEXT:    movdqa %xmm6, 80(%rdi)
1508; SSSE3-NEXT:    movdqa %xmm2, 48(%rdi)
1509; SSSE3-NEXT:    movdqa %xmm4, 32(%rdi)
1510; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
1511; SSSE3-NEXT:    retq
1512;
1513; SSE41-LABEL: umulo_v32i8:
1514; SSE41:       # %bb.0:
1515; SSE41-NEXT:    movq %rdi, %rax
1516; SSE41-NEXT:    movdqa %xmm2, %xmm4
1517; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1518; SSE41-NEXT:    movdqa %xmm0, %xmm7
1519; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1520; SSE41-NEXT:    pmullw %xmm4, %xmm7
1521; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
1522; SSE41-NEXT:    pand %xmm10, %xmm7
1523; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1524; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1525; SSE41-NEXT:    pmullw %xmm5, %xmm6
1526; SSE41-NEXT:    movdqa %xmm6, %xmm8
1527; SSE41-NEXT:    pand %xmm10, %xmm8
1528; SSE41-NEXT:    packuswb %xmm7, %xmm8
1529; SSE41-NEXT:    movdqa %xmm3, %xmm7
1530; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1531; SSE41-NEXT:    movdqa %xmm1, %xmm5
1532; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1533; SSE41-NEXT:    pmullw %xmm7, %xmm5
1534; SSE41-NEXT:    pand %xmm10, %xmm5
1535; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1536; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1537; SSE41-NEXT:    pmullw %xmm9, %xmm7
1538; SSE41-NEXT:    pand %xmm7, %xmm10
1539; SSE41-NEXT:    packuswb %xmm5, %xmm10
1540; SSE41-NEXT:    pxor %xmm5, %xmm5
1541; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
1542; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
1543; SSE41-NEXT:    pmullw %xmm3, %xmm1
1544; SSE41-NEXT:    psrlw $8, %xmm1
1545; SSE41-NEXT:    psrlw $8, %xmm7
1546; SSE41-NEXT:    packuswb %xmm1, %xmm7
1547; SSE41-NEXT:    pcmpeqb %xmm5, %xmm7
1548; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
1549; SSE41-NEXT:    pxor %xmm1, %xmm7
1550; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
1551; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15]
1552; SSE41-NEXT:    pmullw %xmm2, %xmm0
1553; SSE41-NEXT:    psrlw $8, %xmm0
1554; SSE41-NEXT:    psrlw $8, %xmm6
1555; SSE41-NEXT:    packuswb %xmm0, %xmm6
1556; SSE41-NEXT:    pcmpeqb %xmm5, %xmm6
1557; SSE41-NEXT:    pxor %xmm1, %xmm6
1558; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
1559; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1560; SSE41-NEXT:    pslld $31, %xmm0
1561; SSE41-NEXT:    psrad $31, %xmm0
1562; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
1563; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
1564; SSE41-NEXT:    pslld $31, %xmm1
1565; SSE41-NEXT:    psrad $31, %xmm1
1566; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
1567; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
1568; SSE41-NEXT:    pslld $31, %xmm2
1569; SSE41-NEXT:    psrad $31, %xmm2
1570; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,1,1]
1571; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
1572; SSE41-NEXT:    pslld $31, %xmm3
1573; SSE41-NEXT:    psrad $31, %xmm3
1574; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]
1575; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
1576; SSE41-NEXT:    pslld $31, %xmm5
1577; SSE41-NEXT:    psrad $31, %xmm5
1578; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[3,3,3,3]
1579; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
1580; SSE41-NEXT:    pslld $31, %xmm4
1581; SSE41-NEXT:    psrad $31, %xmm4
1582; SSE41-NEXT:    pmovsxbd %xmm6, %xmm6
1583; SSE41-NEXT:    pmovsxbd %xmm7, %xmm7
1584; SSE41-NEXT:    movdqa %xmm10, 16(%rsi)
1585; SSE41-NEXT:    movdqa %xmm8, (%rsi)
1586; SSE41-NEXT:    movdqa %xmm7, 64(%rdi)
1587; SSE41-NEXT:    movdqa %xmm6, (%rdi)
1588; SSE41-NEXT:    movdqa %xmm4, 112(%rdi)
1589; SSE41-NEXT:    movdqa %xmm5, 96(%rdi)
1590; SSE41-NEXT:    movdqa %xmm3, 80(%rdi)
1591; SSE41-NEXT:    movdqa %xmm2, 48(%rdi)
1592; SSE41-NEXT:    movdqa %xmm1, 32(%rdi)
1593; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
1594; SSE41-NEXT:    retq
1595;
1596; AVX1-LABEL: umulo_v32i8:
1597; AVX1:       # %bb.0:
1598; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1599; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1600; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
1601; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
1602; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
1603; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1604; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1605; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm9
1606; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm4
1607; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm8
1608; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1609; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1610; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
1611; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1612; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
1613; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm4
1614; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1615; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
1616; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm6
1617; AVX1-NEXT:    vpand %xmm3, %xmm6, %xmm3
1618; AVX1-NEXT:    vpackuswb %xmm4, %xmm3, %xmm5
1619; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
1620; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
1621; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
1622; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
1623; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1624; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm4
1625; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
1626; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
1627; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
1628; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
1629; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1630; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
1631; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
1632; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1633; AVX1-NEXT:    vpsrlw $8, %xmm9, %xmm1
1634; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
1635; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
1636; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
1637; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
1638; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1639; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
1640; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1641; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm2
1642; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
1643; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1644; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
1645; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
1646; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1647; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
1648; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
1649; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
1650; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
1651; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
1652; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,3,3]
1653; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
1654; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
1655; AVX1-NEXT:    vmovdqa %xmm5, 16(%rdi)
1656; AVX1-NEXT:    vmovdqa %xmm8, (%rdi)
1657; AVX1-NEXT:    retq
1658;
1659; AVX2-LABEL: umulo_v32i8:
1660; AVX2:       # %bb.0:
1661; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1662; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
1663; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1664; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1665; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
1666; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1667; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
1668; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1669; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
1670; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
1671; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1672; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
1673; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
1674; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
1675; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
1676; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
1677; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
1678; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
1679; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1680; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
1681; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
1682; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
1683; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
1684; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
1685; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1686; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
1687; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1688; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
1689; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1690; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
1691; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
1692; AVX2-NEXT:    retq
1693;
1694; AVX512F-LABEL: umulo_v32i8:
1695; AVX512F:       # %bb.0:
1696; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
1697; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1698; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
1699; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
1700; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
1701; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm3
1702; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1703; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
1704; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
1705; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
1706; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
1707; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
1708; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k1
1709; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1710; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1711; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm3
1712; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm0
1713; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1714; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1715; AVX512F-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
1716; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
1717; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
1718; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
1719; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
1720; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1721; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
1722; AVX512F-NEXT:    vpmovdb %zmm2, 16(%rdi)
1723; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
1724; AVX512F-NEXT:    vpmovdb %zmm2, (%rdi)
1725; AVX512F-NEXT:    retq
1726;
1727; AVX512BW-LABEL: umulo_v32i8:
1728; AVX512BW:       # %bb.0:
1729; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
1730; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1731; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
1732; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm0
1733; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
1734; AVX512BW-NEXT:    vptestmb %ymm0, %ymm0, %k1
1735; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1736; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
1737; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
1738; AVX512BW-NEXT:    vpmovwb %zmm2, (%rdi)
1739; AVX512BW-NEXT:    retq
1740  %t = call {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)
1741  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 0
1742  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 1
1743  %res = sext <32 x i1> %obit to <32 x i32>
1744  store <32 x i8> %val, <32 x i8>* %p2
1745  ret <32 x i32> %res
1746}
1747
1748define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nounwind {
1749; SSE2-LABEL: umulo_v64i8:
1750; SSE2:       # %bb.0:
1751; SSE2-NEXT:    movq %rdi, %rax
1752; SSE2-NEXT:    movdqa %xmm4, %xmm8
1753; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1754; SSE2-NEXT:    movdqa %xmm0, %xmm9
1755; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1756; SSE2-NEXT:    pmullw %xmm8, %xmm9
1757; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [255,255,255,255,255,255,255,255]
1758; SSE2-NEXT:    pand %xmm12, %xmm9
1759; SSE2-NEXT:    movdqa %xmm4, %xmm10
1760; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1761; SSE2-NEXT:    movdqa %xmm0, %xmm8
1762; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1763; SSE2-NEXT:    pmullw %xmm10, %xmm8
1764; SSE2-NEXT:    pand %xmm12, %xmm8
1765; SSE2-NEXT:    packuswb %xmm9, %xmm8
1766; SSE2-NEXT:    movdqa %xmm5, %xmm9
1767; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1768; SSE2-NEXT:    movdqa %xmm1, %xmm10
1769; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1770; SSE2-NEXT:    pmullw %xmm9, %xmm10
1771; SSE2-NEXT:    pand %xmm12, %xmm10
1772; SSE2-NEXT:    movdqa %xmm5, %xmm11
1773; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1774; SSE2-NEXT:    movdqa %xmm1, %xmm9
1775; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1776; SSE2-NEXT:    pmullw %xmm11, %xmm9
1777; SSE2-NEXT:    pand %xmm12, %xmm9
1778; SSE2-NEXT:    packuswb %xmm10, %xmm9
1779; SSE2-NEXT:    movdqa %xmm6, %xmm10
1780; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1781; SSE2-NEXT:    movdqa %xmm2, %xmm11
1782; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1783; SSE2-NEXT:    pmullw %xmm10, %xmm11
1784; SSE2-NEXT:    pand %xmm12, %xmm11
1785; SSE2-NEXT:    movdqa %xmm6, %xmm13
1786; SSE2-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1787; SSE2-NEXT:    movdqa %xmm2, %xmm10
1788; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1789; SSE2-NEXT:    pmullw %xmm13, %xmm10
1790; SSE2-NEXT:    pand %xmm12, %xmm10
1791; SSE2-NEXT:    packuswb %xmm11, %xmm10
1792; SSE2-NEXT:    movdqa %xmm7, %xmm11
1793; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1794; SSE2-NEXT:    movdqa %xmm3, %xmm13
1795; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1796; SSE2-NEXT:    pmullw %xmm11, %xmm13
1797; SSE2-NEXT:    pand %xmm12, %xmm13
1798; SSE2-NEXT:    movdqa %xmm7, %xmm14
1799; SSE2-NEXT:    punpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1800; SSE2-NEXT:    movdqa %xmm3, %xmm11
1801; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1802; SSE2-NEXT:    pmullw %xmm14, %xmm11
1803; SSE2-NEXT:    pand %xmm12, %xmm11
1804; SSE2-NEXT:    packuswb %xmm13, %xmm11
1805; SSE2-NEXT:    pxor %xmm12, %xmm12
1806; SSE2-NEXT:    movdqa %xmm7, %xmm13
1807; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
1808; SSE2-NEXT:    movdqa %xmm3, %xmm14
1809; SSE2-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm12[8],xmm14[9],xmm12[9],xmm14[10],xmm12[10],xmm14[11],xmm12[11],xmm14[12],xmm12[12],xmm14[13],xmm12[13],xmm14[14],xmm12[14],xmm14[15],xmm12[15]
1810; SSE2-NEXT:    pmullw %xmm13, %xmm14
1811; SSE2-NEXT:    psrlw $8, %xmm14
1812; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1],xmm7[2],xmm12[2],xmm7[3],xmm12[3],xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
1813; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
1814; SSE2-NEXT:    pmullw %xmm7, %xmm3
1815; SSE2-NEXT:    psrlw $8, %xmm3
1816; SSE2-NEXT:    packuswb %xmm14, %xmm3
1817; SSE2-NEXT:    movdqa %xmm6, %xmm13
1818; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
1819; SSE2-NEXT:    movdqa %xmm2, %xmm7
1820; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
1821; SSE2-NEXT:    pmullw %xmm13, %xmm7
1822; SSE2-NEXT:    psrlw $8, %xmm7
1823; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]
1824; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
1825; SSE2-NEXT:    pmullw %xmm6, %xmm2
1826; SSE2-NEXT:    psrlw $8, %xmm2
1827; SSE2-NEXT:    packuswb %xmm7, %xmm2
1828; SSE2-NEXT:    movdqa %xmm5, %xmm6
1829; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
1830; SSE2-NEXT:    movdqa %xmm1, %xmm7
1831; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
1832; SSE2-NEXT:    pmullw %xmm6, %xmm7
1833; SSE2-NEXT:    psrlw $8, %xmm7
1834; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3],xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
1835; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
1836; SSE2-NEXT:    pmullw %xmm5, %xmm1
1837; SSE2-NEXT:    psrlw $8, %xmm1
1838; SSE2-NEXT:    packuswb %xmm7, %xmm1
1839; SSE2-NEXT:    movdqa %xmm4, %xmm5
1840; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm12[8],xmm5[9],xmm12[9],xmm5[10],xmm12[10],xmm5[11],xmm12[11],xmm5[12],xmm12[12],xmm5[13],xmm12[13],xmm5[14],xmm12[14],xmm5[15],xmm12[15]
1841; SSE2-NEXT:    movdqa %xmm0, %xmm6
1842; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
1843; SSE2-NEXT:    pmullw %xmm5, %xmm6
1844; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
1845; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
1846; SSE2-NEXT:    pmullw %xmm4, %xmm0
1847; SSE2-NEXT:    psrlw $8, %xmm6
1848; SSE2-NEXT:    psrlw $8, %xmm0
1849; SSE2-NEXT:    packuswb %xmm6, %xmm0
1850; SSE2-NEXT:    pcmpeqb %xmm12, %xmm3
1851; SSE2-NEXT:    pcmpeqb %xmm12, %xmm2
1852; SSE2-NEXT:    pcmpeqb %xmm12, %xmm1
1853; SSE2-NEXT:    pcmpeqb %xmm12, %xmm0
1854; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
1855; SSE2-NEXT:    pxor %xmm4, %xmm3
1856; SSE2-NEXT:    pxor %xmm4, %xmm2
1857; SSE2-NEXT:    pxor %xmm4, %xmm1
1858; SSE2-NEXT:    pxor %xmm4, %xmm0
1859; SSE2-NEXT:    movdqa %xmm11, 48(%rsi)
1860; SSE2-NEXT:    movdqa %xmm10, 32(%rsi)
1861; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
1862; SSE2-NEXT:    movdqa %xmm3, %xmm4
1863; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1864; SSE2-NEXT:    movdqa %xmm8, (%rsi)
1865; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
1866; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1867; SSE2-NEXT:    psrad $24, %xmm5
1868; SSE2-NEXT:    movdqa %xmm5, 192(%rdi)
1869; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
1870; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1871; SSE2-NEXT:    psrad $24, %xmm5
1872; SSE2-NEXT:    movdqa %xmm5, 128(%rdi)
1873; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
1874; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1875; SSE2-NEXT:    psrad $24, %xmm5
1876; SSE2-NEXT:    movdqa %xmm5, 64(%rdi)
1877; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
1878; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
1879; SSE2-NEXT:    psrad $24, %xmm5
1880; SSE2-NEXT:    movdqa %xmm5, (%rdi)
1881; SSE2-NEXT:    movdqa %xmm4, %xmm5
1882; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1883; SSE2-NEXT:    pslld $31, %xmm4
1884; SSE2-NEXT:    psrad $31, %xmm4
1885; SSE2-NEXT:    movdqa %xmm4, 224(%rdi)
1886; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
1887; SSE2-NEXT:    pslld $31, %xmm5
1888; SSE2-NEXT:    psrad $31, %xmm5
1889; SSE2-NEXT:    movdqa %xmm5, 240(%rdi)
1890; SSE2-NEXT:    movdqa %xmm2, %xmm4
1891; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1892; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1893; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1894; SSE2-NEXT:    pslld $31, %xmm3
1895; SSE2-NEXT:    psrad $31, %xmm3
1896; SSE2-NEXT:    movdqa %xmm3, 208(%rdi)
1897; SSE2-NEXT:    movdqa %xmm4, %xmm3
1898; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
1899; SSE2-NEXT:    pslld $31, %xmm4
1900; SSE2-NEXT:    psrad $31, %xmm4
1901; SSE2-NEXT:    movdqa %xmm4, 160(%rdi)
1902; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
1903; SSE2-NEXT:    pslld $31, %xmm3
1904; SSE2-NEXT:    psrad $31, %xmm3
1905; SSE2-NEXT:    movdqa %xmm3, 176(%rdi)
1906; SSE2-NEXT:    movdqa %xmm1, %xmm3
1907; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1908; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1909; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1910; SSE2-NEXT:    pslld $31, %xmm2
1911; SSE2-NEXT:    psrad $31, %xmm2
1912; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
1913; SSE2-NEXT:    movdqa %xmm3, %xmm2
1914; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
1915; SSE2-NEXT:    pslld $31, %xmm3
1916; SSE2-NEXT:    psrad $31, %xmm3
1917; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
1918; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
1919; SSE2-NEXT:    pslld $31, %xmm2
1920; SSE2-NEXT:    psrad $31, %xmm2
1921; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
1922; SSE2-NEXT:    movdqa %xmm0, %xmm2
1923; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1924; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1925; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1926; SSE2-NEXT:    pslld $31, %xmm1
1927; SSE2-NEXT:    psrad $31, %xmm1
1928; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
1929; SSE2-NEXT:    movdqa %xmm2, %xmm1
1930; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
1931; SSE2-NEXT:    pslld $31, %xmm2
1932; SSE2-NEXT:    psrad $31, %xmm2
1933; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
1934; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
1935; SSE2-NEXT:    pslld $31, %xmm1
1936; SSE2-NEXT:    psrad $31, %xmm1
1937; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
1938; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1939; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1940; SSE2-NEXT:    pslld $31, %xmm0
1941; SSE2-NEXT:    psrad $31, %xmm0
1942; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
1943; SSE2-NEXT:    retq
1944;
1945; SSSE3-LABEL: umulo_v64i8:
1946; SSSE3:       # %bb.0:
1947; SSSE3-NEXT:    movq %rdi, %rax
1948; SSSE3-NEXT:    movdqa %xmm4, %xmm8
1949; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1950; SSSE3-NEXT:    movdqa %xmm0, %xmm9
1951; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1952; SSSE3-NEXT:    pmullw %xmm8, %xmm9
1953; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [255,255,255,255,255,255,255,255]
1954; SSSE3-NEXT:    pand %xmm12, %xmm9
1955; SSSE3-NEXT:    movdqa %xmm4, %xmm10
1956; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1957; SSSE3-NEXT:    movdqa %xmm0, %xmm8
1958; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1959; SSSE3-NEXT:    pmullw %xmm10, %xmm8
1960; SSSE3-NEXT:    pand %xmm12, %xmm8
1961; SSSE3-NEXT:    packuswb %xmm9, %xmm8
1962; SSSE3-NEXT:    movdqa %xmm5, %xmm9
1963; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1964; SSSE3-NEXT:    movdqa %xmm1, %xmm10
1965; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1966; SSSE3-NEXT:    pmullw %xmm9, %xmm10
1967; SSSE3-NEXT:    pand %xmm12, %xmm10
1968; SSSE3-NEXT:    movdqa %xmm5, %xmm11
1969; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1970; SSSE3-NEXT:    movdqa %xmm1, %xmm9
1971; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1972; SSSE3-NEXT:    pmullw %xmm11, %xmm9
1973; SSSE3-NEXT:    pand %xmm12, %xmm9
1974; SSSE3-NEXT:    packuswb %xmm10, %xmm9
1975; SSSE3-NEXT:    movdqa %xmm6, %xmm10
1976; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1977; SSSE3-NEXT:    movdqa %xmm2, %xmm11
1978; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1979; SSSE3-NEXT:    pmullw %xmm10, %xmm11
1980; SSSE3-NEXT:    pand %xmm12, %xmm11
1981; SSSE3-NEXT:    movdqa %xmm6, %xmm13
1982; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1983; SSSE3-NEXT:    movdqa %xmm2, %xmm10
1984; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1985; SSSE3-NEXT:    pmullw %xmm13, %xmm10
1986; SSSE3-NEXT:    pand %xmm12, %xmm10
1987; SSSE3-NEXT:    packuswb %xmm11, %xmm10
1988; SSSE3-NEXT:    movdqa %xmm7, %xmm11
1989; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1990; SSSE3-NEXT:    movdqa %xmm3, %xmm13
1991; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1992; SSSE3-NEXT:    pmullw %xmm11, %xmm13
1993; SSSE3-NEXT:    pand %xmm12, %xmm13
1994; SSSE3-NEXT:    movdqa %xmm7, %xmm14
1995; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1996; SSSE3-NEXT:    movdqa %xmm3, %xmm11
1997; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1998; SSSE3-NEXT:    pmullw %xmm14, %xmm11
1999; SSSE3-NEXT:    pand %xmm12, %xmm11
2000; SSSE3-NEXT:    packuswb %xmm13, %xmm11
2001; SSSE3-NEXT:    pxor %xmm12, %xmm12
2002; SSSE3-NEXT:    movdqa %xmm7, %xmm13
2003; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
2004; SSSE3-NEXT:    movdqa %xmm3, %xmm14
2005; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm12[8],xmm14[9],xmm12[9],xmm14[10],xmm12[10],xmm14[11],xmm12[11],xmm14[12],xmm12[12],xmm14[13],xmm12[13],xmm14[14],xmm12[14],xmm14[15],xmm12[15]
2006; SSSE3-NEXT:    pmullw %xmm13, %xmm14
2007; SSSE3-NEXT:    psrlw $8, %xmm14
2008; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1],xmm7[2],xmm12[2],xmm7[3],xmm12[3],xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
2009; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
2010; SSSE3-NEXT:    pmullw %xmm7, %xmm3
2011; SSSE3-NEXT:    psrlw $8, %xmm3
2012; SSSE3-NEXT:    packuswb %xmm14, %xmm3
2013; SSSE3-NEXT:    movdqa %xmm6, %xmm13
2014; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
2015; SSSE3-NEXT:    movdqa %xmm2, %xmm7
2016; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
2017; SSSE3-NEXT:    pmullw %xmm13, %xmm7
2018; SSSE3-NEXT:    psrlw $8, %xmm7
2019; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]
2020; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
2021; SSSE3-NEXT:    pmullw %xmm6, %xmm2
2022; SSSE3-NEXT:    psrlw $8, %xmm2
2023; SSSE3-NEXT:    packuswb %xmm7, %xmm2
2024; SSSE3-NEXT:    movdqa %xmm5, %xmm6
2025; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
2026; SSSE3-NEXT:    movdqa %xmm1, %xmm7
2027; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm12[8],xmm7[9],xmm12[9],xmm7[10],xmm12[10],xmm7[11],xmm12[11],xmm7[12],xmm12[12],xmm7[13],xmm12[13],xmm7[14],xmm12[14],xmm7[15],xmm12[15]
2028; SSSE3-NEXT:    pmullw %xmm6, %xmm7
2029; SSSE3-NEXT:    psrlw $8, %xmm7
2030; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3],xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
2031; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
2032; SSSE3-NEXT:    pmullw %xmm5, %xmm1
2033; SSSE3-NEXT:    psrlw $8, %xmm1
2034; SSSE3-NEXT:    packuswb %xmm7, %xmm1
2035; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2036; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm12[8],xmm5[9],xmm12[9],xmm5[10],xmm12[10],xmm5[11],xmm12[11],xmm5[12],xmm12[12],xmm5[13],xmm12[13],xmm5[14],xmm12[14],xmm5[15],xmm12[15]
2037; SSSE3-NEXT:    movdqa %xmm0, %xmm6
2038; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm12[8],xmm6[9],xmm12[9],xmm6[10],xmm12[10],xmm6[11],xmm12[11],xmm6[12],xmm12[12],xmm6[13],xmm12[13],xmm6[14],xmm12[14],xmm6[15],xmm12[15]
2039; SSSE3-NEXT:    pmullw %xmm5, %xmm6
2040; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3],xmm4[4],xmm12[4],xmm4[5],xmm12[5],xmm4[6],xmm12[6],xmm4[7],xmm12[7]
2041; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
2042; SSSE3-NEXT:    pmullw %xmm4, %xmm0
2043; SSSE3-NEXT:    psrlw $8, %xmm6
2044; SSSE3-NEXT:    psrlw $8, %xmm0
2045; SSSE3-NEXT:    packuswb %xmm6, %xmm0
2046; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm3
2047; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm2
2048; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm1
2049; SSSE3-NEXT:    pcmpeqb %xmm12, %xmm0
2050; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
2051; SSSE3-NEXT:    pxor %xmm4, %xmm3
2052; SSSE3-NEXT:    pxor %xmm4, %xmm2
2053; SSSE3-NEXT:    pxor %xmm4, %xmm1
2054; SSSE3-NEXT:    pxor %xmm4, %xmm0
2055; SSSE3-NEXT:    movdqa %xmm11, 48(%rsi)
2056; SSSE3-NEXT:    movdqa %xmm10, 32(%rsi)
2057; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
2058; SSSE3-NEXT:    movdqa %xmm3, %xmm4
2059; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2060; SSSE3-NEXT:    movdqa %xmm8, (%rsi)
2061; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
2062; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
2063; SSSE3-NEXT:    psrad $24, %xmm5
2064; SSSE3-NEXT:    movdqa %xmm5, 192(%rdi)
2065; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
2066; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
2067; SSSE3-NEXT:    psrad $24, %xmm5
2068; SSSE3-NEXT:    movdqa %xmm5, 128(%rdi)
2069; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
2070; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
2071; SSSE3-NEXT:    psrad $24, %xmm5
2072; SSSE3-NEXT:    movdqa %xmm5, 64(%rdi)
2073; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2074; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
2075; SSSE3-NEXT:    psrad $24, %xmm5
2076; SSSE3-NEXT:    movdqa %xmm5, (%rdi)
2077; SSSE3-NEXT:    movdqa %xmm4, %xmm5
2078; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2079; SSSE3-NEXT:    pslld $31, %xmm4
2080; SSSE3-NEXT:    psrad $31, %xmm4
2081; SSSE3-NEXT:    movdqa %xmm4, 224(%rdi)
2082; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
2083; SSSE3-NEXT:    pslld $31, %xmm5
2084; SSSE3-NEXT:    psrad $31, %xmm5
2085; SSSE3-NEXT:    movdqa %xmm5, 240(%rdi)
2086; SSSE3-NEXT:    movdqa %xmm2, %xmm4
2087; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2088; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2089; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
2090; SSSE3-NEXT:    pslld $31, %xmm3
2091; SSSE3-NEXT:    psrad $31, %xmm3
2092; SSSE3-NEXT:    movdqa %xmm3, 208(%rdi)
2093; SSSE3-NEXT:    movdqa %xmm4, %xmm3
2094; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
2095; SSSE3-NEXT:    pslld $31, %xmm4
2096; SSSE3-NEXT:    psrad $31, %xmm4
2097; SSSE3-NEXT:    movdqa %xmm4, 160(%rdi)
2098; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
2099; SSSE3-NEXT:    pslld $31, %xmm3
2100; SSSE3-NEXT:    psrad $31, %xmm3
2101; SSSE3-NEXT:    movdqa %xmm3, 176(%rdi)
2102; SSSE3-NEXT:    movdqa %xmm1, %xmm3
2103; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2104; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2105; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2106; SSSE3-NEXT:    pslld $31, %xmm2
2107; SSSE3-NEXT:    psrad $31, %xmm2
2108; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
2109; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2110; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
2111; SSSE3-NEXT:    pslld $31, %xmm3
2112; SSSE3-NEXT:    psrad $31, %xmm3
2113; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
2114; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
2115; SSSE3-NEXT:    pslld $31, %xmm2
2116; SSSE3-NEXT:    psrad $31, %xmm2
2117; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
2118; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2119; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2120; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2121; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2122; SSSE3-NEXT:    pslld $31, %xmm1
2123; SSSE3-NEXT:    psrad $31, %xmm1
2124; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
2125; SSSE3-NEXT:    movdqa %xmm2, %xmm1
2126; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
2127; SSSE3-NEXT:    pslld $31, %xmm2
2128; SSSE3-NEXT:    psrad $31, %xmm2
2129; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
2130; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2131; SSSE3-NEXT:    pslld $31, %xmm1
2132; SSSE3-NEXT:    psrad $31, %xmm1
2133; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
2134; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2135; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2136; SSSE3-NEXT:    pslld $31, %xmm0
2137; SSSE3-NEXT:    psrad $31, %xmm0
2138; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
2139; SSSE3-NEXT:    retq
2140;
2141; SSE41-LABEL: umulo_v64i8:
2142; SSE41:       # %bb.0:
2143; SSE41-NEXT:    movdqa %xmm5, %xmm11
2144; SSE41-NEXT:    movdqa %xmm4, %xmm13
2145; SSE41-NEXT:    movdqa %xmm3, %xmm4
2146; SSE41-NEXT:    movdqa %xmm0, %xmm5
2147; SSE41-NEXT:    movq %rdi, %rax
2148; SSE41-NEXT:    movdqa %xmm13, %xmm8
2149; SSE41-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2150; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2151; SSE41-NEXT:    pmullw %xmm8, %xmm0
2152; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
2153; SSE41-NEXT:    pand %xmm10, %xmm0
2154; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
2155; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2156; SSE41-NEXT:    pmullw %xmm9, %xmm8
2157; SSE41-NEXT:    movdqa %xmm8, %xmm3
2158; SSE41-NEXT:    pand %xmm10, %xmm3
2159; SSE41-NEXT:    packuswb %xmm0, %xmm3
2160; SSE41-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2161; SSE41-NEXT:    movdqa %xmm11, %xmm9
2162; SSE41-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2163; SSE41-NEXT:    movdqa %xmm1, %xmm0
2164; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2165; SSE41-NEXT:    pmullw %xmm9, %xmm0
2166; SSE41-NEXT:    pand %xmm10, %xmm0
2167; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2168; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2169; SSE41-NEXT:    pmullw %xmm12, %xmm9
2170; SSE41-NEXT:    movdqa %xmm9, %xmm3
2171; SSE41-NEXT:    pand %xmm10, %xmm3
2172; SSE41-NEXT:    packuswb %xmm0, %xmm3
2173; SSE41-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2174; SSE41-NEXT:    movdqa %xmm6, %xmm12
2175; SSE41-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2176; SSE41-NEXT:    movdqa %xmm2, %xmm0
2177; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2178; SSE41-NEXT:    pmullw %xmm12, %xmm0
2179; SSE41-NEXT:    pand %xmm10, %xmm0
2180; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2181; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2182; SSE41-NEXT:    pmullw %xmm14, %xmm12
2183; SSE41-NEXT:    movdqa %xmm12, %xmm14
2184; SSE41-NEXT:    pand %xmm10, %xmm14
2185; SSE41-NEXT:    packuswb %xmm0, %xmm14
2186; SSE41-NEXT:    movdqa %xmm7, %xmm0
2187; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2188; SSE41-NEXT:    movdqa %xmm4, %xmm3
2189; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2190; SSE41-NEXT:    pmullw %xmm0, %xmm3
2191; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2192; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm15 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2193; SSE41-NEXT:    pmullw %xmm0, %xmm15
2194; SSE41-NEXT:    pand %xmm10, %xmm3
2195; SSE41-NEXT:    pand %xmm15, %xmm10
2196; SSE41-NEXT:    packuswb %xmm3, %xmm10
2197; SSE41-NEXT:    pxor %xmm0, %xmm0
2198; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2199; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2200; SSE41-NEXT:    pmullw %xmm7, %xmm4
2201; SSE41-NEXT:    psrlw $8, %xmm4
2202; SSE41-NEXT:    psrlw $8, %xmm15
2203; SSE41-NEXT:    packuswb %xmm4, %xmm15
2204; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
2205; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2206; SSE41-NEXT:    pmullw %xmm6, %xmm2
2207; SSE41-NEXT:    psrlw $8, %xmm2
2208; SSE41-NEXT:    psrlw $8, %xmm12
2209; SSE41-NEXT:    packuswb %xmm2, %xmm12
2210; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2211; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2212; SSE41-NEXT:    pmullw %xmm11, %xmm1
2213; SSE41-NEXT:    psrlw $8, %xmm1
2214; SSE41-NEXT:    psrlw $8, %xmm9
2215; SSE41-NEXT:    packuswb %xmm1, %xmm9
2216; SSE41-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm0[8],xmm13[9],xmm0[9],xmm13[10],xmm0[10],xmm13[11],xmm0[11],xmm13[12],xmm0[12],xmm13[13],xmm0[13],xmm13[14],xmm0[14],xmm13[15],xmm0[15]
2217; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
2218; SSE41-NEXT:    pmullw %xmm13, %xmm5
2219; SSE41-NEXT:    psrlw $8, %xmm5
2220; SSE41-NEXT:    psrlw $8, %xmm8
2221; SSE41-NEXT:    packuswb %xmm5, %xmm8
2222; SSE41-NEXT:    pcmpeqb %xmm0, %xmm15
2223; SSE41-NEXT:    pcmpeqb %xmm0, %xmm12
2224; SSE41-NEXT:    pcmpeqb %xmm0, %xmm9
2225; SSE41-NEXT:    pcmpeqb %xmm0, %xmm8
2226; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
2227; SSE41-NEXT:    pxor %xmm0, %xmm15
2228; SSE41-NEXT:    pxor %xmm0, %xmm12
2229; SSE41-NEXT:    pxor %xmm0, %xmm9
2230; SSE41-NEXT:    pxor %xmm0, %xmm8
2231; SSE41-NEXT:    movdqa %xmm10, 48(%rsi)
2232; SSE41-NEXT:    movdqa %xmm14, 32(%rsi)
2233; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2234; SSE41-NEXT:    movaps %xmm0, 16(%rsi)
2235; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2236; SSE41-NEXT:    movaps %xmm0, (%rsi)
2237; SSE41-NEXT:    pmovsxbd %xmm15, %xmm0
2238; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
2239; SSE41-NEXT:    pmovsxbd %xmm12, %xmm0
2240; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
2241; SSE41-NEXT:    pmovsxbd %xmm9, %xmm0
2242; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
2243; SSE41-NEXT:    pmovsxbd %xmm8, %xmm0
2244; SSE41-NEXT:    movdqa %xmm0, (%rdi)
2245; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[2,3,2,3]
2246; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2247; SSE41-NEXT:    pslld $31, %xmm0
2248; SSE41-NEXT:    psrad $31, %xmm0
2249; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
2250; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[3,3,3,3]
2251; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2252; SSE41-NEXT:    pslld $31, %xmm0
2253; SSE41-NEXT:    psrad $31, %xmm0
2254; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
2255; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[1,1,1,1]
2256; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2257; SSE41-NEXT:    pslld $31, %xmm0
2258; SSE41-NEXT:    psrad $31, %xmm0
2259; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
2260; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[2,3,2,3]
2261; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2262; SSE41-NEXT:    pslld $31, %xmm0
2263; SSE41-NEXT:    psrad $31, %xmm0
2264; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
2265; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[3,3,3,3]
2266; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2267; SSE41-NEXT:    pslld $31, %xmm0
2268; SSE41-NEXT:    psrad $31, %xmm0
2269; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
2270; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,1,1]
2271; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2272; SSE41-NEXT:    pslld $31, %xmm0
2273; SSE41-NEXT:    psrad $31, %xmm0
2274; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
2275; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
2276; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2277; SSE41-NEXT:    pslld $31, %xmm0
2278; SSE41-NEXT:    psrad $31, %xmm0
2279; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
2280; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[3,3,3,3]
2281; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2282; SSE41-NEXT:    pslld $31, %xmm0
2283; SSE41-NEXT:    psrad $31, %xmm0
2284; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
2285; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,1,1]
2286; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2287; SSE41-NEXT:    pslld $31, %xmm0
2288; SSE41-NEXT:    psrad $31, %xmm0
2289; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
2290; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
2291; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2292; SSE41-NEXT:    pslld $31, %xmm0
2293; SSE41-NEXT:    psrad $31, %xmm0
2294; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
2295; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
2296; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2297; SSE41-NEXT:    pslld $31, %xmm0
2298; SSE41-NEXT:    psrad $31, %xmm0
2299; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
2300; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
2301; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
2302; SSE41-NEXT:    pslld $31, %xmm0
2303; SSE41-NEXT:    psrad $31, %xmm0
2304; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
2305; SSE41-NEXT:    retq
2306;
2307; AVX1-LABEL: umulo_v64i8:
2308; AVX1:       # %bb.0:
2309; AVX1-NEXT:    vmovdqa %ymm2, %ymm10
2310; AVX1-NEXT:    vmovdqa %ymm0, %ymm2
2311; AVX1-NEXT:    movq %rdi, %rax
2312; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2313; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2314; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2315; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
2316; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
2317; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
2318; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2319; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm12
2320; AVX1-NEXT:    vpand %xmm7, %xmm12, %xmm5
2321; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm0
2322; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2323; AVX1-NEXT:    vextractf128 $1, %ymm10, %xmm13
2324; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm13[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2325; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm14
2326; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm14[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2327; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
2328; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm5
2329; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
2330; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
2331; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm15
2332; AVX1-NEXT:    vpand %xmm7, %xmm15, %xmm6
2333; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm0
2334; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2335; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2336; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2337; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
2338; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
2339; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2340; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2341; AVX1-NEXT:    vpmullw %xmm6, %xmm4, %xmm0
2342; AVX1-NEXT:    vpand %xmm7, %xmm0, %xmm6
2343; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm4
2344; AVX1-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2345; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
2346; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2347; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
2348; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2349; AVX1-NEXT:    vpmullw %xmm6, %xmm8, %xmm8
2350; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2351; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm9 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2352; AVX1-NEXT:    vpmullw %xmm6, %xmm9, %xmm6
2353; AVX1-NEXT:    vpand %xmm7, %xmm8, %xmm8
2354; AVX1-NEXT:    vpand %xmm7, %xmm6, %xmm7
2355; AVX1-NEXT:    vpackuswb %xmm8, %xmm7, %xmm8
2356; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
2357; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
2358; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
2359; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
2360; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
2361; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm5
2362; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
2363; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm7[8],xmm3[9],xmm7[9],xmm3[10],xmm7[10],xmm3[11],xmm7[11],xmm3[12],xmm7[12],xmm3[13],xmm7[13],xmm3[14],xmm7[14],xmm3[15],xmm7[15]
2364; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
2365; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
2366; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2367; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
2368; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
2369; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm7[8],xmm13[9],xmm7[9],xmm13[10],xmm7[10],xmm13[11],xmm7[11],xmm13[12],xmm7[12],xmm13[13],xmm7[13],xmm13[14],xmm7[14],xmm13[15],xmm7[15]
2370; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm7[8],xmm14[9],xmm7[9],xmm14[10],xmm7[10],xmm14[11],xmm7[11],xmm14[12],xmm7[12],xmm14[13],xmm7[13],xmm14[14],xmm7[14],xmm14[15],xmm7[15]
2371; AVX1-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
2372; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
2373; AVX1-NEXT:    vpsrlw $8, %xmm15, %xmm3
2374; AVX1-NEXT:    vpackuswb %xmm1, %xmm3, %xmm1
2375; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
2376; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]
2377; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
2378; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
2379; AVX1-NEXT:    vpsrlw $8, %xmm12, %xmm3
2380; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
2381; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm3
2382; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm0, %xmm0
2383; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm1, %xmm1
2384; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm2, %xmm4
2385; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
2386; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
2387; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm2
2388; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
2389; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm0
2390; AVX1-NEXT:    vmovdqa %xmm8, 48(%rsi)
2391; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2392; AVX1-NEXT:    vmovaps %xmm4, 32(%rsi)
2393; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2394; AVX1-NEXT:    vmovaps %xmm4, 16(%rsi)
2395; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2396; AVX1-NEXT:    vmovaps %xmm4, (%rsi)
2397; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm4
2398; AVX1-NEXT:    vmovdqa %xmm4, 192(%rdi)
2399; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm4
2400; AVX1-NEXT:    vmovdqa %xmm4, 128(%rdi)
2401; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
2402; AVX1-NEXT:    vmovdqa %xmm4, 64(%rdi)
2403; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
2404; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)
2405; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
2406; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
2407; AVX1-NEXT:    vmovdqa %xmm4, 224(%rdi)
2408; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
2409; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
2410; AVX1-NEXT:    vmovdqa %xmm4, 240(%rdi)
2411; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
2412; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2413; AVX1-NEXT:    vmovdqa %xmm3, 208(%rdi)
2414; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
2415; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2416; AVX1-NEXT:    vmovdqa %xmm3, 160(%rdi)
2417; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
2418; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
2419; AVX1-NEXT:    vmovdqa %xmm3, 176(%rdi)
2420; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
2421; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2422; AVX1-NEXT:    vmovdqa %xmm2, 144(%rdi)
2423; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2424; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2425; AVX1-NEXT:    vmovdqa %xmm2, 96(%rdi)
2426; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
2427; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
2428; AVX1-NEXT:    vmovdqa %xmm2, 112(%rdi)
2429; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
2430; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2431; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
2432; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
2433; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2434; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
2435; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2436; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
2437; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
2438; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
2439; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
2440; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
2441; AVX1-NEXT:    vzeroupper
2442; AVX1-NEXT:    retq
2443;
2444; AVX2-LABEL: umulo_v64i8:
2445; AVX2:       # %bb.0:
2446; AVX2-NEXT:    movq %rdi, %rax
2447; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2448; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2449; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
2450; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2451; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
2452; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2453; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2454; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2455; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2456; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm9
2457; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2458; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2459; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
2460; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
2461; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2462; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2463; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2464; AVX2-NEXT:    vpand %ymm5, %ymm7, %ymm5
2465; AVX2-NEXT:    vpackuswb %ymm6, %ymm5, %ymm10
2466; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
2467; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm3[8],ymm6[8],ymm3[9],ymm6[9],ymm3[10],ymm6[10],ymm3[11],ymm6[11],ymm3[12],ymm6[12],ymm3[13],ymm6[13],ymm3[14],ymm6[14],ymm3[15],ymm6[15],ymm3[24],ymm6[24],ymm3[25],ymm6[25],ymm3[26],ymm6[26],ymm3[27],ymm6[27],ymm3[28],ymm6[28],ymm3[29],ymm6[29],ymm3[30],ymm6[30],ymm3[31],ymm6[31]
2468; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm1[8],ymm6[8],ymm1[9],ymm6[9],ymm1[10],ymm6[10],ymm1[11],ymm6[11],ymm1[12],ymm6[12],ymm1[13],ymm6[13],ymm1[14],ymm6[14],ymm1[15],ymm6[15],ymm1[24],ymm6[24],ymm1[25],ymm6[25],ymm1[26],ymm6[26],ymm1[27],ymm6[27],ymm1[28],ymm6[28],ymm1[29],ymm6[29],ymm1[30],ymm6[30],ymm1[31],ymm6[31]
2469; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2470; AVX2-NEXT:    vpsrlw $8, %ymm7, %ymm7
2471; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[2],ymm6[2],ymm3[3],ymm6[3],ymm3[4],ymm6[4],ymm3[5],ymm6[5],ymm3[6],ymm6[6],ymm3[7],ymm6[7],ymm3[16],ymm6[16],ymm3[17],ymm6[17],ymm3[18],ymm6[18],ymm3[19],ymm6[19],ymm3[20],ymm6[20],ymm3[21],ymm6[21],ymm3[22],ymm6[22],ymm3[23],ymm6[23]
2472; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[1],ymm6[1],ymm1[2],ymm6[2],ymm1[3],ymm6[3],ymm1[4],ymm6[4],ymm1[5],ymm6[5],ymm1[6],ymm6[6],ymm1[7],ymm6[7],ymm1[16],ymm6[16],ymm1[17],ymm6[17],ymm1[18],ymm6[18],ymm1[19],ymm6[19],ymm1[20],ymm6[20],ymm1[21],ymm6[21],ymm1[22],ymm6[22],ymm1[23],ymm6[23]
2473; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
2474; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
2475; AVX2-NEXT:    vpackuswb %ymm7, %ymm1, %ymm1
2476; AVX2-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
2477; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
2478; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
2479; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm2[8],ymm6[8],ymm2[9],ymm6[9],ymm2[10],ymm6[10],ymm2[11],ymm6[11],ymm2[12],ymm6[12],ymm2[13],ymm6[13],ymm2[14],ymm6[14],ymm2[15],ymm6[15],ymm2[24],ymm6[24],ymm2[25],ymm6[25],ymm2[26],ymm6[26],ymm2[27],ymm6[27],ymm2[28],ymm6[28],ymm2[29],ymm6[29],ymm2[30],ymm6[30],ymm2[31],ymm6[31]
2480; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8],ymm6[8],ymm0[9],ymm6[9],ymm0[10],ymm6[10],ymm0[11],ymm6[11],ymm0[12],ymm6[12],ymm0[13],ymm6[13],ymm0[14],ymm6[14],ymm0[15],ymm6[15],ymm0[24],ymm6[24],ymm0[25],ymm6[25],ymm0[26],ymm6[26],ymm0[27],ymm6[27],ymm0[28],ymm6[28],ymm0[29],ymm6[29],ymm0[30],ymm6[30],ymm0[31],ymm6[31]
2481; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm7
2482; AVX2-NEXT:    vpsrlw $8, %ymm7, %ymm7
2483; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[4],ymm6[4],ymm2[5],ymm6[5],ymm2[6],ymm6[6],ymm2[7],ymm6[7],ymm2[16],ymm6[16],ymm2[17],ymm6[17],ymm2[18],ymm6[18],ymm2[19],ymm6[19],ymm2[20],ymm6[20],ymm2[21],ymm6[21],ymm2[22],ymm6[22],ymm2[23],ymm6[23]
2484; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[4],ymm6[4],ymm0[5],ymm6[5],ymm0[6],ymm6[6],ymm0[7],ymm6[7],ymm0[16],ymm6[16],ymm0[17],ymm6[17],ymm0[18],ymm6[18],ymm0[19],ymm6[19],ymm0[20],ymm6[20],ymm0[21],ymm6[21],ymm0[22],ymm6[22],ymm0[23],ymm6[23]
2485; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
2486; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
2487; AVX2-NEXT:    vpackuswb %ymm7, %ymm0, %ymm0
2488; AVX2-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
2489; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
2490; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2491; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
2492; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
2493; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
2494; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
2495; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
2496; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
2497; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
2498; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
2499; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
2500; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
2501; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
2502; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
2503; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
2504; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
2505; AVX2-NEXT:    vmovdqa %ymm9, (%rsi)
2506; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
2507; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
2508; AVX2-NEXT:    vmovdqa %ymm3, 64(%rdi)
2509; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
2510; AVX2-NEXT:    vmovdqa %ymm5, 224(%rdi)
2511; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
2512; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
2513; AVX2-NEXT:    vmovdqa %ymm2, 32(%rdi)
2514; AVX2-NEXT:    vzeroupper
2515; AVX2-NEXT:    retq
2516;
2517; AVX512F-LABEL: umulo_v64i8:
2518; AVX512F:       # %bb.0:
2519; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
2520; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm2
2521; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2522; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
2523; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm4
2524; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2525; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm4
2526; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm2
2527; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
2528; AVX512F-NEXT:    vpmovdb %zmm2, %xmm6
2529; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2530; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm6, %xmm6
2531; AVX512F-NEXT:    vpternlogq $15, %xmm6, %xmm6, %xmm6
2532; AVX512F-NEXT:    vpmovsxbd %xmm6, %zmm6
2533; AVX512F-NEXT:    vptestmd %zmm6, %zmm6, %k1
2534; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2535; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
2536; AVX512F-NEXT:    vpmullw %ymm3, %ymm5, %ymm5
2537; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm3
2538; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2539; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2540; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
2541; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
2542; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
2543; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k2
2544; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
2545; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
2546; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm6
2547; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
2548; AVX512F-NEXT:    vpmullw %ymm3, %ymm6, %ymm6
2549; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm3
2550; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
2551; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
2552; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
2553; AVX512F-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
2554; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
2555; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k3
2556; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2557; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2558; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm7
2559; AVX512F-NEXT:    vpsrlw $8, %ymm7, %ymm0
2560; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
2561; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2562; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2563; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2564; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
2565; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k4
2566; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
2567; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
2568; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
2569; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2570; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
2571; AVX512F-NEXT:    vpmovdb %zmm4, 48(%rdi)
2572; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
2573; AVX512F-NEXT:    vpmovdb %zmm4, 32(%rdi)
2574; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm6[0],zero,ymm6[1],zero,ymm6[2],zero,ymm6[3],zero,ymm6[4],zero,ymm6[5],zero,ymm6[6],zero,ymm6[7],zero,ymm6[8],zero,ymm6[9],zero,ymm6[10],zero,ymm6[11],zero,ymm6[12],zero,ymm6[13],zero,ymm6[14],zero,ymm6[15],zero
2575; AVX512F-NEXT:    vpmovdb %zmm4, 16(%rdi)
2576; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm7[0],zero,ymm7[1],zero,ymm7[2],zero,ymm7[3],zero,ymm7[4],zero,ymm7[5],zero,ymm7[6],zero,ymm7[7],zero,ymm7[8],zero,ymm7[9],zero,ymm7[10],zero,ymm7[11],zero,ymm7[12],zero,ymm7[13],zero,ymm7[14],zero,ymm7[15],zero
2577; AVX512F-NEXT:    vpmovdb %zmm4, (%rdi)
2578; AVX512F-NEXT:    retq
2579;
2580; AVX512BW-LABEL: umulo_v64i8:
2581; AVX512BW:       # %bb.0:
2582; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2583; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2584; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
2585; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2586; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
2587; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2588; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm5 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2589; AVX512BW-NEXT:    vpmullw %zmm4, %zmm5, %zmm4
2590; AVX512BW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
2591; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm4
2592; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2593; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
2594; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm5 = zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[10],zmm2[10],zmm0[11],zmm2[11],zmm0[12],zmm2[12],zmm0[13],zmm2[13],zmm0[14],zmm2[14],zmm0[15],zmm2[15],zmm0[24],zmm2[24],zmm0[25],zmm2[25],zmm0[26],zmm2[26],zmm0[27],zmm2[27],zmm0[28],zmm2[28],zmm0[29],zmm2[29],zmm0[30],zmm2[30],zmm0[31],zmm2[31],zmm0[40],zmm2[40],zmm0[41],zmm2[41],zmm0[42],zmm2[42],zmm0[43],zmm2[43],zmm0[44],zmm2[44],zmm0[45],zmm2[45],zmm0[46],zmm2[46],zmm0[47],zmm2[47],zmm0[56],zmm2[56],zmm0[57],zmm2[57],zmm0[58],zmm2[58],zmm0[59],zmm2[59],zmm0[60],zmm2[60],zmm0[61],zmm2[61],zmm0[62],zmm2[62],zmm0[63],zmm2[63]
2595; AVX512BW-NEXT:    vpmullw %zmm3, %zmm5, %zmm3
2596; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
2597; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]
2598; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[2],zmm2[2],zmm0[3],zmm2[3],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[6],zmm2[6],zmm0[7],zmm2[7],zmm0[16],zmm2[16],zmm0[17],zmm2[17],zmm0[18],zmm2[18],zmm0[19],zmm2[19],zmm0[20],zmm2[20],zmm0[21],zmm2[21],zmm0[22],zmm2[22],zmm0[23],zmm2[23],zmm0[32],zmm2[32],zmm0[33],zmm2[33],zmm0[34],zmm2[34],zmm0[35],zmm2[35],zmm0[36],zmm2[36],zmm0[37],zmm2[37],zmm0[38],zmm2[38],zmm0[39],zmm2[39],zmm0[48],zmm2[48],zmm0[49],zmm2[49],zmm0[50],zmm2[50],zmm0[51],zmm2[51],zmm0[52],zmm2[52],zmm0[53],zmm2[53],zmm0[54],zmm2[54],zmm0[55],zmm2[55]
2599; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
2600; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
2601; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
2602; AVX512BW-NEXT:    vptestmb %zmm0, %zmm0, %k1
2603; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2604; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
2605; AVX512BW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2606; AVX512BW-NEXT:    kshiftrq $32, %k1, %k1
2607; AVX512BW-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
2608; AVX512BW-NEXT:    kshiftrd $16, %k1, %k1
2609; AVX512BW-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
2610; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdi)
2611; AVX512BW-NEXT:    retq
2612  %t = call {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)
2613  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 0
2614  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 1
2615  %res = sext <64 x i1> %obit to <64 x i32>
2616  store <64 x i8> %val, <64 x i8>* %p2
2617  ret <64 x i32> %res
2618}
2619
2620define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
2621; SSE2-LABEL: umulo_v8i16:
2622; SSE2:       # %bb.0:
2623; SSE2-NEXT:    movdqa %xmm0, %xmm2
2624; SSE2-NEXT:    pmullw %xmm1, %xmm2
2625; SSE2-NEXT:    pmulhuw %xmm1, %xmm0
2626; SSE2-NEXT:    pxor %xmm3, %xmm3
2627; SSE2-NEXT:    pcmpeqw %xmm0, %xmm3
2628; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
2629; SSE2-NEXT:    pxor %xmm3, %xmm1
2630; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2631; SSE2-NEXT:    psrad $16, %xmm0
2632; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2633; SSE2-NEXT:    pslld $31, %xmm1
2634; SSE2-NEXT:    psrad $31, %xmm1
2635; SSE2-NEXT:    movdqa %xmm2, (%rdi)
2636; SSE2-NEXT:    retq
2637;
2638; SSSE3-LABEL: umulo_v8i16:
2639; SSSE3:       # %bb.0:
2640; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2641; SSSE3-NEXT:    pmullw %xmm1, %xmm2
2642; SSSE3-NEXT:    pmulhuw %xmm1, %xmm0
2643; SSSE3-NEXT:    pxor %xmm3, %xmm3
2644; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm3
2645; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
2646; SSSE3-NEXT:    pxor %xmm3, %xmm1
2647; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2648; SSSE3-NEXT:    psrad $16, %xmm0
2649; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2650; SSSE3-NEXT:    pslld $31, %xmm1
2651; SSSE3-NEXT:    psrad $31, %xmm1
2652; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
2653; SSSE3-NEXT:    retq
2654;
2655; SSE41-LABEL: umulo_v8i16:
2656; SSE41:       # %bb.0:
2657; SSE41-NEXT:    movdqa %xmm0, %xmm2
2658; SSE41-NEXT:    pmullw %xmm1, %xmm2
2659; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
2660; SSE41-NEXT:    pxor %xmm3, %xmm3
2661; SSE41-NEXT:    pcmpeqw %xmm0, %xmm3
2662; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
2663; SSE41-NEXT:    pxor %xmm3, %xmm1
2664; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
2665; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2666; SSE41-NEXT:    pslld $31, %xmm1
2667; SSE41-NEXT:    psrad $31, %xmm1
2668; SSE41-NEXT:    movdqa %xmm2, (%rdi)
2669; SSE41-NEXT:    retq
2670;
2671; AVX1-LABEL: umulo_v8i16:
2672; AVX1:       # %bb.0:
2673; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2674; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2675; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2676; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2677; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
2678; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
2679; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
2680; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2681; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
2682; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2683; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
2684; AVX1-NEXT:    retq
2685;
2686; AVX2-LABEL: umulo_v8i16:
2687; AVX2:       # %bb.0:
2688; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2689; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2690; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2691; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2692; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
2693; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
2694; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
2695; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
2696; AVX2-NEXT:    retq
2697;
2698; AVX512F-LABEL: umulo_v8i16:
2699; AVX512F:       # %bb.0:
2700; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2701; AVX512F-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2702; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2703; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2704; AVX512F-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
2705; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
2706; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
2707; AVX512F-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
2708; AVX512F-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
2709; AVX512F-NEXT:    vmovdqa %xmm2, (%rdi)
2710; AVX512F-NEXT:    retq
2711;
2712; AVX512BW-LABEL: umulo_v8i16:
2713; AVX512BW:       # %bb.0:
2714; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm2
2715; AVX512BW-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
2716; AVX512BW-NEXT:    vptestmw %xmm0, %xmm0, %k1
2717; AVX512BW-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
2718; AVX512BW-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
2719; AVX512BW-NEXT:    vmovdqa %xmm2, (%rdi)
2720; AVX512BW-NEXT:    retq
2721  %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
2722  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
2723  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
2724  %res = sext <8 x i1> %obit to <8 x i32>
2725  store <8 x i16> %val, <8 x i16>* %p2
2726  ret <8 x i32> %res
2727}
2728
2729define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
2730; SSE2-LABEL: umulo_v2i64:
2731; SSE2:       # %bb.0:
2732; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2733; SSE2-NEXT:    movq %xmm2, %r8
2734; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2735; SSE2-NEXT:    movq %xmm2, %r10
2736; SSE2-NEXT:    movq %xmm0, %rax
2737; SSE2-NEXT:    movq %xmm1, %rdx
2738; SSE2-NEXT:    xorl %ecx, %ecx
2739; SSE2-NEXT:    mulq %rdx
2740; SSE2-NEXT:    movq $-1, %r9
2741; SSE2-NEXT:    movl $0, %esi
2742; SSE2-NEXT:    cmovoq %r9, %rsi
2743; SSE2-NEXT:    movq %rax, %xmm1
2744; SSE2-NEXT:    movq %r8, %rax
2745; SSE2-NEXT:    mulq %r10
2746; SSE2-NEXT:    movq %rax, %xmm0
2747; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2748; SSE2-NEXT:    movq %rsi, %xmm0
2749; SSE2-NEXT:    cmovoq %r9, %rcx
2750; SSE2-NEXT:    movq %rcx, %xmm2
2751; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2752; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2753; SSE2-NEXT:    movdqa %xmm1, (%rdi)
2754; SSE2-NEXT:    retq
2755;
2756; SSSE3-LABEL: umulo_v2i64:
2757; SSSE3:       # %bb.0:
2758; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
2759; SSSE3-NEXT:    movq %xmm2, %r8
2760; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
2761; SSSE3-NEXT:    movq %xmm2, %r10
2762; SSSE3-NEXT:    movq %xmm0, %rax
2763; SSSE3-NEXT:    movq %xmm1, %rdx
2764; SSSE3-NEXT:    xorl %ecx, %ecx
2765; SSSE3-NEXT:    mulq %rdx
2766; SSSE3-NEXT:    movq $-1, %r9
2767; SSSE3-NEXT:    movl $0, %esi
2768; SSSE3-NEXT:    cmovoq %r9, %rsi
2769; SSSE3-NEXT:    movq %rax, %xmm1
2770; SSSE3-NEXT:    movq %r8, %rax
2771; SSSE3-NEXT:    mulq %r10
2772; SSSE3-NEXT:    movq %rax, %xmm0
2773; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2774; SSSE3-NEXT:    movq %rsi, %xmm0
2775; SSSE3-NEXT:    cmovoq %r9, %rcx
2776; SSSE3-NEXT:    movq %rcx, %xmm2
2777; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2778; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2779; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
2780; SSSE3-NEXT:    retq
2781;
2782; SSE41-LABEL: umulo_v2i64:
2783; SSE41:       # %bb.0:
2784; SSE41-NEXT:    movq %xmm0, %r10
2785; SSE41-NEXT:    movq %xmm1, %r8
2786; SSE41-NEXT:    pextrq $1, %xmm0, %rax
2787; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
2788; SSE41-NEXT:    xorl %esi, %esi
2789; SSE41-NEXT:    mulq %rdx
2790; SSE41-NEXT:    movq $-1, %r9
2791; SSE41-NEXT:    movl $0, %ecx
2792; SSE41-NEXT:    cmovoq %r9, %rcx
2793; SSE41-NEXT:    movq %rax, %xmm0
2794; SSE41-NEXT:    movq %r10, %rax
2795; SSE41-NEXT:    mulq %r8
2796; SSE41-NEXT:    movq %rax, %xmm1
2797; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2798; SSE41-NEXT:    movq %rcx, %xmm0
2799; SSE41-NEXT:    cmovoq %r9, %rsi
2800; SSE41-NEXT:    movq %rsi, %xmm2
2801; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
2802; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2803; SSE41-NEXT:    movdqa %xmm1, (%rdi)
2804; SSE41-NEXT:    retq
2805;
2806; AVX-LABEL: umulo_v2i64:
2807; AVX:       # %bb.0:
2808; AVX-NEXT:    vmovq %xmm0, %r10
2809; AVX-NEXT:    vmovq %xmm1, %r8
2810; AVX-NEXT:    vpextrq $1, %xmm0, %rax
2811; AVX-NEXT:    vpextrq $1, %xmm1, %rdx
2812; AVX-NEXT:    xorl %esi, %esi
2813; AVX-NEXT:    mulq %rdx
2814; AVX-NEXT:    movq $-1, %r9
2815; AVX-NEXT:    movl $0, %ecx
2816; AVX-NEXT:    cmovoq %r9, %rcx
2817; AVX-NEXT:    vmovq %rax, %xmm0
2818; AVX-NEXT:    movq %r10, %rax
2819; AVX-NEXT:    mulq %r8
2820; AVX-NEXT:    vmovq %rax, %xmm1
2821; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2822; AVX-NEXT:    vmovq %rcx, %xmm0
2823; AVX-NEXT:    cmovoq %r9, %rsi
2824; AVX-NEXT:    vmovq %rsi, %xmm2
2825; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
2826; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2827; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
2828; AVX-NEXT:    retq
2829;
2830; AVX512F-LABEL: umulo_v2i64:
2831; AVX512F:       # %bb.0:
2832; AVX512F-NEXT:    vmovq %xmm0, %rcx
2833; AVX512F-NEXT:    vmovq %xmm1, %rsi
2834; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
2835; AVX512F-NEXT:    vpextrq $1, %xmm1, %rdx
2836; AVX512F-NEXT:    mulq %rdx
2837; AVX512F-NEXT:    seto %r8b
2838; AVX512F-NEXT:    vmovq %rax, %xmm0
2839; AVX512F-NEXT:    movq %rcx, %rax
2840; AVX512F-NEXT:    mulq %rsi
2841; AVX512F-NEXT:    vmovq %rax, %xmm1
2842; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2843; AVX512F-NEXT:    seto %al
2844; AVX512F-NEXT:    movw $-3, %cx
2845; AVX512F-NEXT:    kmovw %ecx, %k0
2846; AVX512F-NEXT:    kmovw %eax, %k1
2847; AVX512F-NEXT:    kandw %k0, %k1, %k0
2848; AVX512F-NEXT:    kmovw %r8d, %k1
2849; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
2850; AVX512F-NEXT:    kshiftrw $14, %k1, %k1
2851; AVX512F-NEXT:    korw %k1, %k0, %k1
2852; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2853; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2854; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)
2855; AVX512F-NEXT:    retq
2856;
2857; AVX512BW-LABEL: umulo_v2i64:
2858; AVX512BW:       # %bb.0:
2859; AVX512BW-NEXT:    vmovq %xmm0, %rcx
2860; AVX512BW-NEXT:    vmovq %xmm1, %rsi
2861; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
2862; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rdx
2863; AVX512BW-NEXT:    mulq %rdx
2864; AVX512BW-NEXT:    seto %r8b
2865; AVX512BW-NEXT:    vmovq %rax, %xmm0
2866; AVX512BW-NEXT:    movq %rcx, %rax
2867; AVX512BW-NEXT:    mulq %rsi
2868; AVX512BW-NEXT:    vmovq %rax, %xmm1
2869; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2870; AVX512BW-NEXT:    seto %al
2871; AVX512BW-NEXT:    movw $-3, %cx
2872; AVX512BW-NEXT:    kmovd %ecx, %k0
2873; AVX512BW-NEXT:    kmovd %eax, %k1
2874; AVX512BW-NEXT:    kandw %k0, %k1, %k0
2875; AVX512BW-NEXT:    kmovd %r8d, %k1
2876; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
2877; AVX512BW-NEXT:    kshiftrw $14, %k1, %k1
2878; AVX512BW-NEXT:    korw %k1, %k0, %k1
2879; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
2880; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
2881; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)
2882; AVX512BW-NEXT:    retq
2883  %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
2884  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
2885  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
2886  %res = sext <2 x i1> %obit to <2 x i32>
2887  store <2 x i64> %val, <2 x i64>* %p2
2888  ret <2 x i32> %res
2889}
2890
2891define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
2892; SSE2-LABEL: umulo_v4i24:
2893; SSE2:       # %bb.0:
2894; SSE2-NEXT:    movdqa %xmm0, %xmm2
2895; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
2896; SSE2-NEXT:    pand %xmm0, %xmm1
2897; SSE2-NEXT:    pand %xmm0, %xmm2
2898; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
2899; SSE2-NEXT:    pmuludq %xmm1, %xmm2
2900; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
2901; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2902; SSE2-NEXT:    pmuludq %xmm0, %xmm1
2903; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
2904; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
2905; SSE2-NEXT:    pxor %xmm4, %xmm4
2906; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
2907; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
2908; SSE2-NEXT:    pxor %xmm5, %xmm3
2909; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2910; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
2911; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
2912; SSE2-NEXT:    psrld $24, %xmm0
2913; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
2914; SSE2-NEXT:    pxor %xmm5, %xmm0
2915; SSE2-NEXT:    por %xmm3, %xmm0
2916; SSE2-NEXT:    movd %xmm2, %eax
2917; SSE2-NEXT:    movw %ax, (%rdi)
2918; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2919; SSE2-NEXT:    movd %xmm2, %ecx
2920; SSE2-NEXT:    movw %cx, 6(%rdi)
2921; SSE2-NEXT:    movd %xmm1, %edx
2922; SSE2-NEXT:    movw %dx, 3(%rdi)
2923; SSE2-NEXT:    shrl $16, %eax
2924; SSE2-NEXT:    movb %al, 2(%rdi)
2925; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
2926; SSE2-NEXT:    movd %xmm1, %eax
2927; SSE2-NEXT:    movw %ax, 9(%rdi)
2928; SSE2-NEXT:    shrl $16, %ecx
2929; SSE2-NEXT:    movb %cl, 8(%rdi)
2930; SSE2-NEXT:    shrl $16, %edx
2931; SSE2-NEXT:    movb %dl, 5(%rdi)
2932; SSE2-NEXT:    shrl $16, %eax
2933; SSE2-NEXT:    movb %al, 11(%rdi)
2934; SSE2-NEXT:    retq
2935;
2936; SSSE3-LABEL: umulo_v4i24:
2937; SSSE3:       # %bb.0:
2938; SSSE3-NEXT:    movdqa %xmm0, %xmm2
2939; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
2940; SSSE3-NEXT:    pand %xmm0, %xmm1
2941; SSSE3-NEXT:    pand %xmm0, %xmm2
2942; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
2943; SSSE3-NEXT:    pmuludq %xmm1, %xmm2
2944; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
2945; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
2946; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
2947; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
2948; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
2949; SSSE3-NEXT:    pxor %xmm4, %xmm4
2950; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
2951; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
2952; SSSE3-NEXT:    pxor %xmm5, %xmm3
2953; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
2954; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
2955; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
2956; SSSE3-NEXT:    psrld $24, %xmm0
2957; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
2958; SSSE3-NEXT:    pxor %xmm5, %xmm0
2959; SSSE3-NEXT:    por %xmm3, %xmm0
2960; SSSE3-NEXT:    movd %xmm2, %eax
2961; SSSE3-NEXT:    movw %ax, (%rdi)
2962; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
2963; SSSE3-NEXT:    movd %xmm2, %ecx
2964; SSSE3-NEXT:    movw %cx, 6(%rdi)
2965; SSSE3-NEXT:    movd %xmm1, %edx
2966; SSSE3-NEXT:    movw %dx, 3(%rdi)
2967; SSSE3-NEXT:    shrl $16, %eax
2968; SSSE3-NEXT:    movb %al, 2(%rdi)
2969; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
2970; SSSE3-NEXT:    movd %xmm1, %eax
2971; SSSE3-NEXT:    movw %ax, 9(%rdi)
2972; SSSE3-NEXT:    shrl $16, %ecx
2973; SSSE3-NEXT:    movb %cl, 8(%rdi)
2974; SSSE3-NEXT:    shrl $16, %edx
2975; SSSE3-NEXT:    movb %dl, 5(%rdi)
2976; SSSE3-NEXT:    shrl $16, %eax
2977; SSSE3-NEXT:    movb %al, 11(%rdi)
2978; SSSE3-NEXT:    retq
2979;
2980; SSE41-LABEL: umulo_v4i24:
2981; SSE41:       # %bb.0:
2982; SSE41-NEXT:    movdqa %xmm0, %xmm2
2983; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
2984; SSE41-NEXT:    pand %xmm0, %xmm2
2985; SSE41-NEXT:    pand %xmm0, %xmm1
2986; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
2987; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
2988; SSE41-NEXT:    pmuludq %xmm0, %xmm3
2989; SSE41-NEXT:    movdqa %xmm2, %xmm0
2990; SSE41-NEXT:    pmuludq %xmm1, %xmm0
2991; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
2992; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
2993; SSE41-NEXT:    pxor %xmm0, %xmm0
2994; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
2995; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
2996; SSE41-NEXT:    pxor %xmm3, %xmm4
2997; SSE41-NEXT:    pmulld %xmm2, %xmm1
2998; SSE41-NEXT:    pextrd $3, %xmm1, %eax
2999; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
3000; SSE41-NEXT:    pextrd $1, %xmm1, %edx
3001; SSE41-NEXT:    movd %xmm1, %esi
3002; SSE41-NEXT:    psrld $24, %xmm1
3003; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
3004; SSE41-NEXT:    pxor %xmm3, %xmm0
3005; SSE41-NEXT:    por %xmm4, %xmm0
3006; SSE41-NEXT:    movw %ax, 9(%rdi)
3007; SSE41-NEXT:    movw %cx, 6(%rdi)
3008; SSE41-NEXT:    movw %dx, 3(%rdi)
3009; SSE41-NEXT:    movw %si, (%rdi)
3010; SSE41-NEXT:    shrl $16, %eax
3011; SSE41-NEXT:    movb %al, 11(%rdi)
3012; SSE41-NEXT:    shrl $16, %ecx
3013; SSE41-NEXT:    movb %cl, 8(%rdi)
3014; SSE41-NEXT:    shrl $16, %edx
3015; SSE41-NEXT:    movb %dl, 5(%rdi)
3016; SSE41-NEXT:    shrl $16, %esi
3017; SSE41-NEXT:    movb %sil, 2(%rdi)
3018; SSE41-NEXT:    retq
3019;
3020; AVX1-LABEL: umulo_v4i24:
3021; AVX1:       # %bb.0:
3022; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3023; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
3024; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
3025; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[1,1,3,3]
3026; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[1,1,3,3]
3027; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
3028; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
3029; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3030; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
3031; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
3032; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
3033; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
3034; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
3035; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3036; AVX1-NEXT:    vpsrld $24, %xmm1, %xmm0
3037; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
3038; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
3039; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
3040; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
3041; AVX1-NEXT:    movw %ax, 9(%rdi)
3042; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
3043; AVX1-NEXT:    movw %cx, 6(%rdi)
3044; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
3045; AVX1-NEXT:    movw %dx, 3(%rdi)
3046; AVX1-NEXT:    vmovd %xmm1, %esi
3047; AVX1-NEXT:    movw %si, (%rdi)
3048; AVX1-NEXT:    shrl $16, %eax
3049; AVX1-NEXT:    movb %al, 11(%rdi)
3050; AVX1-NEXT:    shrl $16, %ecx
3051; AVX1-NEXT:    movb %cl, 8(%rdi)
3052; AVX1-NEXT:    shrl $16, %edx
3053; AVX1-NEXT:    movb %dl, 5(%rdi)
3054; AVX1-NEXT:    shrl $16, %esi
3055; AVX1-NEXT:    movb %sil, 2(%rdi)
3056; AVX1-NEXT:    retq
3057;
3058; AVX2-LABEL: umulo_v4i24:
3059; AVX2:       # %bb.0:
3060; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3061; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
3062; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
3063; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
3064; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
3065; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
3066; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
3067; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3068; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
3069; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
3070; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
3071; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
3072; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
3073; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3074; AVX2-NEXT:    vpsrld $24, %xmm1, %xmm0
3075; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
3076; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
3077; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
3078; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
3079; AVX2-NEXT:    movw %ax, 9(%rdi)
3080; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
3081; AVX2-NEXT:    movw %cx, 6(%rdi)
3082; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
3083; AVX2-NEXT:    movw %dx, 3(%rdi)
3084; AVX2-NEXT:    vmovd %xmm1, %esi
3085; AVX2-NEXT:    movw %si, (%rdi)
3086; AVX2-NEXT:    shrl $16, %eax
3087; AVX2-NEXT:    movb %al, 11(%rdi)
3088; AVX2-NEXT:    shrl $16, %ecx
3089; AVX2-NEXT:    movb %cl, 8(%rdi)
3090; AVX2-NEXT:    shrl $16, %edx
3091; AVX2-NEXT:    movb %dl, 5(%rdi)
3092; AVX2-NEXT:    shrl $16, %esi
3093; AVX2-NEXT:    movb %sil, 2(%rdi)
3094; AVX2-NEXT:    retq
3095;
3096; AVX512-LABEL: umulo_v4i24:
3097; AVX512:       # %bb.0:
3098; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
3099; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
3100; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
3101; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
3102; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
3103; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
3104; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
3105; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
3106; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
3107; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
3108; AVX512-NEXT:    vpsrld $24, %xmm1, %xmm0
3109; AVX512-NEXT:    vpor %xmm4, %xmm0, %xmm0
3110; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
3111; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3112; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3113; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
3114; AVX512-NEXT:    movw %ax, 9(%rdi)
3115; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
3116; AVX512-NEXT:    movw %cx, 6(%rdi)
3117; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
3118; AVX512-NEXT:    movw %dx, 3(%rdi)
3119; AVX512-NEXT:    vmovd %xmm1, %esi
3120; AVX512-NEXT:    movw %si, (%rdi)
3121; AVX512-NEXT:    shrl $16, %eax
3122; AVX512-NEXT:    movb %al, 11(%rdi)
3123; AVX512-NEXT:    shrl $16, %ecx
3124; AVX512-NEXT:    movb %cl, 8(%rdi)
3125; AVX512-NEXT:    shrl $16, %edx
3126; AVX512-NEXT:    movb %dl, 5(%rdi)
3127; AVX512-NEXT:    shrl $16, %esi
3128; AVX512-NEXT:    movb %sil, 2(%rdi)
3129; AVX512-NEXT:    retq
3130  %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
3131  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
3132  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
3133  %res = sext <4 x i1> %obit to <4 x i32>
3134  store <4 x i24> %val, <4 x i24>* %p2
3135  ret <4 x i32> %res
3136}
3137
3138define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
3139; SSE-LABEL: umulo_v4i1:
3140; SSE:       # %bb.0:
3141; SSE-NEXT:    pand %xmm1, %xmm0
3142; SSE-NEXT:    pslld $31, %xmm0
3143; SSE-NEXT:    movmskps %xmm0, %eax
3144; SSE-NEXT:    movb %al, (%rdi)
3145; SSE-NEXT:    xorps %xmm0, %xmm0
3146; SSE-NEXT:    retq
3147;
3148; AVX-LABEL: umulo_v4i1:
3149; AVX:       # %bb.0:
3150; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
3151; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
3152; AVX-NEXT:    vmovmskps %xmm0, %eax
3153; AVX-NEXT:    movb %al, (%rdi)
3154; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
3155; AVX-NEXT:    retq
3156;
3157; AVX512F-LABEL: umulo_v4i1:
3158; AVX512F:       # %bb.0:
3159; AVX512F-NEXT:    vpand %xmm1, %xmm0, %xmm0
3160; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
3161; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k0
3162; AVX512F-NEXT:    kmovw %k0, %eax
3163; AVX512F-NEXT:    movb %al, (%rdi)
3164; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
3165; AVX512F-NEXT:    retq
3166;
3167; AVX512BW-LABEL: umulo_v4i1:
3168; AVX512BW:       # %bb.0:
3169; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
3170; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
3171; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k0
3172; AVX512BW-NEXT:    kmovd %k0, %eax
3173; AVX512BW-NEXT:    movb %al, (%rdi)
3174; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
3175; AVX512BW-NEXT:    retq
3176  %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
3177  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
3178  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
3179  %res = sext <4 x i1> %obit to <4 x i32>
3180  store <4 x i1> %val, <4 x i1>* %p2
3181  ret <4 x i32> %res
3182}
3183
3184define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
3185; SSE2-LABEL: umulo_v2i128:
3186; SSE2:       # %bb.0:
3187; SSE2-NEXT:    pushq %rbp
3188; SSE2-NEXT:    pushq %r15
3189; SSE2-NEXT:    pushq %r14
3190; SSE2-NEXT:    pushq %r13
3191; SSE2-NEXT:    pushq %r12
3192; SSE2-NEXT:    pushq %rbx
3193; SSE2-NEXT:    movq %r9, %r10
3194; SSE2-NEXT:    movq %rcx, %r12
3195; SSE2-NEXT:    movq %rdx, %r11
3196; SSE2-NEXT:    movq %rsi, %rax
3197; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3198; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3199; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3200; SSE2-NEXT:    testq %r10, %r10
3201; SSE2-NEXT:    setne %cl
3202; SSE2-NEXT:    testq %rsi, %rsi
3203; SSE2-NEXT:    setne %r13b
3204; SSE2-NEXT:    andb %cl, %r13b
3205; SSE2-NEXT:    mulq %r8
3206; SSE2-NEXT:    movq %rax, %rsi
3207; SSE2-NEXT:    seto %bpl
3208; SSE2-NEXT:    movq %r10, %rax
3209; SSE2-NEXT:    mulq %rdi
3210; SSE2-NEXT:    movq %rax, %rcx
3211; SSE2-NEXT:    seto %bl
3212; SSE2-NEXT:    orb %bpl, %bl
3213; SSE2-NEXT:    addq %rsi, %rcx
3214; SSE2-NEXT:    movq %rdi, %rax
3215; SSE2-NEXT:    mulq %r8
3216; SSE2-NEXT:    movq %rax, %rdi
3217; SSE2-NEXT:    movq %rdx, %rsi
3218; SSE2-NEXT:    addq %rcx, %rsi
3219; SSE2-NEXT:    setb %cl
3220; SSE2-NEXT:    orb %bl, %cl
3221; SSE2-NEXT:    orb %r13b, %cl
3222; SSE2-NEXT:    testq %r9, %r9
3223; SSE2-NEXT:    setne %al
3224; SSE2-NEXT:    testq %r12, %r12
3225; SSE2-NEXT:    setne %r8b
3226; SSE2-NEXT:    andb %al, %r8b
3227; SSE2-NEXT:    movq %r12, %rax
3228; SSE2-NEXT:    mulq %r15
3229; SSE2-NEXT:    movq %rax, %rbp
3230; SSE2-NEXT:    seto %r10b
3231; SSE2-NEXT:    movq %r9, %rax
3232; SSE2-NEXT:    mulq %r11
3233; SSE2-NEXT:    movq %rax, %rbx
3234; SSE2-NEXT:    seto %r9b
3235; SSE2-NEXT:    orb %r10b, %r9b
3236; SSE2-NEXT:    addq %rbp, %rbx
3237; SSE2-NEXT:    movq %r11, %rax
3238; SSE2-NEXT:    mulq %r15
3239; SSE2-NEXT:    addq %rbx, %rdx
3240; SSE2-NEXT:    setb %bl
3241; SSE2-NEXT:    orb %r9b, %bl
3242; SSE2-NEXT:    orb %r8b, %bl
3243; SSE2-NEXT:    movzbl %bl, %ebp
3244; SSE2-NEXT:    negl %ebp
3245; SSE2-NEXT:    movd %ebp, %xmm1
3246; SSE2-NEXT:    movzbl %cl, %ecx
3247; SSE2-NEXT:    negl %ecx
3248; SSE2-NEXT:    movd %ecx, %xmm0
3249; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3250; SSE2-NEXT:    movq %rax, 16(%r14)
3251; SSE2-NEXT:    movq %rdi, (%r14)
3252; SSE2-NEXT:    movq %rdx, 24(%r14)
3253; SSE2-NEXT:    movq %rsi, 8(%r14)
3254; SSE2-NEXT:    popq %rbx
3255; SSE2-NEXT:    popq %r12
3256; SSE2-NEXT:    popq %r13
3257; SSE2-NEXT:    popq %r14
3258; SSE2-NEXT:    popq %r15
3259; SSE2-NEXT:    popq %rbp
3260; SSE2-NEXT:    retq
3261;
3262; SSSE3-LABEL: umulo_v2i128:
3263; SSSE3:       # %bb.0:
3264; SSSE3-NEXT:    pushq %rbp
3265; SSSE3-NEXT:    pushq %r15
3266; SSSE3-NEXT:    pushq %r14
3267; SSSE3-NEXT:    pushq %r13
3268; SSSE3-NEXT:    pushq %r12
3269; SSSE3-NEXT:    pushq %rbx
3270; SSSE3-NEXT:    movq %r9, %r10
3271; SSSE3-NEXT:    movq %rcx, %r12
3272; SSSE3-NEXT:    movq %rdx, %r11
3273; SSSE3-NEXT:    movq %rsi, %rax
3274; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3275; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3276; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3277; SSSE3-NEXT:    testq %r10, %r10
3278; SSSE3-NEXT:    setne %cl
3279; SSSE3-NEXT:    testq %rsi, %rsi
3280; SSSE3-NEXT:    setne %r13b
3281; SSSE3-NEXT:    andb %cl, %r13b
3282; SSSE3-NEXT:    mulq %r8
3283; SSSE3-NEXT:    movq %rax, %rsi
3284; SSSE3-NEXT:    seto %bpl
3285; SSSE3-NEXT:    movq %r10, %rax
3286; SSSE3-NEXT:    mulq %rdi
3287; SSSE3-NEXT:    movq %rax, %rcx
3288; SSSE3-NEXT:    seto %bl
3289; SSSE3-NEXT:    orb %bpl, %bl
3290; SSSE3-NEXT:    addq %rsi, %rcx
3291; SSSE3-NEXT:    movq %rdi, %rax
3292; SSSE3-NEXT:    mulq %r8
3293; SSSE3-NEXT:    movq %rax, %rdi
3294; SSSE3-NEXT:    movq %rdx, %rsi
3295; SSSE3-NEXT:    addq %rcx, %rsi
3296; SSSE3-NEXT:    setb %cl
3297; SSSE3-NEXT:    orb %bl, %cl
3298; SSSE3-NEXT:    orb %r13b, %cl
3299; SSSE3-NEXT:    testq %r9, %r9
3300; SSSE3-NEXT:    setne %al
3301; SSSE3-NEXT:    testq %r12, %r12
3302; SSSE3-NEXT:    setne %r8b
3303; SSSE3-NEXT:    andb %al, %r8b
3304; SSSE3-NEXT:    movq %r12, %rax
3305; SSSE3-NEXT:    mulq %r15
3306; SSSE3-NEXT:    movq %rax, %rbp
3307; SSSE3-NEXT:    seto %r10b
3308; SSSE3-NEXT:    movq %r9, %rax
3309; SSSE3-NEXT:    mulq %r11
3310; SSSE3-NEXT:    movq %rax, %rbx
3311; SSSE3-NEXT:    seto %r9b
3312; SSSE3-NEXT:    orb %r10b, %r9b
3313; SSSE3-NEXT:    addq %rbp, %rbx
3314; SSSE3-NEXT:    movq %r11, %rax
3315; SSSE3-NEXT:    mulq %r15
3316; SSSE3-NEXT:    addq %rbx, %rdx
3317; SSSE3-NEXT:    setb %bl
3318; SSSE3-NEXT:    orb %r9b, %bl
3319; SSSE3-NEXT:    orb %r8b, %bl
3320; SSSE3-NEXT:    movzbl %bl, %ebp
3321; SSSE3-NEXT:    negl %ebp
3322; SSSE3-NEXT:    movd %ebp, %xmm1
3323; SSSE3-NEXT:    movzbl %cl, %ecx
3324; SSSE3-NEXT:    negl %ecx
3325; SSSE3-NEXT:    movd %ecx, %xmm0
3326; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3327; SSSE3-NEXT:    movq %rax, 16(%r14)
3328; SSSE3-NEXT:    movq %rdi, (%r14)
3329; SSSE3-NEXT:    movq %rdx, 24(%r14)
3330; SSSE3-NEXT:    movq %rsi, 8(%r14)
3331; SSSE3-NEXT:    popq %rbx
3332; SSSE3-NEXT:    popq %r12
3333; SSSE3-NEXT:    popq %r13
3334; SSSE3-NEXT:    popq %r14
3335; SSSE3-NEXT:    popq %r15
3336; SSSE3-NEXT:    popq %rbp
3337; SSSE3-NEXT:    retq
3338;
3339; SSE41-LABEL: umulo_v2i128:
3340; SSE41:       # %bb.0:
3341; SSE41-NEXT:    pushq %rbp
3342; SSE41-NEXT:    pushq %r15
3343; SSE41-NEXT:    pushq %r14
3344; SSE41-NEXT:    pushq %r13
3345; SSE41-NEXT:    pushq %r12
3346; SSE41-NEXT:    pushq %rbx
3347; SSE41-NEXT:    movq %r9, %r10
3348; SSE41-NEXT:    movq %rcx, %r12
3349; SSE41-NEXT:    movq %rdx, %r11
3350; SSE41-NEXT:    movq %rsi, %rax
3351; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3352; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3353; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3354; SSE41-NEXT:    testq %r10, %r10
3355; SSE41-NEXT:    setne %cl
3356; SSE41-NEXT:    testq %rsi, %rsi
3357; SSE41-NEXT:    setne %r13b
3358; SSE41-NEXT:    andb %cl, %r13b
3359; SSE41-NEXT:    mulq %r8
3360; SSE41-NEXT:    movq %rax, %rsi
3361; SSE41-NEXT:    seto %bpl
3362; SSE41-NEXT:    movq %r10, %rax
3363; SSE41-NEXT:    mulq %rdi
3364; SSE41-NEXT:    movq %rax, %rcx
3365; SSE41-NEXT:    seto %bl
3366; SSE41-NEXT:    orb %bpl, %bl
3367; SSE41-NEXT:    addq %rsi, %rcx
3368; SSE41-NEXT:    movq %rdi, %rax
3369; SSE41-NEXT:    mulq %r8
3370; SSE41-NEXT:    movq %rax, %rdi
3371; SSE41-NEXT:    movq %rdx, %rsi
3372; SSE41-NEXT:    addq %rcx, %rsi
3373; SSE41-NEXT:    setb %cl
3374; SSE41-NEXT:    orb %bl, %cl
3375; SSE41-NEXT:    orb %r13b, %cl
3376; SSE41-NEXT:    testq %r9, %r9
3377; SSE41-NEXT:    setne %al
3378; SSE41-NEXT:    testq %r12, %r12
3379; SSE41-NEXT:    setne %r8b
3380; SSE41-NEXT:    andb %al, %r8b
3381; SSE41-NEXT:    movq %r12, %rax
3382; SSE41-NEXT:    mulq %r15
3383; SSE41-NEXT:    movq %rax, %rbp
3384; SSE41-NEXT:    seto %r10b
3385; SSE41-NEXT:    movq %r9, %rax
3386; SSE41-NEXT:    mulq %r11
3387; SSE41-NEXT:    movq %rax, %rbx
3388; SSE41-NEXT:    seto %r9b
3389; SSE41-NEXT:    orb %r10b, %r9b
3390; SSE41-NEXT:    addq %rbp, %rbx
3391; SSE41-NEXT:    movq %r11, %rax
3392; SSE41-NEXT:    mulq %r15
3393; SSE41-NEXT:    addq %rbx, %rdx
3394; SSE41-NEXT:    setb %bl
3395; SSE41-NEXT:    orb %r9b, %bl
3396; SSE41-NEXT:    orb %r8b, %bl
3397; SSE41-NEXT:    movzbl %bl, %ebp
3398; SSE41-NEXT:    negl %ebp
3399; SSE41-NEXT:    movzbl %cl, %ecx
3400; SSE41-NEXT:    negl %ecx
3401; SSE41-NEXT:    movd %ecx, %xmm0
3402; SSE41-NEXT:    pinsrd $1, %ebp, %xmm0
3403; SSE41-NEXT:    movq %rax, 16(%r14)
3404; SSE41-NEXT:    movq %rdi, (%r14)
3405; SSE41-NEXT:    movq %rdx, 24(%r14)
3406; SSE41-NEXT:    movq %rsi, 8(%r14)
3407; SSE41-NEXT:    popq %rbx
3408; SSE41-NEXT:    popq %r12
3409; SSE41-NEXT:    popq %r13
3410; SSE41-NEXT:    popq %r14
3411; SSE41-NEXT:    popq %r15
3412; SSE41-NEXT:    popq %rbp
3413; SSE41-NEXT:    retq
3414;
3415; AVX-LABEL: umulo_v2i128:
3416; AVX:       # %bb.0:
3417; AVX-NEXT:    pushq %rbp
3418; AVX-NEXT:    pushq %r15
3419; AVX-NEXT:    pushq %r14
3420; AVX-NEXT:    pushq %r13
3421; AVX-NEXT:    pushq %r12
3422; AVX-NEXT:    pushq %rbx
3423; AVX-NEXT:    movq %r9, %r10
3424; AVX-NEXT:    movq %rcx, %r12
3425; AVX-NEXT:    movq %rdx, %r11
3426; AVX-NEXT:    movq %rsi, %rax
3427; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3428; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3429; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r9
3430; AVX-NEXT:    testq %r10, %r10
3431; AVX-NEXT:    setne %cl
3432; AVX-NEXT:    testq %rsi, %rsi
3433; AVX-NEXT:    setne %r13b
3434; AVX-NEXT:    andb %cl, %r13b
3435; AVX-NEXT:    mulq %r8
3436; AVX-NEXT:    movq %rax, %rsi
3437; AVX-NEXT:    seto %bpl
3438; AVX-NEXT:    movq %r10, %rax
3439; AVX-NEXT:    mulq %rdi
3440; AVX-NEXT:    movq %rax, %rcx
3441; AVX-NEXT:    seto %bl
3442; AVX-NEXT:    orb %bpl, %bl
3443; AVX-NEXT:    addq %rsi, %rcx
3444; AVX-NEXT:    movq %rdi, %rax
3445; AVX-NEXT:    mulq %r8
3446; AVX-NEXT:    movq %rax, %rdi
3447; AVX-NEXT:    movq %rdx, %rsi
3448; AVX-NEXT:    addq %rcx, %rsi
3449; AVX-NEXT:    setb %cl
3450; AVX-NEXT:    orb %bl, %cl
3451; AVX-NEXT:    orb %r13b, %cl
3452; AVX-NEXT:    testq %r9, %r9
3453; AVX-NEXT:    setne %al
3454; AVX-NEXT:    testq %r12, %r12
3455; AVX-NEXT:    setne %r8b
3456; AVX-NEXT:    andb %al, %r8b
3457; AVX-NEXT:    movq %r12, %rax
3458; AVX-NEXT:    mulq %r15
3459; AVX-NEXT:    movq %rax, %rbp
3460; AVX-NEXT:    seto %r10b
3461; AVX-NEXT:    movq %r9, %rax
3462; AVX-NEXT:    mulq %r11
3463; AVX-NEXT:    movq %rax, %rbx
3464; AVX-NEXT:    seto %r9b
3465; AVX-NEXT:    orb %r10b, %r9b
3466; AVX-NEXT:    addq %rbp, %rbx
3467; AVX-NEXT:    movq %r11, %rax
3468; AVX-NEXT:    mulq %r15
3469; AVX-NEXT:    addq %rbx, %rdx
3470; AVX-NEXT:    setb %bl
3471; AVX-NEXT:    orb %r9b, %bl
3472; AVX-NEXT:    orb %r8b, %bl
3473; AVX-NEXT:    movzbl %bl, %ebp
3474; AVX-NEXT:    negl %ebp
3475; AVX-NEXT:    movzbl %cl, %ecx
3476; AVX-NEXT:    negl %ecx
3477; AVX-NEXT:    vmovd %ecx, %xmm0
3478; AVX-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
3479; AVX-NEXT:    movq %rax, 16(%r14)
3480; AVX-NEXT:    movq %rdi, (%r14)
3481; AVX-NEXT:    movq %rdx, 24(%r14)
3482; AVX-NEXT:    movq %rsi, 8(%r14)
3483; AVX-NEXT:    popq %rbx
3484; AVX-NEXT:    popq %r12
3485; AVX-NEXT:    popq %r13
3486; AVX-NEXT:    popq %r14
3487; AVX-NEXT:    popq %r15
3488; AVX-NEXT:    popq %rbp
3489; AVX-NEXT:    retq
3490;
3491; AVX512F-LABEL: umulo_v2i128:
3492; AVX512F:       # %bb.0:
3493; AVX512F-NEXT:    pushq %rbp
3494; AVX512F-NEXT:    pushq %r15
3495; AVX512F-NEXT:    pushq %r14
3496; AVX512F-NEXT:    pushq %r13
3497; AVX512F-NEXT:    pushq %r12
3498; AVX512F-NEXT:    pushq %rbx
3499; AVX512F-NEXT:    movq %rcx, %rax
3500; AVX512F-NEXT:    movq %rdx, %r12
3501; AVX512F-NEXT:    movq %rdi, %r11
3502; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3503; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3504; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
3505; AVX512F-NEXT:    testq %r10, %r10
3506; AVX512F-NEXT:    setne %dl
3507; AVX512F-NEXT:    testq %rcx, %rcx
3508; AVX512F-NEXT:    setne %r13b
3509; AVX512F-NEXT:    andb %dl, %r13b
3510; AVX512F-NEXT:    mulq %r15
3511; AVX512F-NEXT:    movq %rax, %rdi
3512; AVX512F-NEXT:    seto %bpl
3513; AVX512F-NEXT:    movq %r10, %rax
3514; AVX512F-NEXT:    mulq %r12
3515; AVX512F-NEXT:    movq %rax, %rbx
3516; AVX512F-NEXT:    seto %cl
3517; AVX512F-NEXT:    orb %bpl, %cl
3518; AVX512F-NEXT:    addq %rdi, %rbx
3519; AVX512F-NEXT:    movq %r12, %rax
3520; AVX512F-NEXT:    mulq %r15
3521; AVX512F-NEXT:    movq %rax, %r10
3522; AVX512F-NEXT:    movq %rdx, %r15
3523; AVX512F-NEXT:    addq %rbx, %r15
3524; AVX512F-NEXT:    setb %al
3525; AVX512F-NEXT:    orb %cl, %al
3526; AVX512F-NEXT:    orb %r13b, %al
3527; AVX512F-NEXT:    kmovw %eax, %k0
3528; AVX512F-NEXT:    testq %r9, %r9
3529; AVX512F-NEXT:    setne %al
3530; AVX512F-NEXT:    testq %rsi, %rsi
3531; AVX512F-NEXT:    setne %cl
3532; AVX512F-NEXT:    andb %al, %cl
3533; AVX512F-NEXT:    movq %rsi, %rax
3534; AVX512F-NEXT:    mulq %r8
3535; AVX512F-NEXT:    movq %rax, %rsi
3536; AVX512F-NEXT:    seto %bpl
3537; AVX512F-NEXT:    movq %r9, %rax
3538; AVX512F-NEXT:    mulq %r11
3539; AVX512F-NEXT:    movq %rax, %rdi
3540; AVX512F-NEXT:    seto %bl
3541; AVX512F-NEXT:    orb %bpl, %bl
3542; AVX512F-NEXT:    addq %rsi, %rdi
3543; AVX512F-NEXT:    movq %r11, %rax
3544; AVX512F-NEXT:    mulq %r8
3545; AVX512F-NEXT:    addq %rdi, %rdx
3546; AVX512F-NEXT:    setb %sil
3547; AVX512F-NEXT:    orb %bl, %sil
3548; AVX512F-NEXT:    orb %cl, %sil
3549; AVX512F-NEXT:    andl $1, %esi
3550; AVX512F-NEXT:    kmovw %esi, %k1
3551; AVX512F-NEXT:    kshiftlw $1, %k0, %k0
3552; AVX512F-NEXT:    korw %k0, %k1, %k1
3553; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3554; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3555; AVX512F-NEXT:    movq %r10, 16(%r14)
3556; AVX512F-NEXT:    movq %rax, (%r14)
3557; AVX512F-NEXT:    movq %r15, 24(%r14)
3558; AVX512F-NEXT:    movq %rdx, 8(%r14)
3559; AVX512F-NEXT:    popq %rbx
3560; AVX512F-NEXT:    popq %r12
3561; AVX512F-NEXT:    popq %r13
3562; AVX512F-NEXT:    popq %r14
3563; AVX512F-NEXT:    popq %r15
3564; AVX512F-NEXT:    popq %rbp
3565; AVX512F-NEXT:    retq
3566;
3567; AVX512BW-LABEL: umulo_v2i128:
3568; AVX512BW:       # %bb.0:
3569; AVX512BW-NEXT:    pushq %rbp
3570; AVX512BW-NEXT:    pushq %r15
3571; AVX512BW-NEXT:    pushq %r14
3572; AVX512BW-NEXT:    pushq %r13
3573; AVX512BW-NEXT:    pushq %r12
3574; AVX512BW-NEXT:    pushq %rbx
3575; AVX512BW-NEXT:    movq %rcx, %rax
3576; AVX512BW-NEXT:    movq %rdx, %r12
3577; AVX512BW-NEXT:    movq %rdi, %r11
3578; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r14
3579; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r15
3580; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
3581; AVX512BW-NEXT:    testq %r10, %r10
3582; AVX512BW-NEXT:    setne %dl
3583; AVX512BW-NEXT:    testq %rcx, %rcx
3584; AVX512BW-NEXT:    setne %r13b
3585; AVX512BW-NEXT:    andb %dl, %r13b
3586; AVX512BW-NEXT:    mulq %r15
3587; AVX512BW-NEXT:    movq %rax, %rdi
3588; AVX512BW-NEXT:    seto %bpl
3589; AVX512BW-NEXT:    movq %r10, %rax
3590; AVX512BW-NEXT:    mulq %r12
3591; AVX512BW-NEXT:    movq %rax, %rbx
3592; AVX512BW-NEXT:    seto %cl
3593; AVX512BW-NEXT:    orb %bpl, %cl
3594; AVX512BW-NEXT:    addq %rdi, %rbx
3595; AVX512BW-NEXT:    movq %r12, %rax
3596; AVX512BW-NEXT:    mulq %r15
3597; AVX512BW-NEXT:    movq %rax, %r10
3598; AVX512BW-NEXT:    movq %rdx, %r15
3599; AVX512BW-NEXT:    addq %rbx, %r15
3600; AVX512BW-NEXT:    setb %al
3601; AVX512BW-NEXT:    orb %cl, %al
3602; AVX512BW-NEXT:    orb %r13b, %al
3603; AVX512BW-NEXT:    kmovd %eax, %k0
3604; AVX512BW-NEXT:    testq %r9, %r9
3605; AVX512BW-NEXT:    setne %al
3606; AVX512BW-NEXT:    testq %rsi, %rsi
3607; AVX512BW-NEXT:    setne %cl
3608; AVX512BW-NEXT:    andb %al, %cl
3609; AVX512BW-NEXT:    movq %rsi, %rax
3610; AVX512BW-NEXT:    mulq %r8
3611; AVX512BW-NEXT:    movq %rax, %rsi
3612; AVX512BW-NEXT:    seto %bpl
3613; AVX512BW-NEXT:    movq %r9, %rax
3614; AVX512BW-NEXT:    mulq %r11
3615; AVX512BW-NEXT:    movq %rax, %rdi
3616; AVX512BW-NEXT:    seto %bl
3617; AVX512BW-NEXT:    orb %bpl, %bl
3618; AVX512BW-NEXT:    addq %rsi, %rdi
3619; AVX512BW-NEXT:    movq %r11, %rax
3620; AVX512BW-NEXT:    mulq %r8
3621; AVX512BW-NEXT:    addq %rdi, %rdx
3622; AVX512BW-NEXT:    setb %sil
3623; AVX512BW-NEXT:    orb %bl, %sil
3624; AVX512BW-NEXT:    orb %cl, %sil
3625; AVX512BW-NEXT:    andl $1, %esi
3626; AVX512BW-NEXT:    kmovw %esi, %k1
3627; AVX512BW-NEXT:    kshiftlw $1, %k0, %k0
3628; AVX512BW-NEXT:    korw %k0, %k1, %k1
3629; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
3630; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
3631; AVX512BW-NEXT:    movq %r10, 16(%r14)
3632; AVX512BW-NEXT:    movq %rax, (%r14)
3633; AVX512BW-NEXT:    movq %r15, 24(%r14)
3634; AVX512BW-NEXT:    movq %rdx, 8(%r14)
3635; AVX512BW-NEXT:    popq %rbx
3636; AVX512BW-NEXT:    popq %r12
3637; AVX512BW-NEXT:    popq %r13
3638; AVX512BW-NEXT:    popq %r14
3639; AVX512BW-NEXT:    popq %r15
3640; AVX512BW-NEXT:    popq %rbp
3641; AVX512BW-NEXT:    retq
3642  %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
3643  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
3644  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
3645  %res = sext <2 x i1> %obit to <2 x i32>
3646  store <2 x i128> %val, <2 x i128>* %p2
3647  ret <2 x i32> %res
3648}
3649