1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
7
8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind  {
9; SSE2-LABEL: mul_v16i8c:
10; SSE2:       # %bb.0: # %entry
11; SSE2-NEXT:    movdqa %xmm0, %xmm1
12; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
13; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
14; SSE2-NEXT:    pmullw %xmm2, %xmm1
15; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
16; SSE2-NEXT:    pand %xmm3, %xmm1
17; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
18; SSE2-NEXT:    pmullw %xmm2, %xmm0
19; SSE2-NEXT:    pand %xmm3, %xmm0
20; SSE2-NEXT:    packuswb %xmm1, %xmm0
21; SSE2-NEXT:    retq
22;
23; SSE41-LABEL: mul_v16i8c:
24; SSE41:       # %bb.0: # %entry
25; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
26; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
27; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
28; SSE41-NEXT:    pmullw %xmm2, %xmm0
29; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
30; SSE41-NEXT:    pand %xmm3, %xmm0
31; SSE41-NEXT:    pmullw %xmm2, %xmm1
32; SSE41-NEXT:    pand %xmm3, %xmm1
33; SSE41-NEXT:    packuswb %xmm0, %xmm1
34; SSE41-NEXT:    movdqa %xmm1, %xmm0
35; SSE41-NEXT:    retq
36;
37; AVX2-LABEL: mul_v16i8c:
38; AVX2:       # %bb.0: # %entry
39; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
40; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
41; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
42; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
43; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
44; AVX2-NEXT:    vzeroupper
45; AVX2-NEXT:    retq
46;
47; AVX512F-LABEL: mul_v16i8c:
48; AVX512F:       # %bb.0: # %entry
49; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
50; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
51; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
52; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
53; AVX512F-NEXT:    vzeroupper
54; AVX512F-NEXT:    retq
55;
56; AVX512BW-LABEL: mul_v16i8c:
57; AVX512BW:       # %bb.0: # %entry
58; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
59; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
60; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
61; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
62; AVX512BW-NEXT:    vzeroupper
63; AVX512BW-NEXT:    retq
64entry:
65  %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
66  ret <16 x i8> %A
67}
68
69define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind  {
70; SSE-LABEL: mul_v8i16c:
71; SSE:       # %bb.0: # %entry
72; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
73; SSE-NEXT:    retq
74;
75; AVX-LABEL: mul_v8i16c:
76; AVX:       # %bb.0: # %entry
77; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
78; AVX-NEXT:    retq
79entry:
80  %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
81  ret <8 x i16> %A
82}
83
84define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind  {
85; SSE2-LABEL: mul_v4i32c:
86; SSE2:       # %bb.0: # %entry
87; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117]
88; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
89; SSE2-NEXT:    pmuludq %xmm1, %xmm0
90; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
91; SSE2-NEXT:    pmuludq %xmm1, %xmm2
92; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
93; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
94; SSE2-NEXT:    retq
95;
96; SSE41-LABEL: mul_v4i32c:
97; SSE41:       # %bb.0: # %entry
98; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
99; SSE41-NEXT:    retq
100;
101; AVX-LABEL: mul_v4i32c:
102; AVX:       # %bb.0: # %entry
103; AVX-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [117,117,117,117]
104; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
105; AVX-NEXT:    retq
106entry:
107  %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 >
108  ret <4 x i32> %A
109}
110
111define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind  {
112; SSE-LABEL: mul_v2i64c:
113; SSE:       # %bb.0: # %entry
114; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [117,117]
115; SSE-NEXT:    movdqa %xmm0, %xmm2
116; SSE-NEXT:    pmuludq %xmm1, %xmm2
117; SSE-NEXT:    psrlq $32, %xmm0
118; SSE-NEXT:    pmuludq %xmm1, %xmm0
119; SSE-NEXT:    psllq $32, %xmm0
120; SSE-NEXT:    paddq %xmm2, %xmm0
121; SSE-NEXT:    retq
122;
123; AVX-LABEL: mul_v2i64c:
124; AVX:       # %bb.0: # %entry
125; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [117,117]
126; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
127; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
128; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
129; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
130; AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
131; AVX-NEXT:    retq
132entry:
133  %A = mul <2 x i64> %i, < i64 117, i64 117 >
134  ret <2 x i64> %A
135}
136
137define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind  {
138; SSE2-LABEL: mul_v16i8:
139; SSE2:       # %bb.0: # %entry
140; SSE2-NEXT:    movdqa %xmm1, %xmm2
141; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
142; SSE2-NEXT:    movdqa %xmm0, %xmm3
143; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
144; SSE2-NEXT:    pmullw %xmm2, %xmm3
145; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
146; SSE2-NEXT:    pand %xmm2, %xmm3
147; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
148; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
149; SSE2-NEXT:    pmullw %xmm1, %xmm0
150; SSE2-NEXT:    pand %xmm2, %xmm0
151; SSE2-NEXT:    packuswb %xmm3, %xmm0
152; SSE2-NEXT:    retq
153;
154; SSE41-LABEL: mul_v16i8:
155; SSE41:       # %bb.0: # %entry
156; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
157; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
158; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
159; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
160; SSE41-NEXT:    pmullw %xmm1, %xmm0
161; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
162; SSE41-NEXT:    pand %xmm1, %xmm0
163; SSE41-NEXT:    pmullw %xmm3, %xmm2
164; SSE41-NEXT:    pand %xmm1, %xmm2
165; SSE41-NEXT:    packuswb %xmm0, %xmm2
166; SSE41-NEXT:    movdqa %xmm2, %xmm0
167; SSE41-NEXT:    retq
168;
169; AVX2-LABEL: mul_v16i8:
170; AVX2:       # %bb.0: # %entry
171; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
172; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
173; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
174; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
175; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
176; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
177; AVX2-NEXT:    vzeroupper
178; AVX2-NEXT:    retq
179;
180; AVX512F-LABEL: mul_v16i8:
181; AVX512F:       # %bb.0: # %entry
182; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
183; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
184; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
185; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
186; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
187; AVX512F-NEXT:    vzeroupper
188; AVX512F-NEXT:    retq
189;
190; AVX512BW-LABEL: mul_v16i8:
191; AVX512BW:       # %bb.0: # %entry
192; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
193; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
194; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
195; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
196; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
197; AVX512BW-NEXT:    vzeroupper
198; AVX512BW-NEXT:    retq
199entry:
200  %A = mul <16 x i8> %i, %j
201  ret <16 x i8> %A
202}
203
204define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind  {
205; SSE-LABEL: mul_v8i16:
206; SSE:       # %bb.0: # %entry
207; SSE-NEXT:    pmullw %xmm1, %xmm0
208; SSE-NEXT:    retq
209;
210; AVX-LABEL: mul_v8i16:
211; AVX:       # %bb.0: # %entry
212; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
213; AVX-NEXT:    retq
214entry:
215  %A = mul <8 x i16> %i, %j
216  ret <8 x i16> %A
217}
218
219define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind  {
220; SSE2-LABEL: mul_v4i32:
221; SSE2:       # %bb.0: # %entry
222; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
223; SSE2-NEXT:    pmuludq %xmm1, %xmm0
224; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
225; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
226; SSE2-NEXT:    pmuludq %xmm2, %xmm1
227; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
228; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
229; SSE2-NEXT:    retq
230;
231; SSE41-LABEL: mul_v4i32:
232; SSE41:       # %bb.0: # %entry
233; SSE41-NEXT:    pmulld %xmm1, %xmm0
234; SSE41-NEXT:    retq
235;
236; AVX-LABEL: mul_v4i32:
237; AVX:       # %bb.0: # %entry
238; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
239; AVX-NEXT:    retq
240entry:
241  %A = mul <4 x i32> %i, %j
242  ret <4 x i32> %A
243}
244
245define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind  {
246; SSE-LABEL: mul_v2i64:
247; SSE:       # %bb.0: # %entry
248; SSE-NEXT:    movdqa %xmm0, %xmm2
249; SSE-NEXT:    psrlq $32, %xmm2
250; SSE-NEXT:    pmuludq %xmm1, %xmm2
251; SSE-NEXT:    movdqa %xmm1, %xmm3
252; SSE-NEXT:    psrlq $32, %xmm3
253; SSE-NEXT:    pmuludq %xmm0, %xmm3
254; SSE-NEXT:    paddq %xmm2, %xmm3
255; SSE-NEXT:    psllq $32, %xmm3
256; SSE-NEXT:    pmuludq %xmm1, %xmm0
257; SSE-NEXT:    paddq %xmm3, %xmm0
258; SSE-NEXT:    retq
259;
260; AVX-LABEL: mul_v2i64:
261; AVX:       # %bb.0: # %entry
262; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm2
263; AVX-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
264; AVX-NEXT:    vpsrlq $32, %xmm1, %xmm3
265; AVX-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
266; AVX-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
267; AVX-NEXT:    vpsllq $32, %xmm2, %xmm2
268; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
269; AVX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
270; AVX-NEXT:    retq
271entry:
272  %A = mul <2 x i64> %i, %j
273  ret <2 x i64> %A
274}
275
276declare void @foo()
277
278define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind  {
279; SSE2-LABEL: mul_v4i32spill:
280; SSE2:       # %bb.0: # %entry
281; SSE2-NEXT:    subq $40, %rsp
282; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
283; SSE2-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
284; SSE2-NEXT:    callq foo
285; SSE2-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
286; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
287; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
288; SSE2-NEXT:    pmuludq %xmm2, %xmm0
289; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
290; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
291; SSE2-NEXT:    pmuludq %xmm1, %xmm2
292; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
293; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
294; SSE2-NEXT:    addq $40, %rsp
295; SSE2-NEXT:    retq
296;
297; SSE41-LABEL: mul_v4i32spill:
298; SSE41:       # %bb.0: # %entry
299; SSE41-NEXT:    subq $40, %rsp
300; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
301; SSE41-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
302; SSE41-NEXT:    callq foo
303; SSE41-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
304; SSE41-NEXT:    pmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
305; SSE41-NEXT:    addq $40, %rsp
306; SSE41-NEXT:    retq
307;
308; AVX-LABEL: mul_v4i32spill:
309; AVX:       # %bb.0: # %entry
310; AVX-NEXT:    subq $40, %rsp
311; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
312; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
313; AVX-NEXT:    callq foo
314; AVX-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
315; AVX-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
316; AVX-NEXT:    addq $40, %rsp
317; AVX-NEXT:    retq
318entry:
319  ; Use a call to force spills.
320  call void @foo()
321  %A = mul <4 x i32> %i, %j
322  ret <4 x i32> %A
323}
324
325define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind  {
326; SSE-LABEL: mul_v2i64spill:
327; SSE:       # %bb.0: # %entry
328; SSE-NEXT:    subq $40, %rsp
329; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
330; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
331; SSE-NEXT:    callq foo
332; SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
333; SSE-NEXT:    movdqa %xmm0, %xmm2
334; SSE-NEXT:    psrlq $32, %xmm2
335; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
336; SSE-NEXT:    pmuludq %xmm3, %xmm2
337; SSE-NEXT:    movdqa %xmm3, %xmm1
338; SSE-NEXT:    psrlq $32, %xmm1
339; SSE-NEXT:    pmuludq %xmm0, %xmm1
340; SSE-NEXT:    paddq %xmm2, %xmm1
341; SSE-NEXT:    psllq $32, %xmm1
342; SSE-NEXT:    pmuludq %xmm3, %xmm0
343; SSE-NEXT:    paddq %xmm1, %xmm0
344; SSE-NEXT:    addq $40, %rsp
345; SSE-NEXT:    retq
346;
347; AVX-LABEL: mul_v2i64spill:
348; AVX:       # %bb.0: # %entry
349; AVX-NEXT:    subq $40, %rsp
350; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
351; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
352; AVX-NEXT:    callq foo
353; AVX-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
354; AVX-NEXT:    vpsrlq $32, %xmm3, %xmm0
355; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
356; AVX-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
357; AVX-NEXT:    vpsrlq $32, %xmm2, %xmm1
358; AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
359; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
360; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
361; AVX-NEXT:    vpmuludq %xmm2, %xmm3, %xmm1
362; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
363; AVX-NEXT:    addq $40, %rsp
364; AVX-NEXT:    retq
365entry:
366  ; Use a call to force spills.
367  call void @foo()
368  %A = mul <2 x i64> %i, %j
369  ret <2 x i64> %A
370}
371
372define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind  {
373; SSE2-LABEL: mul_v32i8c:
374; SSE2:       # %bb.0: # %entry
375; SSE2-NEXT:    movdqa %xmm0, %xmm2
376; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
377; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [117,117,117,117,117,117,117,117]
378; SSE2-NEXT:    pmullw %xmm3, %xmm2
379; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
380; SSE2-NEXT:    pand %xmm4, %xmm2
381; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
382; SSE2-NEXT:    pmullw %xmm3, %xmm0
383; SSE2-NEXT:    pand %xmm4, %xmm0
384; SSE2-NEXT:    packuswb %xmm2, %xmm0
385; SSE2-NEXT:    movdqa %xmm1, %xmm2
386; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
387; SSE2-NEXT:    pmullw %xmm3, %xmm2
388; SSE2-NEXT:    pand %xmm4, %xmm2
389; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
390; SSE2-NEXT:    pmullw %xmm3, %xmm1
391; SSE2-NEXT:    pand %xmm4, %xmm1
392; SSE2-NEXT:    packuswb %xmm2, %xmm1
393; SSE2-NEXT:    retq
394;
395; SSE41-LABEL: mul_v32i8c:
396; SSE41:       # %bb.0: # %entry
397; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
398; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
399; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117]
400; SSE41-NEXT:    pmullw %xmm4, %xmm0
401; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
402; SSE41-NEXT:    pand %xmm5, %xmm0
403; SSE41-NEXT:    pmullw %xmm4, %xmm2
404; SSE41-NEXT:    pand %xmm5, %xmm2
405; SSE41-NEXT:    packuswb %xmm0, %xmm2
406; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
407; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
408; SSE41-NEXT:    pmullw %xmm4, %xmm1
409; SSE41-NEXT:    pand %xmm5, %xmm1
410; SSE41-NEXT:    pmullw %xmm4, %xmm3
411; SSE41-NEXT:    pand %xmm5, %xmm3
412; SSE41-NEXT:    packuswb %xmm1, %xmm3
413; SSE41-NEXT:    movdqa %xmm2, %xmm0
414; SSE41-NEXT:    movdqa %xmm3, %xmm1
415; SSE41-NEXT:    retq
416;
417; AVX2-LABEL: mul_v32i8c:
418; AVX2:       # %bb.0: # %entry
419; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
420; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
421; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
422; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
423; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
424; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
425; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
426; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
427; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
428; AVX2-NEXT:    retq
429;
430; AVX512F-LABEL: mul_v32i8c:
431; AVX512F:       # %bb.0: # %entry
432; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
433; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
434; AVX512F-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
435; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
436; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
437; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
438; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
439; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
440; AVX512F-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
441; AVX512F-NEXT:    retq
442;
443; AVX512BW-LABEL: mul_v32i8c:
444; AVX512BW:       # %bb.0: # %entry
445; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
446; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm0, %zmm0
447; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
448; AVX512BW-NEXT:    retq
449entry:
450  %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
451  ret <32 x i8> %A
452}
453
454define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind  {
455; SSE-LABEL: mul_v16i16c:
456; SSE:       # %bb.0: # %entry
457; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
458; SSE-NEXT:    pmullw %xmm2, %xmm0
459; SSE-NEXT:    pmullw %xmm2, %xmm1
460; SSE-NEXT:    retq
461;
462; AVX-LABEL: mul_v16i16c:
463; AVX:       # %bb.0: # %entry
464; AVX-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
465; AVX-NEXT:    retq
466entry:
467  %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
468  ret <16 x i16> %A
469}
470
471define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind  {
472; SSE2-LABEL: mul_v8i32c:
473; SSE2:       # %bb.0: # %entry
474; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
475; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
476; SSE2-NEXT:    pmuludq %xmm2, %xmm0
477; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
478; SSE2-NEXT:    pmuludq %xmm2, %xmm3
479; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
480; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
481; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
482; SSE2-NEXT:    pmuludq %xmm2, %xmm1
483; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
484; SSE2-NEXT:    pmuludq %xmm2, %xmm3
485; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
486; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
487; SSE2-NEXT:    retq
488;
489; SSE41-LABEL: mul_v8i32c:
490; SSE41:       # %bb.0: # %entry
491; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
492; SSE41-NEXT:    pmulld %xmm2, %xmm0
493; SSE41-NEXT:    pmulld %xmm2, %xmm1
494; SSE41-NEXT:    retq
495;
496; AVX-LABEL: mul_v8i32c:
497; AVX:       # %bb.0: # %entry
498; AVX-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [117,117,117,117,117,117,117,117]
499; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
500; AVX-NEXT:    retq
501entry:
502  %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 >
503  ret <8 x i32> %A
504}
505
506define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind  {
507; SSE-LABEL: mul_v4i64c:
508; SSE:       # %bb.0: # %entry
509; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117]
510; SSE-NEXT:    movdqa %xmm0, %xmm3
511; SSE-NEXT:    pmuludq %xmm2, %xmm3
512; SSE-NEXT:    psrlq $32, %xmm0
513; SSE-NEXT:    pmuludq %xmm2, %xmm0
514; SSE-NEXT:    psllq $32, %xmm0
515; SSE-NEXT:    paddq %xmm3, %xmm0
516; SSE-NEXT:    movdqa %xmm1, %xmm3
517; SSE-NEXT:    pmuludq %xmm2, %xmm3
518; SSE-NEXT:    psrlq $32, %xmm1
519; SSE-NEXT:    pmuludq %xmm2, %xmm1
520; SSE-NEXT:    psllq $32, %xmm1
521; SSE-NEXT:    paddq %xmm3, %xmm1
522; SSE-NEXT:    retq
523;
524; AVX-LABEL: mul_v4i64c:
525; AVX:       # %bb.0: # %entry
526; AVX-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [117,117,117,117]
527; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
528; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm0
529; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
530; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
531; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
532; AVX-NEXT:    retq
533entry:
534  %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 >
535  ret <4 x i64> %A
536}
537
538define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind  {
539; SSE2-LABEL: mul_v32i8:
540; SSE2:       # %bb.0: # %entry
541; SSE2-NEXT:    movdqa %xmm2, %xmm4
542; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
543; SSE2-NEXT:    movdqa %xmm0, %xmm5
544; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
545; SSE2-NEXT:    pmullw %xmm4, %xmm5
546; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
547; SSE2-NEXT:    pand %xmm4, %xmm5
548; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
549; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
550; SSE2-NEXT:    pmullw %xmm2, %xmm0
551; SSE2-NEXT:    pand %xmm4, %xmm0
552; SSE2-NEXT:    packuswb %xmm5, %xmm0
553; SSE2-NEXT:    movdqa %xmm3, %xmm2
554; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
555; SSE2-NEXT:    movdqa %xmm1, %xmm5
556; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
557; SSE2-NEXT:    pmullw %xmm2, %xmm5
558; SSE2-NEXT:    pand %xmm4, %xmm5
559; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
560; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
561; SSE2-NEXT:    pmullw %xmm3, %xmm1
562; SSE2-NEXT:    pand %xmm4, %xmm1
563; SSE2-NEXT:    packuswb %xmm5, %xmm1
564; SSE2-NEXT:    retq
565;
566; SSE41-LABEL: mul_v32i8:
567; SSE41:       # %bb.0: # %entry
568; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
569; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
570; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
571; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
572; SSE41-NEXT:    pmullw %xmm2, %xmm0
573; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
574; SSE41-NEXT:    pand %xmm6, %xmm0
575; SSE41-NEXT:    pmullw %xmm5, %xmm4
576; SSE41-NEXT:    pand %xmm6, %xmm4
577; SSE41-NEXT:    packuswb %xmm0, %xmm4
578; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
579; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
580; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
581; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
582; SSE41-NEXT:    pmullw %xmm3, %xmm1
583; SSE41-NEXT:    pand %xmm6, %xmm1
584; SSE41-NEXT:    pmullw %xmm0, %xmm2
585; SSE41-NEXT:    pand %xmm6, %xmm2
586; SSE41-NEXT:    packuswb %xmm1, %xmm2
587; SSE41-NEXT:    movdqa %xmm4, %xmm0
588; SSE41-NEXT:    movdqa %xmm2, %xmm1
589; SSE41-NEXT:    retq
590;
591; AVX2-LABEL: mul_v32i8:
592; AVX2:       # %bb.0: # %entry
593; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
594; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
595; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
596; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
597; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
598; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
599; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
600; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
601; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
602; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
603; AVX2-NEXT:    retq
604;
605; AVX512F-LABEL: mul_v32i8:
606; AVX512F:       # %bb.0: # %entry
607; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
608; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
609; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
610; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
611; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
612; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
613; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
614; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
615; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
616; AVX512F-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
617; AVX512F-NEXT:    retq
618;
619; AVX512BW-LABEL: mul_v32i8:
620; AVX512BW:       # %bb.0: # %entry
621; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
622; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
623; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
624; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
625; AVX512BW-NEXT:    retq
626entry:
627  %A = mul <32 x i8> %i, %j
628  ret <32 x i8> %A
629}
630
631define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind  {
632; SSE-LABEL: mul_v16i16:
633; SSE:       # %bb.0: # %entry
634; SSE-NEXT:    pmullw %xmm2, %xmm0
635; SSE-NEXT:    pmullw %xmm3, %xmm1
636; SSE-NEXT:    retq
637;
638; AVX-LABEL: mul_v16i16:
639; AVX:       # %bb.0: # %entry
640; AVX-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
641; AVX-NEXT:    retq
642entry:
643  %A = mul <16 x i16> %i, %j
644  ret <16 x i16> %A
645}
646
647define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind  {
648; SSE2-LABEL: mul_v8i32:
649; SSE2:       # %bb.0: # %entry
650; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
651; SSE2-NEXT:    pmuludq %xmm2, %xmm0
652; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
653; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
654; SSE2-NEXT:    pmuludq %xmm4, %xmm2
655; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
656; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
657; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
658; SSE2-NEXT:    pmuludq %xmm3, %xmm1
659; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
660; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
661; SSE2-NEXT:    pmuludq %xmm2, %xmm3
662; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
663; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
664; SSE2-NEXT:    retq
665;
666; SSE41-LABEL: mul_v8i32:
667; SSE41:       # %bb.0: # %entry
668; SSE41-NEXT:    pmulld %xmm2, %xmm0
669; SSE41-NEXT:    pmulld %xmm3, %xmm1
670; SSE41-NEXT:    retq
671;
672; AVX-LABEL: mul_v8i32:
673; AVX:       # %bb.0: # %entry
674; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
675; AVX-NEXT:    retq
676entry:
677  %A = mul <8 x i32> %i, %j
678  ret <8 x i32> %A
679}
680
681define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind  {
682; SSE-LABEL: mul_v4i64:
683; SSE:       # %bb.0: # %entry
684; SSE-NEXT:    movdqa %xmm0, %xmm4
685; SSE-NEXT:    psrlq $32, %xmm4
686; SSE-NEXT:    pmuludq %xmm2, %xmm4
687; SSE-NEXT:    movdqa %xmm2, %xmm5
688; SSE-NEXT:    psrlq $32, %xmm5
689; SSE-NEXT:    pmuludq %xmm0, %xmm5
690; SSE-NEXT:    paddq %xmm4, %xmm5
691; SSE-NEXT:    psllq $32, %xmm5
692; SSE-NEXT:    pmuludq %xmm2, %xmm0
693; SSE-NEXT:    paddq %xmm5, %xmm0
694; SSE-NEXT:    movdqa %xmm1, %xmm2
695; SSE-NEXT:    psrlq $32, %xmm2
696; SSE-NEXT:    pmuludq %xmm3, %xmm2
697; SSE-NEXT:    movdqa %xmm3, %xmm4
698; SSE-NEXT:    psrlq $32, %xmm4
699; SSE-NEXT:    pmuludq %xmm1, %xmm4
700; SSE-NEXT:    paddq %xmm2, %xmm4
701; SSE-NEXT:    psllq $32, %xmm4
702; SSE-NEXT:    pmuludq %xmm3, %xmm1
703; SSE-NEXT:    paddq %xmm4, %xmm1
704; SSE-NEXT:    retq
705;
706; AVX-LABEL: mul_v4i64:
707; AVX:       # %bb.0: # %entry
708; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm2
709; AVX-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2
710; AVX-NEXT:    vpsrlq $32, %ymm1, %ymm3
711; AVX-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3
712; AVX-NEXT:    vpaddq %ymm2, %ymm3, %ymm2
713; AVX-NEXT:    vpsllq $32, %ymm2, %ymm2
714; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
715; AVX-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
716; AVX-NEXT:    retq
717entry:
718  %A = mul <4 x i64> %i, %j
719  ret <4 x i64> %A
720}
721
722define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
723; SSE2-LABEL: mul_v64i8c:
724; SSE2:       # %bb.0: # %entry
725; SSE2-NEXT:    movdqa %xmm0, %xmm6
726; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
727; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117]
728; SSE2-NEXT:    pmullw %xmm4, %xmm6
729; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
730; SSE2-NEXT:    pand %xmm5, %xmm6
731; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
732; SSE2-NEXT:    pmullw %xmm4, %xmm0
733; SSE2-NEXT:    pand %xmm5, %xmm0
734; SSE2-NEXT:    packuswb %xmm6, %xmm0
735; SSE2-NEXT:    movdqa %xmm1, %xmm6
736; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
737; SSE2-NEXT:    pmullw %xmm4, %xmm6
738; SSE2-NEXT:    pand %xmm5, %xmm6
739; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
740; SSE2-NEXT:    pmullw %xmm4, %xmm1
741; SSE2-NEXT:    pand %xmm5, %xmm1
742; SSE2-NEXT:    packuswb %xmm6, %xmm1
743; SSE2-NEXT:    movdqa %xmm2, %xmm6
744; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
745; SSE2-NEXT:    pmullw %xmm4, %xmm6
746; SSE2-NEXT:    pand %xmm5, %xmm6
747; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
748; SSE2-NEXT:    pmullw %xmm4, %xmm2
749; SSE2-NEXT:    pand %xmm5, %xmm2
750; SSE2-NEXT:    packuswb %xmm6, %xmm2
751; SSE2-NEXT:    movdqa %xmm3, %xmm6
752; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
753; SSE2-NEXT:    pmullw %xmm4, %xmm6
754; SSE2-NEXT:    pand %xmm5, %xmm6
755; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
756; SSE2-NEXT:    pmullw %xmm4, %xmm3
757; SSE2-NEXT:    pand %xmm5, %xmm3
758; SSE2-NEXT:    packuswb %xmm6, %xmm3
759; SSE2-NEXT:    retq
760;
761; SSE41-LABEL: mul_v64i8c:
762; SSE41:       # %bb.0: # %entry
763; SSE41-NEXT:    movdqa %xmm1, %xmm4
764; SSE41-NEXT:    movdqa %xmm0, %xmm1
765; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
766; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
767; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [117,117,117,117,117,117,117,117]
768; SSE41-NEXT:    pmullw %xmm6, %xmm1
769; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
770; SSE41-NEXT:    pand %xmm7, %xmm1
771; SSE41-NEXT:    pmullw %xmm6, %xmm0
772; SSE41-NEXT:    pand %xmm7, %xmm0
773; SSE41-NEXT:    packuswb %xmm1, %xmm0
774; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
775; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
776; SSE41-NEXT:    pmullw %xmm6, %xmm4
777; SSE41-NEXT:    pand %xmm7, %xmm4
778; SSE41-NEXT:    pmullw %xmm6, %xmm1
779; SSE41-NEXT:    pand %xmm7, %xmm1
780; SSE41-NEXT:    packuswb %xmm4, %xmm1
781; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
782; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
783; SSE41-NEXT:    pmullw %xmm6, %xmm2
784; SSE41-NEXT:    pand %xmm7, %xmm2
785; SSE41-NEXT:    pmullw %xmm6, %xmm4
786; SSE41-NEXT:    pand %xmm7, %xmm4
787; SSE41-NEXT:    packuswb %xmm2, %xmm4
788; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
789; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
790; SSE41-NEXT:    pmullw %xmm6, %xmm3
791; SSE41-NEXT:    pand %xmm7, %xmm3
792; SSE41-NEXT:    pmullw %xmm6, %xmm5
793; SSE41-NEXT:    pand %xmm7, %xmm5
794; SSE41-NEXT:    packuswb %xmm3, %xmm5
795; SSE41-NEXT:    movdqa %xmm4, %xmm2
796; SSE41-NEXT:    movdqa %xmm5, %xmm3
797; SSE41-NEXT:    retq
798;
799; AVX2-LABEL: mul_v64i8c:
800; AVX2:       # %bb.0: # %entry
801; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
802; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
803; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
804; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
805; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
806; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
807; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
808; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
809; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
810; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
811; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
812; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
813; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
814; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
815; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
816; AVX2-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
817; AVX2-NEXT:    retq
818;
819; AVX512F-LABEL: mul_v64i8c:
820; AVX512F:       # %bb.0: # %entry
821; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
822; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
823; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
824; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
825; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
826; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
827; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
828; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
829; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
830; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
831; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
832; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
833; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
834; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
835; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
836; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
837; AVX512F-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
838; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
839; AVX512F-NEXT:    retq
840;
841; AVX512BW-LABEL: mul_v64i8c:
842; AVX512BW:       # %bb.0: # %entry
843; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
844; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
845; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
846; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
847; AVX512BW-NEXT:    vpandq %zmm3, %zmm1, %zmm1
848; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
849; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
850; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
851; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
852; AVX512BW-NEXT:    retq
853entry:
854  %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
855  ret <64 x i8> %A
856}
857
858define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
859; SSE2-LABEL: mul_v64i8:
860; SSE2:       # %bb.0: # %entry
861; SSE2-NEXT:    movdqa %xmm4, %xmm8
862; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
863; SSE2-NEXT:    movdqa %xmm0, %xmm9
864; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
865; SSE2-NEXT:    pmullw %xmm8, %xmm9
866; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
867; SSE2-NEXT:    pand %xmm8, %xmm9
868; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
869; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
870; SSE2-NEXT:    pmullw %xmm4, %xmm0
871; SSE2-NEXT:    pand %xmm8, %xmm0
872; SSE2-NEXT:    packuswb %xmm9, %xmm0
873; SSE2-NEXT:    movdqa %xmm5, %xmm9
874; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
875; SSE2-NEXT:    movdqa %xmm1, %xmm4
876; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
877; SSE2-NEXT:    pmullw %xmm9, %xmm4
878; SSE2-NEXT:    pand %xmm8, %xmm4
879; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
880; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
881; SSE2-NEXT:    pmullw %xmm5, %xmm1
882; SSE2-NEXT:    pand %xmm8, %xmm1
883; SSE2-NEXT:    packuswb %xmm4, %xmm1
884; SSE2-NEXT:    movdqa %xmm6, %xmm4
885; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
886; SSE2-NEXT:    movdqa %xmm2, %xmm5
887; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
888; SSE2-NEXT:    pmullw %xmm4, %xmm5
889; SSE2-NEXT:    pand %xmm8, %xmm5
890; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
891; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
892; SSE2-NEXT:    pmullw %xmm6, %xmm2
893; SSE2-NEXT:    pand %xmm8, %xmm2
894; SSE2-NEXT:    packuswb %xmm5, %xmm2
895; SSE2-NEXT:    movdqa %xmm7, %xmm4
896; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
897; SSE2-NEXT:    movdqa %xmm3, %xmm5
898; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
899; SSE2-NEXT:    pmullw %xmm4, %xmm5
900; SSE2-NEXT:    pand %xmm8, %xmm5
901; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
902; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
903; SSE2-NEXT:    pmullw %xmm7, %xmm3
904; SSE2-NEXT:    pand %xmm8, %xmm3
905; SSE2-NEXT:    packuswb %xmm5, %xmm3
906; SSE2-NEXT:    retq
907;
908; SSE41-LABEL: mul_v64i8:
909; SSE41:       # %bb.0: # %entry
910; SSE41-NEXT:    movdqa %xmm1, %xmm8
911; SSE41-NEXT:    movdqa %xmm0, %xmm1
912; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
913; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
914; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
915; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
916; SSE41-NEXT:    pmullw %xmm4, %xmm1
917; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
918; SSE41-NEXT:    pand %xmm9, %xmm1
919; SSE41-NEXT:    pmullw %xmm10, %xmm0
920; SSE41-NEXT:    pand %xmm9, %xmm0
921; SSE41-NEXT:    packuswb %xmm1, %xmm0
922; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
923; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
924; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
925; SSE41-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
926; SSE41-NEXT:    pmullw %xmm5, %xmm8
927; SSE41-NEXT:    pand %xmm9, %xmm8
928; SSE41-NEXT:    pmullw %xmm4, %xmm1
929; SSE41-NEXT:    pand %xmm9, %xmm1
930; SSE41-NEXT:    packuswb %xmm8, %xmm1
931; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
932; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
933; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
934; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
935; SSE41-NEXT:    pmullw %xmm6, %xmm2
936; SSE41-NEXT:    pand %xmm9, %xmm2
937; SSE41-NEXT:    pmullw %xmm5, %xmm4
938; SSE41-NEXT:    pand %xmm9, %xmm4
939; SSE41-NEXT:    packuswb %xmm2, %xmm4
940; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
941; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
942; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
943; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
944; SSE41-NEXT:    pmullw %xmm7, %xmm3
945; SSE41-NEXT:    pand %xmm9, %xmm3
946; SSE41-NEXT:    pmullw %xmm2, %xmm5
947; SSE41-NEXT:    pand %xmm9, %xmm5
948; SSE41-NEXT:    packuswb %xmm3, %xmm5
949; SSE41-NEXT:    movdqa %xmm4, %xmm2
950; SSE41-NEXT:    movdqa %xmm5, %xmm3
951; SSE41-NEXT:    retq
952;
953; AVX2-LABEL: mul_v64i8:
954; AVX2:       # %bb.0: # %entry
955; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
956; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
957; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
958; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
959; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
960; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
961; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
962; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
963; AVX2-NEXT:    vpand %ymm5, %ymm0, %ymm0
964; AVX2-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
965; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
966; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
967; AVX2-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
968; AVX2-NEXT:    vpand %ymm5, %ymm2, %ymm2
969; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
970; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
971; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
972; AVX2-NEXT:    vpand %ymm5, %ymm1, %ymm1
973; AVX2-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
974; AVX2-NEXT:    retq
975;
976; AVX512F-LABEL: mul_v64i8:
977; AVX512F:       # %bb.0: # %entry
978; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
979; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
980; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
981; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
982; AVX512F-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
983; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
984; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
985; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
986; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
987; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
988; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
989; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
990; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
991; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
992; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
993; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
994; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
995; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
996; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
997; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
998; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
999; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
1000; AVX512F-NEXT:    retq
1001;
1002; AVX512BW-LABEL: mul_v64i8:
1003; AVX512BW:       # %bb.0: # %entry
1004; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
1005; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
1006; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
1007; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1008; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
1009; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
1010; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
1011; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1012; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
1013; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
1014; AVX512BW-NEXT:    retq
1015entry:
1016  %A = mul <64 x i8> %i, %j
1017  ret <64 x i8> %A
1018}
1019
1020; PR30845
1021define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) {
1022; SSE2-LABEL: mul_v4i64_zero_upper:
1023; SSE2:       # %bb.0: # %entry
1024; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
1025; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
1026; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
1027; SSE2-NEXT:    pmuludq %xmm2, %xmm0
1028; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]
1029; SSE2-NEXT:    pmuludq %xmm3, %xmm1
1030; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1031; SSE2-NEXT:    retq
1032;
1033; SSE41-LABEL: mul_v4i64_zero_upper:
1034; SSE41:       # %bb.0: # %entry
1035; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
1036; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
1037; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1038; SSE41-NEXT:    pmuludq %xmm2, %xmm0
1039; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1040; SSE41-NEXT:    pmuludq %xmm3, %xmm1
1041; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1042; SSE41-NEXT:    retq
1043;
1044; AVX-LABEL: mul_v4i64_zero_upper:
1045; AVX:       # %bb.0: # %entry
1046; AVX-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1047; AVX-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1048; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1049; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm1
1050; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1051; AVX-NEXT:    vzeroupper
1052; AVX-NEXT:    retq
1053entry:
1054  %val1a = zext <4 x i32> %val1 to <4 x i64>
1055  %val2a = zext <4 x i32> %val2 to <4 x i64>
1056  %res64 = mul <4 x i64> %val1a, %val2a
1057  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1058  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1059  ret <4 x i32> %res
1060}
1061
1062define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) {
1063; SSE2-LABEL: mul_v4i64_zero_upper_left:
1064; SSE2:       # %bb.0: # %entry
1065; SSE2-NEXT:    pxor %xmm3, %xmm3
1066; SSE2-NEXT:    movdqa %xmm0, %xmm4
1067; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
1068; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
1069; SSE2-NEXT:    movdqa %xmm0, %xmm3
1070; SSE2-NEXT:    pmuludq %xmm2, %xmm3
1071; SSE2-NEXT:    psrlq $32, %xmm2
1072; SSE2-NEXT:    pmuludq %xmm0, %xmm2
1073; SSE2-NEXT:    psllq $32, %xmm2
1074; SSE2-NEXT:    paddq %xmm3, %xmm2
1075; SSE2-NEXT:    movdqa %xmm4, %xmm0
1076; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1077; SSE2-NEXT:    psrlq $32, %xmm1
1078; SSE2-NEXT:    pmuludq %xmm4, %xmm1
1079; SSE2-NEXT:    psllq $32, %xmm1
1080; SSE2-NEXT:    paddq %xmm1, %xmm0
1081; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
1082; SSE2-NEXT:    retq
1083;
1084; SSE41-LABEL: mul_v4i64_zero_upper_left:
1085; SSE41:       # %bb.0: # %entry
1086; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
1087; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
1088; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
1089; SSE41-NEXT:    movdqa %xmm4, %xmm0
1090; SSE41-NEXT:    pmuludq %xmm1, %xmm0
1091; SSE41-NEXT:    psrlq $32, %xmm1
1092; SSE41-NEXT:    pmuludq %xmm4, %xmm1
1093; SSE41-NEXT:    psllq $32, %xmm1
1094; SSE41-NEXT:    paddq %xmm1, %xmm0
1095; SSE41-NEXT:    movdqa %xmm3, %xmm1
1096; SSE41-NEXT:    pmuludq %xmm2, %xmm1
1097; SSE41-NEXT:    psrlq $32, %xmm2
1098; SSE41-NEXT:    pmuludq %xmm3, %xmm2
1099; SSE41-NEXT:    psllq $32, %xmm2
1100; SSE41-NEXT:    paddq %xmm1, %xmm2
1101; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
1102; SSE41-NEXT:    retq
1103;
1104; AVX-LABEL: mul_v4i64_zero_upper_left:
1105; AVX:       # %bb.0: # %entry
1106; AVX-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1107; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
1108; AVX-NEXT:    vpsrlq $32, %ymm1, %ymm1
1109; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1110; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
1111; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
1112; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm1
1113; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1114; AVX-NEXT:    vzeroupper
1115; AVX-NEXT:    retq
1116entry:
1117  %val1a = zext <4 x i32> %val1 to <4 x i64>
1118  %res64 = mul <4 x i64> %val1a, %val2
1119  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1120  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1121  ret <4 x i32> %res
1122}
1123
1124define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) {
1125; SSE2-LABEL: mul_v4i64_zero_lower:
1126; SSE2:       # %bb.0: # %entry
1127; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
1128; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,1,3,3]
1129; SSE2-NEXT:    psrlq $32, %xmm2
1130; SSE2-NEXT:    pmuludq %xmm0, %xmm2
1131; SSE2-NEXT:    psrlq $32, %xmm1
1132; SSE2-NEXT:    pmuludq %xmm1, %xmm3
1133; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]
1134; SSE2-NEXT:    movaps %xmm3, %xmm0
1135; SSE2-NEXT:    retq
1136;
1137; SSE41-LABEL: mul_v4i64_zero_lower:
1138; SSE41:       # %bb.0: # %entry
1139; SSE41-NEXT:    psrlq $32, %xmm2
1140; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
1141; SSE41-NEXT:    pmuludq %xmm2, %xmm3
1142; SSE41-NEXT:    psrlq $32, %xmm1
1143; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
1144; SSE41-NEXT:    pmuludq %xmm1, %xmm0
1145; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
1146; SSE41-NEXT:    retq
1147;
1148; AVX-LABEL: mul_v4i64_zero_lower:
1149; AVX:       # %bb.0: # %entry
1150; AVX-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1151; AVX-NEXT:    vpsrlq $32, %ymm1, %ymm1
1152; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1153; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
1154; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm1
1155; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1156; AVX-NEXT:    vzeroupper
1157; AVX-NEXT:    retq
1158entry:
1159  %val1a = zext <4 x i32> %val1 to <4 x i64>
1160  %val2a = and <4 x i64> %val2, <i64 -4294967296, i64 -4294967296, i64 -4294967296, i64 -4294967296>
1161  %res64 = mul <4 x i64> %val1a, %val2a
1162  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1163  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1164  ret <4 x i32> %res
1165}
1166
1167define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {
1168; SSE2-LABEL: mul_v8i64_zero_upper:
1169; SSE2:       # %bb.0: # %entry
1170; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,1,3]
1171; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,1,3,3]
1172; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,1,1,3]
1173; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,1,3,3]
1174; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1175; SSE2-NEXT:    pmuludq %xmm4, %xmm0
1176; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1177; SSE2-NEXT:    pmuludq %xmm5, %xmm1
1178; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1179; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,1,1,3]
1180; SSE2-NEXT:    pmuludq %xmm6, %xmm1
1181; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3]
1182; SSE2-NEXT:    pmuludq %xmm7, %xmm2
1183; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
1184; SSE2-NEXT:    retq
1185;
1186; SSE41-LABEL: mul_v8i64_zero_upper:
1187; SSE41:       # %bb.0: # %entry
1188; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
1189; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,2,3,3]
1190; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
1191; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,2,3,3]
1192; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
1193; SSE41-NEXT:    pmuludq %xmm4, %xmm0
1194; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1195; SSE41-NEXT:    pmuludq %xmm5, %xmm1
1196; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
1197; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero
1198; SSE41-NEXT:    pmuludq %xmm6, %xmm1
1199; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]
1200; SSE41-NEXT:    pmuludq %xmm7, %xmm2
1201; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
1202; SSE41-NEXT:    retq
1203;
1204; AVX2-LABEL: mul_v8i64_zero_upper:
1205; AVX2:       # %bb.0: # %entry
1206; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1207; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1208; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1209; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1210; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
1211; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
1212; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1213; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1214; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm2[1,3],ymm0[1,3],ymm2[5,7],ymm0[5,7]
1215; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
1216; AVX2-NEXT:    retq
1217;
1218; AVX512-LABEL: mul_v8i64_zero_upper:
1219; AVX512:       # %bb.0: # %entry
1220; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
1221; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
1222; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
1223; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1224; AVX512-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
1225; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
1226; AVX512-NEXT:    retq
1227entry:
1228  %val1a = zext <8 x i32> %val1 to <8 x i64>
1229  %val2a = zext <8 x i32> %val2 to <8 x i64>
1230  %res64 = mul <8 x i64> %val1a, %val2a
1231  %rescast = bitcast <8 x i64> %res64 to <16 x i32>
1232  %res = shufflevector <16 x i32> %rescast, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 9, i32 11, i32 13, i32 15 >
1233  ret <8 x i32> %res
1234}
1235
1236define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
1237; SSE2-LABEL: mul_v8i64_sext:
1238; SSE2:       # %bb.0:
1239; SSE2-NEXT:    movdqa %xmm1, %xmm12
1240; SSE2-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7]
1241; SSE2-NEXT:    psrad $16, %xmm10
1242; SSE2-NEXT:    pxor %xmm8, %xmm8
1243; SSE2-NEXT:    pxor %xmm13, %xmm13
1244; SSE2-NEXT:    pcmpgtd %xmm10, %xmm13
1245; SSE2-NEXT:    movdqa %xmm10, %xmm9
1246; SSE2-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm13[2],xmm9[3],xmm13[3]
1247; SSE2-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1]
1248; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1249; SSE2-NEXT:    psrad $16, %xmm0
1250; SSE2-NEXT:    pxor %xmm15, %xmm15
1251; SSE2-NEXT:    pcmpgtd %xmm0, %xmm15
1252; SSE2-NEXT:    movdqa %xmm0, %xmm11
1253; SSE2-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm15[2],xmm11[3],xmm15[3]
1254; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1]
1255; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1256; SSE2-NEXT:    pxor %xmm14, %xmm14
1257; SSE2-NEXT:    pcmpgtd %xmm3, %xmm14
1258; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1]
1259; SSE2-NEXT:    pxor %xmm5, %xmm5
1260; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
1261; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1262; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1263; SSE2-NEXT:    pxor %xmm6, %xmm6
1264; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
1265; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
1266; SSE2-NEXT:    pxor %xmm7, %xmm7
1267; SSE2-NEXT:    pcmpgtd %xmm12, %xmm7
1268; SSE2-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]
1269; SSE2-NEXT:    movdqa %xmm15, %xmm4
1270; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]
1271; SSE2-NEXT:    pmuludq %xmm12, %xmm4
1272; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
1273; SSE2-NEXT:    pmuludq %xmm0, %xmm7
1274; SSE2-NEXT:    paddq %xmm4, %xmm7
1275; SSE2-NEXT:    psllq $32, %xmm7
1276; SSE2-NEXT:    pmuludq %xmm12, %xmm0
1277; SSE2-NEXT:    paddq %xmm7, %xmm0
1278; SSE2-NEXT:    punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm8[2],xmm15[3],xmm8[3]
1279; SSE2-NEXT:    pmuludq %xmm1, %xmm15
1280; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
1281; SSE2-NEXT:    pmuludq %xmm11, %xmm6
1282; SSE2-NEXT:    paddq %xmm15, %xmm6
1283; SSE2-NEXT:    psllq $32, %xmm6
1284; SSE2-NEXT:    pmuludq %xmm11, %xmm1
1285; SSE2-NEXT:    paddq %xmm6, %xmm1
1286; SSE2-NEXT:    movdqa %xmm13, %xmm4
1287; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]
1288; SSE2-NEXT:    pmuludq %xmm2, %xmm4
1289; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
1290; SSE2-NEXT:    pmuludq %xmm10, %xmm5
1291; SSE2-NEXT:    paddq %xmm4, %xmm5
1292; SSE2-NEXT:    psllq $32, %xmm5
1293; SSE2-NEXT:    pmuludq %xmm10, %xmm2
1294; SSE2-NEXT:    paddq %xmm5, %xmm2
1295; SSE2-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm8[2],xmm13[3],xmm8[3]
1296; SSE2-NEXT:    pmuludq %xmm3, %xmm13
1297; SSE2-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1]
1298; SSE2-NEXT:    pmuludq %xmm9, %xmm14
1299; SSE2-NEXT:    paddq %xmm13, %xmm14
1300; SSE2-NEXT:    psllq $32, %xmm14
1301; SSE2-NEXT:    pmuludq %xmm9, %xmm3
1302; SSE2-NEXT:    paddq %xmm14, %xmm3
1303; SSE2-NEXT:    retq
1304;
1305; SSE41-LABEL: mul_v8i64_sext:
1306; SSE41:       # %bb.0:
1307; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
1308; SSE41-NEXT:    pmovsxwq %xmm3, %xmm4
1309; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
1310; SSE41-NEXT:    pmovsxwq %xmm3, %xmm5
1311; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
1312; SSE41-NEXT:    pmovsxwq %xmm3, %xmm6
1313; SSE41-NEXT:    pmovsxwq %xmm0, %xmm7
1314; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
1315; SSE41-NEXT:    pmuldq %xmm4, %xmm3
1316; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
1317; SSE41-NEXT:    pmuldq %xmm5, %xmm2
1318; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,2,3,3]
1319; SSE41-NEXT:    pmuldq %xmm6, %xmm4
1320; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1321; SSE41-NEXT:    pmuldq %xmm7, %xmm0
1322; SSE41-NEXT:    movdqa %xmm4, %xmm1
1323; SSE41-NEXT:    retq
1324;
1325; AVX2-LABEL: mul_v8i64_sext:
1326; AVX2:       # %bb.0:
1327; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1328; AVX2-NEXT:    vpmovsxwq %xmm2, %ymm2
1329; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
1330; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1331; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
1332; AVX2-NEXT:    vpmuldq %ymm3, %ymm2, %ymm2
1333; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1334; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0
1335; AVX2-NEXT:    vmovdqa %ymm2, %ymm1
1336; AVX2-NEXT:    retq
1337;
1338; AVX512-LABEL: mul_v8i64_sext:
1339; AVX512:       # %bb.0:
1340; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
1341; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
1342; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm0
1343; AVX512-NEXT:    retq
1344  %1 = sext <8 x i16> %val1 to <8 x i64>
1345  %2 = sext <8 x i32> %val2 to <8 x i64>
1346  %3 = mul <8 x i64> %1, %2
1347  ret <8 x i64> %3
1348}
1349
1350define <2 x i64> @pmuldq_square(<2 x i64> %x) {
1351; SSE2-LABEL: pmuldq_square:
1352; SSE2:       # %bb.0:
1353; SSE2-NEXT:    psllq $32, %xmm0
1354; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
1355; SSE2-NEXT:    psrad $31, %xmm0
1356; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
1357; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1358; SSE2-NEXT:    psrlq $32, %xmm0
1359; SSE2-NEXT:    pmuludq %xmm1, %xmm0
1360; SSE2-NEXT:    pmuludq %xmm1, %xmm1
1361; SSE2-NEXT:    paddq %xmm0, %xmm0
1362; SSE2-NEXT:    psllq $32, %xmm0
1363; SSE2-NEXT:    paddq %xmm1, %xmm0
1364; SSE2-NEXT:    retq
1365;
1366; SSE41-LABEL: pmuldq_square:
1367; SSE41:       # %bb.0:
1368; SSE41-NEXT:    pmuldq %xmm0, %xmm0
1369; SSE41-NEXT:    retq
1370;
1371; AVX-LABEL: pmuldq_square:
1372; AVX:       # %bb.0:
1373; AVX-NEXT:    vpmuldq %xmm0, %xmm0, %xmm0
1374; AVX-NEXT:    retq
1375  %1 = shl <2 x i64> %x, <i64 32, i64 32>
1376  %2 = ashr exact <2 x i64> %1, <i64 32, i64 32>
1377  %3 = mul nsw <2 x i64> %2, %2
1378  ret <2 x i64> %3
1379}
1380
1381define <2 x i64> @pmuludq_square(<2 x i64> %x) {
1382; SSE-LABEL: pmuludq_square:
1383; SSE:       # %bb.0:
1384; SSE-NEXT:    pmuludq %xmm0, %xmm0
1385; SSE-NEXT:    retq
1386;
1387; AVX-LABEL: pmuludq_square:
1388; AVX:       # %bb.0:
1389; AVX-NEXT:    vpmuludq %xmm0, %xmm0, %xmm0
1390; AVX-NEXT:    retq
1391  %1 = and <2 x i64> %x, <i64 4294967295, i64 4294967295>
1392  %2 = mul nuw <2 x i64> %1, %1
1393  ret <2 x i64> %2
1394}
1395