1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
7
8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind  {
9; SSE2-LABEL: mul_v16i8c:
10; SSE2:       # BB#0: # %entry
11; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
12; SSE2-NEXT:    psraw $8, %xmm1
13; SSE2-NEXT:    movdqa %xmm0, %xmm2
14; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
15; SSE2-NEXT:    psraw $8, %xmm2
16; SSE2-NEXT:    pmullw %xmm1, %xmm2
17; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
18; SSE2-NEXT:    pand %xmm3, %xmm2
19; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
20; SSE2-NEXT:    psraw $8, %xmm0
21; SSE2-NEXT:    pmullw %xmm1, %xmm0
22; SSE2-NEXT:    pand %xmm3, %xmm0
23; SSE2-NEXT:    packuswb %xmm2, %xmm0
24; SSE2-NEXT:    retq
25;
26; SSE41-LABEL: mul_v16i8c:
27; SSE41:       # BB#0: # %entry
28; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
29; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
30; SSE41-NEXT:    pmullw %xmm2, %xmm1
31; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
32; SSE41-NEXT:    pand %xmm3, %xmm1
33; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
34; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
35; SSE41-NEXT:    pmullw %xmm2, %xmm0
36; SSE41-NEXT:    pand %xmm3, %xmm0
37; SSE41-NEXT:    packuswb %xmm0, %xmm1
38; SSE41-NEXT:    movdqa %xmm1, %xmm0
39; SSE41-NEXT:    retq
40;
41; AVX2-LABEL: mul_v16i8c:
42; AVX2:       # BB#0: # %entry
43; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
44; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
45; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
46; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
47; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
48; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
49; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
50; AVX2-NEXT:    vzeroupper
51; AVX2-NEXT:    retq
52;
53; AVX512F-LABEL: mul_v16i8c:
54; AVX512F:       # BB#0: # %entry
55; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
56; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
57; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
58; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
59; AVX512F-NEXT:    retq
60;
61; AVX512BW-LABEL: mul_v16i8c:
62; AVX512BW:       # BB#0: # %entry
63; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
64; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
65; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
66; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
67; AVX512BW-NEXT:    retq
68entry:
69  %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
70  ret <16 x i8> %A
71}
72
73define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind  {
74; SSE-LABEL: mul_v8i16c:
75; SSE:       # BB#0: # %entry
76; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
77; SSE-NEXT:    retq
78;
79; AVX-LABEL: mul_v8i16c:
80; AVX:       # BB#0: # %entry
81; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
82; AVX-NEXT:    retq
83entry:
84  %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
85  ret <8 x i16> %A
86}
87
88define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind  {
89; SSE2-LABEL: mul_v4i32c:
90; SSE2:       # BB#0: # %entry
91; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117]
92; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
93; SSE2-NEXT:    pmuludq %xmm1, %xmm0
94; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
95; SSE2-NEXT:    pmuludq %xmm1, %xmm2
96; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
97; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
98; SSE2-NEXT:    retq
99;
100; SSE41-LABEL: mul_v4i32c:
101; SSE41:       # BB#0: # %entry
102; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
103; SSE41-NEXT:    retq
104;
105; AVX-LABEL: mul_v4i32c:
106; AVX:       # BB#0: # %entry
107; AVX-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
108; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
109; AVX-NEXT:    retq
110entry:
111  %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 >
112  ret <4 x i32> %A
113}
114
115define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind  {
116; SSE-LABEL: mul_v2i64c:
117; SSE:       # BB#0: # %entry
118; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [117,117]
119; SSE-NEXT:    movdqa %xmm0, %xmm2
120; SSE-NEXT:    pmuludq %xmm1, %xmm2
121; SSE-NEXT:    psrlq $32, %xmm0
122; SSE-NEXT:    pmuludq %xmm1, %xmm0
123; SSE-NEXT:    psllq $32, %xmm0
124; SSE-NEXT:    paddq %xmm2, %xmm0
125; SSE-NEXT:    retq
126;
127; AVX-LABEL: mul_v2i64c:
128; AVX:       # BB#0: # %entry
129; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [117,117]
130; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
131; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
132; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
133; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
134; AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
135; AVX-NEXT:    retq
136entry:
137  %A = mul <2 x i64> %i, < i64 117, i64 117 >
138  ret <2 x i64> %A
139}
140
141define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind  {
142; SSE2-LABEL: mul_v16i8:
143; SSE2:       # BB#0: # %entry
144; SSE2-NEXT:    movdqa %xmm1, %xmm2
145; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
146; SSE2-NEXT:    psraw $8, %xmm2
147; SSE2-NEXT:    movdqa %xmm0, %xmm3
148; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
149; SSE2-NEXT:    psraw $8, %xmm3
150; SSE2-NEXT:    pmullw %xmm2, %xmm3
151; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
152; SSE2-NEXT:    pand %xmm2, %xmm3
153; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
154; SSE2-NEXT:    psraw $8, %xmm1
155; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
156; SSE2-NEXT:    psraw $8, %xmm0
157; SSE2-NEXT:    pmullw %xmm1, %xmm0
158; SSE2-NEXT:    pand %xmm2, %xmm0
159; SSE2-NEXT:    packuswb %xmm3, %xmm0
160; SSE2-NEXT:    retq
161;
162; SSE41-LABEL: mul_v16i8:
163; SSE41:       # BB#0: # %entry
164; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
165; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
166; SSE41-NEXT:    pmullw %xmm3, %xmm2
167; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
168; SSE41-NEXT:    pand %xmm3, %xmm2
169; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
170; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
171; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
172; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
173; SSE41-NEXT:    pmullw %xmm1, %xmm0
174; SSE41-NEXT:    pand %xmm3, %xmm0
175; SSE41-NEXT:    packuswb %xmm0, %xmm2
176; SSE41-NEXT:    movdqa %xmm2, %xmm0
177; SSE41-NEXT:    retq
178;
179; AVX2-LABEL: mul_v16i8:
180; AVX2:       # BB#0: # %entry
181; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
182; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
183; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
184; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
185; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
186; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
187; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
188; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
189; AVX2-NEXT:    vzeroupper
190; AVX2-NEXT:    retq
191;
192; AVX512F-LABEL: mul_v16i8:
193; AVX512F:       # BB#0: # %entry
194; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
195; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
196; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
197; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
198; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
199; AVX512F-NEXT:    retq
200;
201; AVX512BW-LABEL: mul_v16i8:
202; AVX512BW:       # BB#0: # %entry
203; AVX512BW-NEXT:    vpmovsxbw %xmm1, %ymm1
204; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
205; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
206; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
207; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
208; AVX512BW-NEXT:    retq
209entry:
210  %A = mul <16 x i8> %i, %j
211  ret <16 x i8> %A
212}
213
214define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind  {
215; SSE-LABEL: mul_v8i16:
216; SSE:       # BB#0: # %entry
217; SSE-NEXT:    pmullw %xmm1, %xmm0
218; SSE-NEXT:    retq
219;
220; AVX-LABEL: mul_v8i16:
221; AVX:       # BB#0: # %entry
222; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
223; AVX-NEXT:    retq
224entry:
225  %A = mul <8 x i16> %i, %j
226  ret <8 x i16> %A
227}
228
229define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind  {
230; SSE2-LABEL: mul_v4i32:
231; SSE2:       # BB#0: # %entry
232; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
233; SSE2-NEXT:    pmuludq %xmm1, %xmm0
234; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
235; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
236; SSE2-NEXT:    pmuludq %xmm2, %xmm1
237; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
238; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
239; SSE2-NEXT:    retq
240;
241; SSE41-LABEL: mul_v4i32:
242; SSE41:       # BB#0: # %entry
243; SSE41-NEXT:    pmulld %xmm1, %xmm0
244; SSE41-NEXT:    retq
245;
246; AVX-LABEL: mul_v4i32:
247; AVX:       # BB#0: # %entry
248; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
249; AVX-NEXT:    retq
250entry:
251  %A = mul <4 x i32> %i, %j
252  ret <4 x i32> %A
253}
254
255define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind  {
256; SSE-LABEL: mul_v2i64:
257; SSE:       # BB#0: # %entry
258; SSE-NEXT:    movdqa %xmm0, %xmm2
259; SSE-NEXT:    pmuludq %xmm1, %xmm2
260; SSE-NEXT:    movdqa %xmm1, %xmm3
261; SSE-NEXT:    psrlq $32, %xmm3
262; SSE-NEXT:    pmuludq %xmm0, %xmm3
263; SSE-NEXT:    psllq $32, %xmm3
264; SSE-NEXT:    psrlq $32, %xmm0
265; SSE-NEXT:    pmuludq %xmm1, %xmm0
266; SSE-NEXT:    psllq $32, %xmm0
267; SSE-NEXT:    paddq %xmm3, %xmm0
268; SSE-NEXT:    paddq %xmm2, %xmm0
269; SSE-NEXT:    retq
270;
271; AVX-LABEL: mul_v2i64:
272; AVX:       # BB#0: # %entry
273; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
274; AVX-NEXT:    vpsrlq $32, %xmm1, %xmm3
275; AVX-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
276; AVX-NEXT:    vpsllq $32, %xmm3, %xmm3
277; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
278; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
279; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
280; AVX-NEXT:    vpaddq %xmm0, %xmm3, %xmm0
281; AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
282; AVX-NEXT:    retq
283entry:
284  %A = mul <2 x i64> %i, %j
285  ret <2 x i64> %A
286}
287
288declare void @foo()
289
290define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind  {
291; SSE2-LABEL: mul_v4i32spill:
292; SSE2:       # BB#0: # %entry
293; SSE2-NEXT:    subq $40, %rsp
294; SSE2-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
295; SSE2-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
296; SSE2-NEXT:    callq foo
297; SSE2-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
298; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
299; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
300; SSE2-NEXT:    pmuludq %xmm2, %xmm0
301; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
302; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
303; SSE2-NEXT:    pmuludq %xmm1, %xmm2
304; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
305; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
306; SSE2-NEXT:    addq $40, %rsp
307; SSE2-NEXT:    retq
308;
309; SSE41-LABEL: mul_v4i32spill:
310; SSE41:       # BB#0: # %entry
311; SSE41-NEXT:    subq $40, %rsp
312; SSE41-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
313; SSE41-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
314; SSE41-NEXT:    callq foo
315; SSE41-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
316; SSE41-NEXT:    pmulld {{[0-9]+}}(%rsp), %xmm0 # 16-byte Folded Reload
317; SSE41-NEXT:    addq $40, %rsp
318; SSE41-NEXT:    retq
319;
320; AVX-LABEL: mul_v4i32spill:
321; AVX:       # BB#0: # %entry
322; AVX-NEXT:    subq $40, %rsp
323; AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
324; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
325; AVX-NEXT:    callq foo
326; AVX-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
327; AVX-NEXT:    vpmulld {{[0-9]+}}(%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
328; AVX-NEXT:    addq $40, %rsp
329; AVX-NEXT:    retq
330entry:
331  ; Use a call to force spills.
332  call void @foo()
333  %A = mul <4 x i32> %i, %j
334  ret <4 x i32> %A
335}
336
337define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind  {
338; SSE-LABEL: mul_v2i64spill:
339; SSE:       # BB#0: # %entry
340; SSE-NEXT:    subq $40, %rsp
341; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
342; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
343; SSE-NEXT:    callq foo
344; SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
345; SSE-NEXT:    movdqa %xmm0, %xmm2
346; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload
347; SSE-NEXT:    pmuludq %xmm3, %xmm2
348; SSE-NEXT:    movdqa %xmm3, %xmm1
349; SSE-NEXT:    psrlq $32, %xmm1
350; SSE-NEXT:    pmuludq %xmm0, %xmm1
351; SSE-NEXT:    psllq $32, %xmm1
352; SSE-NEXT:    psrlq $32, %xmm0
353; SSE-NEXT:    pmuludq %xmm3, %xmm0
354; SSE-NEXT:    psllq $32, %xmm0
355; SSE-NEXT:    paddq %xmm1, %xmm0
356; SSE-NEXT:    paddq %xmm2, %xmm0
357; SSE-NEXT:    addq $40, %rsp
358; SSE-NEXT:    retq
359;
360; AVX-LABEL: mul_v2i64spill:
361; AVX:       # BB#0: # %entry
362; AVX-NEXT:    subq $40, %rsp
363; AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
364; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
365; AVX-NEXT:    callq foo
366; AVX-NEXT:    vmovdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
367; AVX-NEXT:    vmovdqa (%rsp), %xmm4 # 16-byte Reload
368; AVX-NEXT:    vpmuludq %xmm2, %xmm4, %xmm0
369; AVX-NEXT:    vpsrlq $32, %xmm2, %xmm1
370; AVX-NEXT:    vmovdqa %xmm2, %xmm3
371; AVX-NEXT:    vpmuludq %xmm1, %xmm4, %xmm1
372; AVX-NEXT:    vpsllq $32, %xmm1, %xmm1
373; AVX-NEXT:    vpsrlq $32, %xmm4, %xmm2
374; AVX-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
375; AVX-NEXT:    vpsllq $32, %xmm2, %xmm2
376; AVX-NEXT:    vpaddq %xmm2, %xmm1, %xmm1
377; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
378; AVX-NEXT:    addq $40, %rsp
379; AVX-NEXT:    retq
380entry:
381  ; Use a call to force spills.
382  call void @foo()
383  %A = mul <2 x i64> %i, %j
384  ret <2 x i64> %A
385}
386
387define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind  {
388; SSE2-LABEL: mul_v32i8c:
389; SSE2:       # BB#0: # %entry
390; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
391; SSE2-NEXT:    psraw $8, %xmm2
392; SSE2-NEXT:    movdqa %xmm0, %xmm3
393; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
394; SSE2-NEXT:    psraw $8, %xmm3
395; SSE2-NEXT:    pmullw %xmm2, %xmm3
396; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
397; SSE2-NEXT:    pand %xmm4, %xmm3
398; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
399; SSE2-NEXT:    psraw $8, %xmm0
400; SSE2-NEXT:    pmullw %xmm2, %xmm0
401; SSE2-NEXT:    pand %xmm4, %xmm0
402; SSE2-NEXT:    packuswb %xmm3, %xmm0
403; SSE2-NEXT:    movdqa %xmm1, %xmm3
404; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
405; SSE2-NEXT:    psraw $8, %xmm3
406; SSE2-NEXT:    pmullw %xmm2, %xmm3
407; SSE2-NEXT:    pand %xmm4, %xmm3
408; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
409; SSE2-NEXT:    psraw $8, %xmm1
410; SSE2-NEXT:    pmullw %xmm2, %xmm1
411; SSE2-NEXT:    pand %xmm4, %xmm1
412; SSE2-NEXT:    packuswb %xmm3, %xmm1
413; SSE2-NEXT:    retq
414;
415; SSE41-LABEL: mul_v32i8c:
416; SSE41:       # BB#0: # %entry
417; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
418; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117]
419; SSE41-NEXT:    pmullw %xmm4, %xmm2
420; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
421; SSE41-NEXT:    pand %xmm5, %xmm2
422; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
423; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
424; SSE41-NEXT:    pmullw %xmm4, %xmm0
425; SSE41-NEXT:    pand %xmm5, %xmm0
426; SSE41-NEXT:    packuswb %xmm0, %xmm2
427; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
428; SSE41-NEXT:    pmullw %xmm4, %xmm3
429; SSE41-NEXT:    pand %xmm5, %xmm3
430; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
431; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
432; SSE41-NEXT:    pmullw %xmm4, %xmm0
433; SSE41-NEXT:    pand %xmm5, %xmm0
434; SSE41-NEXT:    packuswb %xmm0, %xmm3
435; SSE41-NEXT:    movdqa %xmm2, %xmm0
436; SSE41-NEXT:    movdqa %xmm3, %xmm1
437; SSE41-NEXT:    retq
438;
439; AVX2-LABEL: mul_v32i8c:
440; AVX2:       # BB#0: # %entry
441; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
442; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
443; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
444; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
445; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
446; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
447; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
448; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
449; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
450; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
451; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
452; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
453; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
454; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
455; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
456; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
457; AVX2-NEXT:    retq
458;
459; AVX512F-LABEL: mul_v32i8c:
460; AVX512F:       # BB#0: # %entry
461; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
462; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
463; AVX512F-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
464; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
465; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
466; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
467; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
468; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
469; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
470; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
471; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
472; AVX512F-NEXT:    retq
473;
474; AVX512BW-LABEL: mul_v32i8c:
475; AVX512BW:       # BB#0: # %entry
476; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
477; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm0, %zmm0
478; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
479; AVX512BW-NEXT:    retq
480entry:
481  %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
482  ret <32 x i8> %A
483}
484
485define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind  {
486; SSE-LABEL: mul_v16i16c:
487; SSE:       # BB#0: # %entry
488; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
489; SSE-NEXT:    pmullw %xmm2, %xmm0
490; SSE-NEXT:    pmullw %xmm2, %xmm1
491; SSE-NEXT:    retq
492;
493; AVX-LABEL: mul_v16i16c:
494; AVX:       # BB#0: # %entry
495; AVX-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
496; AVX-NEXT:    retq
497entry:
498  %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
499  ret <16 x i16> %A
500}
501
502define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind  {
503; SSE2-LABEL: mul_v8i32c:
504; SSE2:       # BB#0: # %entry
505; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
506; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
507; SSE2-NEXT:    pmuludq %xmm2, %xmm0
508; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
509; SSE2-NEXT:    pmuludq %xmm2, %xmm3
510; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
511; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
512; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
513; SSE2-NEXT:    pmuludq %xmm2, %xmm1
514; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
515; SSE2-NEXT:    pmuludq %xmm2, %xmm3
516; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
517; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
518; SSE2-NEXT:    retq
519;
520; SSE41-LABEL: mul_v8i32c:
521; SSE41:       # BB#0: # %entry
522; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
523; SSE41-NEXT:    pmulld %xmm2, %xmm0
524; SSE41-NEXT:    pmulld %xmm2, %xmm1
525; SSE41-NEXT:    retq
526;
527; AVX-LABEL: mul_v8i32c:
528; AVX:       # BB#0: # %entry
529; AVX-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
530; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
531; AVX-NEXT:    retq
532entry:
533  %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 >
534  ret <8 x i32> %A
535}
536
537define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind  {
538; SSE-LABEL: mul_v4i64c:
539; SSE:       # BB#0: # %entry
540; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117]
541; SSE-NEXT:    movdqa %xmm0, %xmm3
542; SSE-NEXT:    pmuludq %xmm2, %xmm3
543; SSE-NEXT:    psrlq $32, %xmm0
544; SSE-NEXT:    pmuludq %xmm2, %xmm0
545; SSE-NEXT:    psllq $32, %xmm0
546; SSE-NEXT:    paddq %xmm3, %xmm0
547; SSE-NEXT:    movdqa %xmm1, %xmm3
548; SSE-NEXT:    pmuludq %xmm2, %xmm3
549; SSE-NEXT:    psrlq $32, %xmm1
550; SSE-NEXT:    pmuludq %xmm2, %xmm1
551; SSE-NEXT:    psllq $32, %xmm1
552; SSE-NEXT:    paddq %xmm3, %xmm1
553; SSE-NEXT:    retq
554;
555; AVX-LABEL: mul_v4i64c:
556; AVX:       # BB#0: # %entry
557; AVX-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
558; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
559; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm0
560; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
561; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
562; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
563; AVX-NEXT:    retq
564entry:
565  %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 >
566  ret <4 x i64> %A
567}
568
569define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind  {
570; SSE2-LABEL: mul_v32i8:
571; SSE2:       # BB#0: # %entry
572; SSE2-NEXT:    movdqa %xmm2, %xmm4
573; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
574; SSE2-NEXT:    psraw $8, %xmm4
575; SSE2-NEXT:    movdqa %xmm0, %xmm5
576; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
577; SSE2-NEXT:    psraw $8, %xmm5
578; SSE2-NEXT:    pmullw %xmm4, %xmm5
579; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
580; SSE2-NEXT:    pand %xmm4, %xmm5
581; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
582; SSE2-NEXT:    psraw $8, %xmm2
583; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
584; SSE2-NEXT:    psraw $8, %xmm0
585; SSE2-NEXT:    pmullw %xmm2, %xmm0
586; SSE2-NEXT:    pand %xmm4, %xmm0
587; SSE2-NEXT:    packuswb %xmm5, %xmm0
588; SSE2-NEXT:    movdqa %xmm3, %xmm2
589; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
590; SSE2-NEXT:    psraw $8, %xmm2
591; SSE2-NEXT:    movdqa %xmm1, %xmm5
592; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
593; SSE2-NEXT:    psraw $8, %xmm5
594; SSE2-NEXT:    pmullw %xmm2, %xmm5
595; SSE2-NEXT:    pand %xmm4, %xmm5
596; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
597; SSE2-NEXT:    psraw $8, %xmm3
598; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
599; SSE2-NEXT:    psraw $8, %xmm1
600; SSE2-NEXT:    pmullw %xmm3, %xmm1
601; SSE2-NEXT:    pand %xmm4, %xmm1
602; SSE2-NEXT:    packuswb %xmm5, %xmm1
603; SSE2-NEXT:    retq
604;
605; SSE41-LABEL: mul_v32i8:
606; SSE41:       # BB#0: # %entry
607; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
608; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
609; SSE41-NEXT:    pmullw %xmm5, %xmm4
610; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
611; SSE41-NEXT:    pand %xmm5, %xmm4
612; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
613; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
614; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
615; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
616; SSE41-NEXT:    pmullw %xmm2, %xmm0
617; SSE41-NEXT:    pand %xmm5, %xmm0
618; SSE41-NEXT:    packuswb %xmm0, %xmm4
619; SSE41-NEXT:    pmovsxbw %xmm3, %xmm0
620; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
621; SSE41-NEXT:    pmullw %xmm0, %xmm2
622; SSE41-NEXT:    pand %xmm5, %xmm2
623; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
624; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
625; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
626; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
627; SSE41-NEXT:    pmullw %xmm0, %xmm1
628; SSE41-NEXT:    pand %xmm5, %xmm1
629; SSE41-NEXT:    packuswb %xmm1, %xmm2
630; SSE41-NEXT:    movdqa %xmm4, %xmm0
631; SSE41-NEXT:    movdqa %xmm2, %xmm1
632; SSE41-NEXT:    retq
633;
634; AVX2-LABEL: mul_v32i8:
635; AVX2:       # BB#0: # %entry
636; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
637; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
638; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
639; AVX2-NEXT:    vpmovsxbw %xmm3, %ymm3
640; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
641; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
642; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
643; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
644; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
645; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
646; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
647; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
648; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
649; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
650; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
651; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
652; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
653; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
654; AVX2-NEXT:    retq
655;
656; AVX512F-LABEL: mul_v32i8:
657; AVX512F:       # BB#0: # %entry
658; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm2
659; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm3
660; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
661; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
662; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
663; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
664; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
665; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
666; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
667; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
668; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
669; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
670; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
671; AVX512F-NEXT:    retq
672;
673; AVX512BW-LABEL: mul_v32i8:
674; AVX512BW:       # BB#0: # %entry
675; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
676; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
677; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
678; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
679; AVX512BW-NEXT:    retq
680entry:
681  %A = mul <32 x i8> %i, %j
682  ret <32 x i8> %A
683}
684
685define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind  {
686; SSE-LABEL: mul_v16i16:
687; SSE:       # BB#0: # %entry
688; SSE-NEXT:    pmullw %xmm2, %xmm0
689; SSE-NEXT:    pmullw %xmm3, %xmm1
690; SSE-NEXT:    retq
691;
692; AVX-LABEL: mul_v16i16:
693; AVX:       # BB#0: # %entry
694; AVX-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
695; AVX-NEXT:    retq
696entry:
697  %A = mul <16 x i16> %i, %j
698  ret <16 x i16> %A
699}
700
701define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind  {
702; SSE2-LABEL: mul_v8i32:
703; SSE2:       # BB#0: # %entry
704; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
705; SSE2-NEXT:    pmuludq %xmm2, %xmm0
706; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
707; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
708; SSE2-NEXT:    pmuludq %xmm4, %xmm2
709; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
710; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
711; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
712; SSE2-NEXT:    pmuludq %xmm3, %xmm1
713; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
714; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
715; SSE2-NEXT:    pmuludq %xmm2, %xmm3
716; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
717; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
718; SSE2-NEXT:    retq
719;
720; SSE41-LABEL: mul_v8i32:
721; SSE41:       # BB#0: # %entry
722; SSE41-NEXT:    pmulld %xmm2, %xmm0
723; SSE41-NEXT:    pmulld %xmm3, %xmm1
724; SSE41-NEXT:    retq
725;
726; AVX-LABEL: mul_v8i32:
727; AVX:       # BB#0: # %entry
728; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
729; AVX-NEXT:    retq
730entry:
731  %A = mul <8 x i32> %i, %j
732  ret <8 x i32> %A
733}
734
735define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind  {
736; SSE-LABEL: mul_v4i64:
737; SSE:       # BB#0: # %entry
738; SSE-NEXT:    movdqa %xmm0, %xmm4
739; SSE-NEXT:    pmuludq %xmm2, %xmm4
740; SSE-NEXT:    movdqa %xmm2, %xmm5
741; SSE-NEXT:    psrlq $32, %xmm5
742; SSE-NEXT:    pmuludq %xmm0, %xmm5
743; SSE-NEXT:    psllq $32, %xmm5
744; SSE-NEXT:    psrlq $32, %xmm0
745; SSE-NEXT:    pmuludq %xmm2, %xmm0
746; SSE-NEXT:    psllq $32, %xmm0
747; SSE-NEXT:    paddq %xmm5, %xmm0
748; SSE-NEXT:    paddq %xmm4, %xmm0
749; SSE-NEXT:    movdqa %xmm1, %xmm2
750; SSE-NEXT:    pmuludq %xmm3, %xmm2
751; SSE-NEXT:    movdqa %xmm3, %xmm4
752; SSE-NEXT:    psrlq $32, %xmm4
753; SSE-NEXT:    pmuludq %xmm1, %xmm4
754; SSE-NEXT:    psllq $32, %xmm4
755; SSE-NEXT:    psrlq $32, %xmm1
756; SSE-NEXT:    pmuludq %xmm3, %xmm1
757; SSE-NEXT:    psllq $32, %xmm1
758; SSE-NEXT:    paddq %xmm4, %xmm1
759; SSE-NEXT:    paddq %xmm2, %xmm1
760; SSE-NEXT:    retq
761;
762; AVX-LABEL: mul_v4i64:
763; AVX:       # BB#0: # %entry
764; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
765; AVX-NEXT:    vpsrlq $32, %ymm1, %ymm3
766; AVX-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3
767; AVX-NEXT:    vpsllq $32, %ymm3, %ymm3
768; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm0
769; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
770; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
771; AVX-NEXT:    vpaddq %ymm0, %ymm3, %ymm0
772; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
773; AVX-NEXT:    retq
774entry:
775  %A = mul <4 x i64> %i, %j
776  ret <4 x i64> %A
777}
778
779define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
780; SSE2-LABEL: mul_v64i8c:
781; SSE2:       # BB#0: # %entry
782; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
783; SSE2-NEXT:    psraw $8, %xmm4
784; SSE2-NEXT:    movdqa %xmm0, %xmm6
785; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
786; SSE2-NEXT:    psraw $8, %xmm6
787; SSE2-NEXT:    pmullw %xmm4, %xmm6
788; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
789; SSE2-NEXT:    pand %xmm5, %xmm6
790; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
791; SSE2-NEXT:    psraw $8, %xmm0
792; SSE2-NEXT:    pmullw %xmm4, %xmm0
793; SSE2-NEXT:    pand %xmm5, %xmm0
794; SSE2-NEXT:    packuswb %xmm6, %xmm0
795; SSE2-NEXT:    movdqa %xmm1, %xmm6
796; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
797; SSE2-NEXT:    psraw $8, %xmm6
798; SSE2-NEXT:    pmullw %xmm4, %xmm6
799; SSE2-NEXT:    pand %xmm5, %xmm6
800; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
801; SSE2-NEXT:    psraw $8, %xmm1
802; SSE2-NEXT:    pmullw %xmm4, %xmm1
803; SSE2-NEXT:    pand %xmm5, %xmm1
804; SSE2-NEXT:    packuswb %xmm6, %xmm1
805; SSE2-NEXT:    movdqa %xmm2, %xmm6
806; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
807; SSE2-NEXT:    psraw $8, %xmm6
808; SSE2-NEXT:    pmullw %xmm4, %xmm6
809; SSE2-NEXT:    pand %xmm5, %xmm6
810; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
811; SSE2-NEXT:    psraw $8, %xmm2
812; SSE2-NEXT:    pmullw %xmm4, %xmm2
813; SSE2-NEXT:    pand %xmm5, %xmm2
814; SSE2-NEXT:    packuswb %xmm6, %xmm2
815; SSE2-NEXT:    movdqa %xmm3, %xmm6
816; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
817; SSE2-NEXT:    psraw $8, %xmm6
818; SSE2-NEXT:    pmullw %xmm4, %xmm6
819; SSE2-NEXT:    pand %xmm5, %xmm6
820; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
821; SSE2-NEXT:    psraw $8, %xmm3
822; SSE2-NEXT:    pmullw %xmm4, %xmm3
823; SSE2-NEXT:    pand %xmm5, %xmm3
824; SSE2-NEXT:    packuswb %xmm6, %xmm3
825; SSE2-NEXT:    retq
826;
827; SSE41-LABEL: mul_v64i8c:
828; SSE41:       # BB#0: # %entry
829; SSE41-NEXT:    movdqa %xmm1, %xmm4
830; SSE41-NEXT:    movdqa %xmm0, %xmm1
831; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
832; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [117,117,117,117,117,117,117,117]
833; SSE41-NEXT:    pmullw %xmm6, %xmm0
834; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
835; SSE41-NEXT:    pand %xmm7, %xmm0
836; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
837; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
838; SSE41-NEXT:    pmullw %xmm6, %xmm1
839; SSE41-NEXT:    pand %xmm7, %xmm1
840; SSE41-NEXT:    packuswb %xmm1, %xmm0
841; SSE41-NEXT:    pmovsxbw %xmm4, %xmm1
842; SSE41-NEXT:    pmullw %xmm6, %xmm1
843; SSE41-NEXT:    pand %xmm7, %xmm1
844; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
845; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
846; SSE41-NEXT:    pmullw %xmm6, %xmm4
847; SSE41-NEXT:    pand %xmm7, %xmm4
848; SSE41-NEXT:    packuswb %xmm4, %xmm1
849; SSE41-NEXT:    pmovsxbw %xmm2, %xmm4
850; SSE41-NEXT:    pmullw %xmm6, %xmm4
851; SSE41-NEXT:    pand %xmm7, %xmm4
852; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
853; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
854; SSE41-NEXT:    pmullw %xmm6, %xmm2
855; SSE41-NEXT:    pand %xmm7, %xmm2
856; SSE41-NEXT:    packuswb %xmm2, %xmm4
857; SSE41-NEXT:    pmovsxbw %xmm3, %xmm5
858; SSE41-NEXT:    pmullw %xmm6, %xmm5
859; SSE41-NEXT:    pand %xmm7, %xmm5
860; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
861; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
862; SSE41-NEXT:    pmullw %xmm6, %xmm2
863; SSE41-NEXT:    pand %xmm7, %xmm2
864; SSE41-NEXT:    packuswb %xmm2, %xmm5
865; SSE41-NEXT:    movdqa %xmm4, %xmm2
866; SSE41-NEXT:    movdqa %xmm5, %xmm3
867; SSE41-NEXT:    retq
868;
869; AVX2-LABEL: mul_v64i8c:
870; AVX2:       # BB#0: # %entry
871; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
872; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
873; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
874; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
875; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
876; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
877; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
878; AVX2-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
879; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
880; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
881; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
882; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
883; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
884; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
885; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
886; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
887; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
888; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
889; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
890; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
891; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
892; AVX2-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
893; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
894; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
895; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
896; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
897; AVX2-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
898; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
899; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
900; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
901; AVX2-NEXT:    retq
902;
903; AVX512F-LABEL: mul_v64i8c:
904; AVX512F:       # BB#0: # %entry
905; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm2
906; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
907; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
908; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
909; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
910; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
911; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
912; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
913; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
914; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
915; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
916; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm2
917; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
918; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
919; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
920; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
921; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
922; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
923; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
924; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
925; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
926; AVX512F-NEXT:    retq
927;
928; AVX512BW-LABEL: mul_v64i8c:
929; AVX512BW:       # BB#0: # %entry
930; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm1
931; AVX512BW-NEXT:    vmovdqu16 {{.*#+}} zmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
932; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
933; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
934; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
935; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
936; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
937; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
938; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
939; AVX512BW-NEXT:    retq
940entry:
941  %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
942  ret <64 x i8> %A
943}
944
945define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
946; SSE2-LABEL: mul_v64i8:
947; SSE2:       # BB#0: # %entry
948; SSE2-NEXT:    movdqa %xmm4, %xmm8
949; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
950; SSE2-NEXT:    psraw $8, %xmm8
951; SSE2-NEXT:    movdqa %xmm0, %xmm9
952; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
953; SSE2-NEXT:    psraw $8, %xmm9
954; SSE2-NEXT:    pmullw %xmm8, %xmm9
955; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
956; SSE2-NEXT:    pand %xmm8, %xmm9
957; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
958; SSE2-NEXT:    psraw $8, %xmm4
959; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
960; SSE2-NEXT:    psraw $8, %xmm0
961; SSE2-NEXT:    pmullw %xmm4, %xmm0
962; SSE2-NEXT:    pand %xmm8, %xmm0
963; SSE2-NEXT:    packuswb %xmm9, %xmm0
964; SSE2-NEXT:    movdqa %xmm5, %xmm9
965; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
966; SSE2-NEXT:    psraw $8, %xmm9
967; SSE2-NEXT:    movdqa %xmm1, %xmm4
968; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
969; SSE2-NEXT:    psraw $8, %xmm4
970; SSE2-NEXT:    pmullw %xmm9, %xmm4
971; SSE2-NEXT:    pand %xmm8, %xmm4
972; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
973; SSE2-NEXT:    psraw $8, %xmm5
974; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
975; SSE2-NEXT:    psraw $8, %xmm1
976; SSE2-NEXT:    pmullw %xmm5, %xmm1
977; SSE2-NEXT:    pand %xmm8, %xmm1
978; SSE2-NEXT:    packuswb %xmm4, %xmm1
979; SSE2-NEXT:    movdqa %xmm6, %xmm4
980; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
981; SSE2-NEXT:    psraw $8, %xmm4
982; SSE2-NEXT:    movdqa %xmm2, %xmm5
983; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
984; SSE2-NEXT:    psraw $8, %xmm5
985; SSE2-NEXT:    pmullw %xmm4, %xmm5
986; SSE2-NEXT:    pand %xmm8, %xmm5
987; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
988; SSE2-NEXT:    psraw $8, %xmm6
989; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
990; SSE2-NEXT:    psraw $8, %xmm2
991; SSE2-NEXT:    pmullw %xmm6, %xmm2
992; SSE2-NEXT:    pand %xmm8, %xmm2
993; SSE2-NEXT:    packuswb %xmm5, %xmm2
994; SSE2-NEXT:    movdqa %xmm7, %xmm4
995; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
996; SSE2-NEXT:    psraw $8, %xmm4
997; SSE2-NEXT:    movdqa %xmm3, %xmm5
998; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
999; SSE2-NEXT:    psraw $8, %xmm5
1000; SSE2-NEXT:    pmullw %xmm4, %xmm5
1001; SSE2-NEXT:    pand %xmm8, %xmm5
1002; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1003; SSE2-NEXT:    psraw $8, %xmm7
1004; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1005; SSE2-NEXT:    psraw $8, %xmm3
1006; SSE2-NEXT:    pmullw %xmm7, %xmm3
1007; SSE2-NEXT:    pand %xmm8, %xmm3
1008; SSE2-NEXT:    packuswb %xmm5, %xmm3
1009; SSE2-NEXT:    retq
1010;
1011; SSE41-LABEL: mul_v64i8:
1012; SSE41:       # BB#0: # %entry
1013; SSE41-NEXT:    movdqa %xmm1, %xmm8
1014; SSE41-NEXT:    movdqa %xmm0, %xmm1
1015; SSE41-NEXT:    pmovsxbw %xmm4, %xmm9
1016; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
1017; SSE41-NEXT:    pmullw %xmm9, %xmm0
1018; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
1019; SSE41-NEXT:    pand %xmm9, %xmm0
1020; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
1021; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
1022; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1023; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
1024; SSE41-NEXT:    pmullw %xmm4, %xmm1
1025; SSE41-NEXT:    pand %xmm9, %xmm1
1026; SSE41-NEXT:    packuswb %xmm1, %xmm0
1027; SSE41-NEXT:    pmovsxbw %xmm5, %xmm4
1028; SSE41-NEXT:    pmovsxbw %xmm8, %xmm1
1029; SSE41-NEXT:    pmullw %xmm4, %xmm1
1030; SSE41-NEXT:    pand %xmm9, %xmm1
1031; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
1032; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
1033; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[2,3,0,1]
1034; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
1035; SSE41-NEXT:    pmullw %xmm4, %xmm5
1036; SSE41-NEXT:    pand %xmm9, %xmm5
1037; SSE41-NEXT:    packuswb %xmm5, %xmm1
1038; SSE41-NEXT:    pmovsxbw %xmm6, %xmm5
1039; SSE41-NEXT:    pmovsxbw %xmm2, %xmm4
1040; SSE41-NEXT:    pmullw %xmm5, %xmm4
1041; SSE41-NEXT:    pand %xmm9, %xmm4
1042; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[2,3,0,1]
1043; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
1044; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1045; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
1046; SSE41-NEXT:    pmullw %xmm5, %xmm2
1047; SSE41-NEXT:    pand %xmm9, %xmm2
1048; SSE41-NEXT:    packuswb %xmm2, %xmm4
1049; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1050; SSE41-NEXT:    pmovsxbw %xmm3, %xmm5
1051; SSE41-NEXT:    pmullw %xmm2, %xmm5
1052; SSE41-NEXT:    pand %xmm9, %xmm5
1053; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[2,3,0,1]
1054; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
1055; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
1056; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
1057; SSE41-NEXT:    pmullw %xmm2, %xmm3
1058; SSE41-NEXT:    pand %xmm9, %xmm3
1059; SSE41-NEXT:    packuswb %xmm3, %xmm5
1060; SSE41-NEXT:    movdqa %xmm4, %xmm2
1061; SSE41-NEXT:    movdqa %xmm5, %xmm3
1062; SSE41-NEXT:    retq
1063;
1064; AVX2-LABEL: mul_v64i8:
1065; AVX2:       # BB#0: # %entry
1066; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
1067; AVX2-NEXT:    vpmovsxbw %xmm4, %ymm4
1068; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
1069; AVX2-NEXT:    vpmovsxbw %xmm5, %ymm5
1070; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
1071; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1072; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1073; AVX2-NEXT:    vpshufb %xmm4, %xmm6, %xmm6
1074; AVX2-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
1075; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
1076; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
1077; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
1078; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
1079; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
1080; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
1081; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
1082; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1083; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm0
1084; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm2
1085; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
1086; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1087; AVX2-NEXT:    vpmovsxbw %xmm5, %ymm5
1088; AVX2-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
1089; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
1090; AVX2-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
1091; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
1092; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
1093; AVX2-NEXT:    vpmovsxbw %xmm3, %ymm3
1094; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
1095; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1096; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1097; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
1098; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
1099; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
1100; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
1101; AVX2-NEXT:    retq
1102;
1103; AVX512F-LABEL: mul_v64i8:
1104; AVX512F:       # BB#0: # %entry
1105; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm4
1106; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm5
1107; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1108; AVX512F-NEXT:    vpmovsxwd %ymm4, %zmm4
1109; AVX512F-NEXT:    vpmovdb %zmm4, %xmm4
1110; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm2
1111; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
1112; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
1113; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1114; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
1115; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
1116; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1117; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm4, %ymm0
1118; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm2
1119; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm4
1120; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
1121; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
1122; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1123; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm3
1124; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
1125; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
1126; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1127; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1128; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
1129; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
1130; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
1131; AVX512F-NEXT:    retq
1132;
1133; AVX512BW-LABEL: mul_v64i8:
1134; AVX512BW:       # BB#0: # %entry
1135; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm2
1136; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm3
1137; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
1138; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
1139; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
1140; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
1141; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
1142; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
1143; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
1144; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
1145; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
1146; AVX512BW-NEXT:    retq
1147entry:
1148  %A = mul <64 x i8> %i, %j
1149  ret <64 x i8> %A
1150}
1151
1152; PR30845
1153define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) {
1154; SSE2-LABEL: mul_v4i64_zero_upper:
1155; SSE2:       # BB#0: # %entry
1156; SSE2-NEXT:    pxor %xmm2, %xmm2
1157; SSE2-NEXT:    movdqa %xmm0, %xmm3
1158; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
1159; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1160; SSE2-NEXT:    movdqa %xmm1, %xmm4
1161; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
1162; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1163; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1164; SSE2-NEXT:    pmuludq %xmm3, %xmm4
1165; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
1166; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1167; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1168; SSE2-NEXT:    retq
1169;
1170; SSE41-LABEL: mul_v4i64_zero_upper:
1171; SSE41:       # BB#0: # %entry
1172; SSE41-NEXT:    pxor %xmm2, %xmm2
1173; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
1174; SSE41-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1175; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
1176; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1177; SSE41-NEXT:    pmuludq %xmm0, %xmm1
1178; SSE41-NEXT:    pmuludq %xmm3, %xmm4
1179; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1180; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]
1181; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
1182; SSE41-NEXT:    retq
1183;
1184; AVX2-LABEL: mul_v4i64_zero_upper:
1185; AVX2:       # BB#0: # %entry
1186; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1187; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1188; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1189; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1190; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1191; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1192; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1193; AVX2-NEXT:    vzeroupper
1194; AVX2-NEXT:    retq
1195;
1196; AVX512-LABEL: mul_v4i64_zero_upper:
1197; AVX512:       # BB#0: # %entry
1198; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1199; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1200; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1201; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
1202; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1203; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1204; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1205; AVX512-NEXT:    retq
1206entry:
1207  %val1a = zext <4 x i32> %val1 to <4 x i64>
1208  %val2a = zext <4 x i32> %val2 to <4 x i64>
1209  %res64 = mul <4 x i64> %val1a, %val2a
1210  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1211  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1212  ret <4 x i32> %res
1213}
1214
1215define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) {
1216; SSE2-LABEL: mul_v4i64_zero_upper_left:
1217; SSE2:       # BB#0: # %entry
1218; SSE2-NEXT:    pxor %xmm3, %xmm3
1219; SSE2-NEXT:    movdqa %xmm0, %xmm4
1220; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
1221; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1222; SSE2-NEXT:    movdqa %xmm0, %xmm3
1223; SSE2-NEXT:    pmuludq %xmm1, %xmm3
1224; SSE2-NEXT:    psrlq $32, %xmm1
1225; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1226; SSE2-NEXT:    psllq $32, %xmm1
1227; SSE2-NEXT:    paddq %xmm3, %xmm1
1228; SSE2-NEXT:    movdqa %xmm4, %xmm0
1229; SSE2-NEXT:    pmuludq %xmm2, %xmm0
1230; SSE2-NEXT:    psrlq $32, %xmm2
1231; SSE2-NEXT:    pmuludq %xmm4, %xmm2
1232; SSE2-NEXT:    psllq $32, %xmm2
1233; SSE2-NEXT:    paddq %xmm0, %xmm2
1234; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
1235; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1236; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1237; SSE2-NEXT:    retq
1238;
1239; SSE41-LABEL: mul_v4i64_zero_upper_left:
1240; SSE41:       # BB#0: # %entry
1241; SSE41-NEXT:    pxor %xmm3, %xmm3
1242; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
1243; SSE41-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
1244; SSE41-NEXT:    movdqa %xmm4, %xmm3
1245; SSE41-NEXT:    pmuludq %xmm1, %xmm3
1246; SSE41-NEXT:    psrlq $32, %xmm1
1247; SSE41-NEXT:    pmuludq %xmm4, %xmm1
1248; SSE41-NEXT:    psllq $32, %xmm1
1249; SSE41-NEXT:    paddq %xmm3, %xmm1
1250; SSE41-NEXT:    movdqa %xmm0, %xmm3
1251; SSE41-NEXT:    pmuludq %xmm2, %xmm3
1252; SSE41-NEXT:    psrlq $32, %xmm2
1253; SSE41-NEXT:    pmuludq %xmm0, %xmm2
1254; SSE41-NEXT:    psllq $32, %xmm2
1255; SSE41-NEXT:    paddq %xmm3, %xmm2
1256; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,1,3]
1257; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1258; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
1259; SSE41-NEXT:    retq
1260;
1261; AVX2-LABEL: mul_v4i64_zero_upper_left:
1262; AVX2:       # BB#0: # %entry
1263; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1264; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
1265; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm1
1266; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1267; AVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
1268; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
1269; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1270; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1271; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1272; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1273; AVX2-NEXT:    vzeroupper
1274; AVX2-NEXT:    retq
1275;
1276; AVX512-LABEL: mul_v4i64_zero_upper_left:
1277; AVX512:       # BB#0: # %entry
1278; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1279; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
1280; AVX512-NEXT:    vpsrlq $32, %ymm1, %ymm1
1281; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1282; AVX512-NEXT:    vpsllq $32, %ymm0, %ymm0
1283; AVX512-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
1284; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
1285; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1286; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1287; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1288; AVX512-NEXT:    retq
1289entry:
1290  %val1a = zext <4 x i32> %val1 to <4 x i64>
1291  %res64 = mul <4 x i64> %val1a, %val2
1292  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1293  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1294  ret <4 x i32> %res
1295}
1296
1297define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) {
1298; SSE2-LABEL: mul_v4i64_zero_lower:
1299; SSE2:       # BB#0: # %entry
1300; SSE2-NEXT:    pxor %xmm3, %xmm3
1301; SSE2-NEXT:    movdqa %xmm0, %xmm4
1302; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
1303; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1304; SSE2-NEXT:    psrlq $32, %xmm1
1305; SSE2-NEXT:    pmuludq %xmm0, %xmm1
1306; SSE2-NEXT:    psllq $32, %xmm1
1307; SSE2-NEXT:    psrlq $32, %xmm2
1308; SSE2-NEXT:    pmuludq %xmm4, %xmm2
1309; SSE2-NEXT:    psllq $32, %xmm2
1310; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
1311; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1312; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1313; SSE2-NEXT:    retq
1314;
1315; SSE41-LABEL: mul_v4i64_zero_lower:
1316; SSE41:       # BB#0: # %entry
1317; SSE41-NEXT:    pxor %xmm3, %xmm3
1318; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
1319; SSE41-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
1320; SSE41-NEXT:    psrlq $32, %xmm1
1321; SSE41-NEXT:    pmuludq %xmm4, %xmm1
1322; SSE41-NEXT:    psllq $32, %xmm1
1323; SSE41-NEXT:    psrlq $32, %xmm2
1324; SSE41-NEXT:    pmuludq %xmm0, %xmm2
1325; SSE41-NEXT:    psllq $32, %xmm2
1326; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,1,3]
1327; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
1328; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
1329; SSE41-NEXT:    retq
1330;
1331; AVX2-LABEL: mul_v4i64_zero_lower:
1332; AVX2:       # BB#0: # %entry
1333; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1334; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm1
1335; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1336; AVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
1337; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1338; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1339; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1340; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1341; AVX2-NEXT:    vzeroupper
1342; AVX2-NEXT:    retq
1343;
1344; AVX512-LABEL: mul_v4i64_zero_lower:
1345; AVX512:       # BB#0: # %entry
1346; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1347; AVX512-NEXT:    vpsrlq $32, %ymm1, %ymm1
1348; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1349; AVX512-NEXT:    vpsllq $32, %ymm0, %ymm0
1350; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
1351; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
1352; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1353; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1354; AVX512-NEXT:    retq
1355entry:
1356  %val1a = zext <4 x i32> %val1 to <4 x i64>
1357  %val2a = and <4 x i64> %val2, <i64 -4294967296, i64 -4294967296, i64 -4294967296, i64 -4294967296>
1358  %res64 = mul <4 x i64> %val1a, %val2a
1359  %rescast = bitcast <4 x i64> %res64 to <8 x i32>
1360  %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1361  ret <4 x i32> %res
1362}
1363
1364define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {
1365; SSE2-LABEL: mul_v8i64_zero_upper:
1366; SSE2:       # BB#0: # %entry
1367; SSE2-NEXT:    pxor %xmm4, %xmm4
1368; SSE2-NEXT:    movdqa %xmm0, %xmm8
1369; SSE2-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm4[2],xmm8[3],xmm4[3]
1370; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1371; SSE2-NEXT:    movdqa %xmm1, %xmm6
1372; SSE2-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
1373; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1374; SSE2-NEXT:    movdqa %xmm2, %xmm7
1375; SSE2-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm4[2],xmm7[3],xmm4[3]
1376; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
1377; SSE2-NEXT:    movdqa %xmm3, %xmm5
1378; SSE2-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
1379; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1380; SSE2-NEXT:    pmuludq %xmm1, %xmm3
1381; SSE2-NEXT:    pmuludq %xmm6, %xmm5
1382; SSE2-NEXT:    pmuludq %xmm0, %xmm2
1383; SSE2-NEXT:    pmuludq %xmm8, %xmm7
1384; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,3,2,3]
1385; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
1386; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1387; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,3,2,3]
1388; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
1389; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1390; SSE2-NEXT:    retq
1391;
1392; SSE41-LABEL: mul_v8i64_zero_upper:
1393; SSE41:       # BB#0: # %entry
1394; SSE41-NEXT:    pxor %xmm4, %xmm4
1395; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm8 = xmm0[0],zero,xmm0[1],zero
1396; SSE41-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
1397; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
1398; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
1399; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm7 = xmm2[0],zero,xmm2[1],zero
1400; SSE41-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
1401; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
1402; SSE41-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
1403; SSE41-NEXT:    pmuludq %xmm1, %xmm3
1404; SSE41-NEXT:    pmuludq %xmm0, %xmm2
1405; SSE41-NEXT:    pmuludq %xmm6, %xmm5
1406; SSE41-NEXT:    pmuludq %xmm8, %xmm7
1407; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]
1408; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,3,2,3]
1409; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
1410; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,1,1,3]
1411; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,3,2,3]
1412; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
1413; SSE41-NEXT:    retq
1414;
1415; AVX2-LABEL: mul_v8i64_zero_upper:
1416; AVX2:       # BB#0: # %entry
1417; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
1418; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
1419; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1420; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1421; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
1422; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1423; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
1424; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm1
1425; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7]
1426; AVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[1,3,2,3,5,7,6,7]
1427; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1428; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1429; AVX2-NEXT:    retq
1430;
1431; AVX512-LABEL: mul_v8i64_zero_upper:
1432; AVX512:       # BB#0: # %entry
1433; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
1434; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
1435; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0
1436; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
1437; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7]
1438; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[1,3,2,3,5,7,6,7]
1439; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1440; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1441; AVX512-NEXT:    retq
1442entry:
1443  %val1a = zext <8 x i32> %val1 to <8 x i64>
1444  %val2a = zext <8 x i32> %val2 to <8 x i64>
1445  %res64 = mul <8 x i64> %val1a, %val2a
1446  %rescast = bitcast <8 x i64> %res64 to <16 x i32>
1447  %res = shufflevector <16 x i32> %rescast, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 9, i32 11, i32 13, i32 15 >
1448  ret <8 x i32> %res
1449}
1450
1451define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
1452; SSE2-LABEL: mul_v8i64_sext:
1453; SSE2:       # BB#0:
1454; SSE2-NEXT:    movdqa %xmm2, %xmm9
1455; SSE2-NEXT:    movdqa %xmm1, %xmm4
1456; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
1457; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
1458; SSE2-NEXT:    movdqa %xmm3, %xmm1
1459; SSE2-NEXT:    psrad $31, %xmm1
1460; SSE2-NEXT:    psrad $16, %xmm3
1461; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1462; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1463; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
1464; SSE2-NEXT:    movdqa %xmm1, %xmm2
1465; SSE2-NEXT:    psrad $31, %xmm2
1466; SSE2-NEXT:    psrad $16, %xmm1
1467; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1468; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1469; SSE2-NEXT:    movdqa %xmm2, %xmm6
1470; SSE2-NEXT:    psrad $31, %xmm6
1471; SSE2-NEXT:    psrad $16, %xmm2
1472; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
1473; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1474; SSE2-NEXT:    movdqa %xmm0, %xmm6
1475; SSE2-NEXT:    psrad $31, %xmm6
1476; SSE2-NEXT:    psrad $16, %xmm0
1477; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
1478; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[2,3,0,1]
1479; SSE2-NEXT:    movdqa %xmm8, %xmm7
1480; SSE2-NEXT:    psrad $31, %xmm7
1481; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
1482; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
1483; SSE2-NEXT:    movdqa %xmm6, %xmm7
1484; SSE2-NEXT:    psrad $31, %xmm7
1485; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
1486; SSE2-NEXT:    movdqa %xmm9, %xmm7
1487; SSE2-NEXT:    psrad $31, %xmm7
1488; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
1489; SSE2-NEXT:    movdqa %xmm4, %xmm7
1490; SSE2-NEXT:    psrad $31, %xmm7
1491; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
1492; SSE2-NEXT:    movdqa %xmm0, %xmm7
1493; SSE2-NEXT:    pmuludq %xmm4, %xmm7
1494; SSE2-NEXT:    movdqa %xmm4, %xmm5
1495; SSE2-NEXT:    psrlq $32, %xmm5
1496; SSE2-NEXT:    pmuludq %xmm0, %xmm5
1497; SSE2-NEXT:    psllq $32, %xmm5
1498; SSE2-NEXT:    psrlq $32, %xmm0
1499; SSE2-NEXT:    pmuludq %xmm4, %xmm0
1500; SSE2-NEXT:    psllq $32, %xmm0
1501; SSE2-NEXT:    paddq %xmm5, %xmm0
1502; SSE2-NEXT:    paddq %xmm7, %xmm0
1503; SSE2-NEXT:    movdqa %xmm2, %xmm4
1504; SSE2-NEXT:    pmuludq %xmm9, %xmm4
1505; SSE2-NEXT:    movdqa %xmm9, %xmm5
1506; SSE2-NEXT:    psrlq $32, %xmm5
1507; SSE2-NEXT:    pmuludq %xmm2, %xmm5
1508; SSE2-NEXT:    psllq $32, %xmm5
1509; SSE2-NEXT:    psrlq $32, %xmm2
1510; SSE2-NEXT:    pmuludq %xmm9, %xmm2
1511; SSE2-NEXT:    psllq $32, %xmm2
1512; SSE2-NEXT:    paddq %xmm5, %xmm2
1513; SSE2-NEXT:    paddq %xmm4, %xmm2
1514; SSE2-NEXT:    movdqa %xmm1, %xmm4
1515; SSE2-NEXT:    pmuludq %xmm6, %xmm4
1516; SSE2-NEXT:    movdqa %xmm6, %xmm5
1517; SSE2-NEXT:    psrlq $32, %xmm5
1518; SSE2-NEXT:    pmuludq %xmm1, %xmm5
1519; SSE2-NEXT:    psllq $32, %xmm5
1520; SSE2-NEXT:    psrlq $32, %xmm1
1521; SSE2-NEXT:    pmuludq %xmm6, %xmm1
1522; SSE2-NEXT:    psllq $32, %xmm1
1523; SSE2-NEXT:    paddq %xmm5, %xmm1
1524; SSE2-NEXT:    paddq %xmm4, %xmm1
1525; SSE2-NEXT:    movdqa %xmm3, %xmm4
1526; SSE2-NEXT:    pmuludq %xmm8, %xmm4
1527; SSE2-NEXT:    movdqa %xmm8, %xmm5
1528; SSE2-NEXT:    psrlq $32, %xmm5
1529; SSE2-NEXT:    pmuludq %xmm3, %xmm5
1530; SSE2-NEXT:    psllq $32, %xmm5
1531; SSE2-NEXT:    psrlq $32, %xmm3
1532; SSE2-NEXT:    pmuludq %xmm8, %xmm3
1533; SSE2-NEXT:    psllq $32, %xmm3
1534; SSE2-NEXT:    paddq %xmm5, %xmm3
1535; SSE2-NEXT:    paddq %xmm4, %xmm3
1536; SSE2-NEXT:    retq
1537;
1538; SSE41-LABEL: mul_v8i64_sext:
1539; SSE41:       # BB#0:
1540; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
1541; SSE41-NEXT:    pmovsxwq %xmm3, %xmm8
1542; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
1543; SSE41-NEXT:    pmovsxwq %xmm3, %xmm6
1544; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
1545; SSE41-NEXT:    pmovsxwq %xmm3, %xmm7
1546; SSE41-NEXT:    pmovsxwq %xmm0, %xmm5
1547; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1548; SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1549; SSE41-NEXT:    pmovsxdq %xmm2, %xmm2
1550; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1551; SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1552; SSE41-NEXT:    pmovsxdq %xmm1, %xmm0
1553; SSE41-NEXT:    pmuldq %xmm5, %xmm0
1554; SSE41-NEXT:    pmuldq %xmm7, %xmm4
1555; SSE41-NEXT:    pmuldq %xmm6, %xmm2
1556; SSE41-NEXT:    pmuldq %xmm8, %xmm3
1557; SSE41-NEXT:    movdqa %xmm4, %xmm1
1558; SSE41-NEXT:    retq
1559;
1560; AVX2-LABEL: mul_v8i64_sext:
1561; AVX2:       # BB#0:
1562; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1563; AVX2-NEXT:    vpmovsxwq %xmm2, %ymm2
1564; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
1565; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1566; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
1567; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
1568; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0
1569; AVX2-NEXT:    vpmuldq %ymm3, %ymm2, %ymm1
1570; AVX2-NEXT:    retq
1571;
1572; AVX512-LABEL: mul_v8i64_sext:
1573; AVX512:       # BB#0:
1574; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
1575; AVX512-NEXT:    vpmovsxdq %ymm1, %zmm1
1576; AVX512-NEXT:    vpmuldq %zmm1, %zmm0, %zmm0
1577; AVX512-NEXT:    retq
1578  %1 = sext <8 x i16> %val1 to <8 x i64>
1579  %2 = sext <8 x i32> %val2 to <8 x i64>
1580  %3 = mul <8 x i64> %1, %2
1581  ret <8 x i64> %3
1582}
1583