1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
7
8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind  {
9; SSE2-LABEL: mul_v16i8c:
10; SSE2:       # BB#0: # %entry
11; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
12; SSE2-NEXT:    psraw $8, %xmm1
13; SSE2-NEXT:    movdqa %xmm0, %xmm2
14; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
15; SSE2-NEXT:    psraw $8, %xmm2
16; SSE2-NEXT:    pmullw %xmm1, %xmm2
17; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
18; SSE2-NEXT:    pand %xmm3, %xmm2
19; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
20; SSE2-NEXT:    psraw $8, %xmm0
21; SSE2-NEXT:    pmullw %xmm1, %xmm0
22; SSE2-NEXT:    pand %xmm3, %xmm0
23; SSE2-NEXT:    packuswb %xmm2, %xmm0
24; SSE2-NEXT:    retq
25;
26; SSE41-LABEL: mul_v16i8c:
27; SSE41:       # BB#0: # %entry
28; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
29; SSE41-NEXT:    pmovsxbw {{.*}}(%rip), %xmm2
30; SSE41-NEXT:    pmullw %xmm2, %xmm1
31; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
32; SSE41-NEXT:    pand %xmm3, %xmm1
33; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
34; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
35; SSE41-NEXT:    pmullw %xmm2, %xmm0
36; SSE41-NEXT:    pand %xmm3, %xmm0
37; SSE41-NEXT:    packuswb %xmm0, %xmm1
38; SSE41-NEXT:    movdqa %xmm1, %xmm0
39; SSE41-NEXT:    retq
40;
41; AVX2-LABEL: mul_v16i8c:
42; AVX2:       # BB#0: # %entry
43; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
44; AVX2-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm1
45; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
46; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
47; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
48; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
49; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
50; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
51; AVX2-NEXT:    vzeroupper
52; AVX2-NEXT:    retq
53;
54; AVX512F-LABEL: mul_v16i8c:
55; AVX512F:       # BB#0: # %entry
56; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
57; AVX512F-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm1
58; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
59; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
60; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
61; AVX512F-NEXT:    retq
62;
63; AVX512BW-LABEL: mul_v16i8c:
64; AVX512BW:       # BB#0: # %entry
65; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
66; AVX512BW-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm1
67; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
68; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
69; AVX512BW-NEXT:    retq
70entry:
71  %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
72  ret <16 x i8> %A
73}
74
75define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind  {
76; SSE-LABEL: mul_v8i16c:
77; SSE:       # BB#0: # %entry
78; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
79; SSE-NEXT:    retq
80;
81; AVX-LABEL: mul_v8i16c:
82; AVX:       # BB#0: # %entry
83; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
84; AVX-NEXT:    retq
85entry:
86  %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
87  ret <8 x i16> %A
88}
89
90define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind  {
91; SSE2-LABEL: mul_v4i32c:
92; SSE2:       # BB#0: # %entry
93; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117]
94; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
95; SSE2-NEXT:    pmuludq %xmm1, %xmm0
96; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
97; SSE2-NEXT:    pmuludq %xmm1, %xmm2
98; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
99; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
100; SSE2-NEXT:    retq
101;
102; SSE41-LABEL: mul_v4i32c:
103; SSE41:       # BB#0: # %entry
104; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
105; SSE41-NEXT:    retq
106;
107; AVX-LABEL: mul_v4i32c:
108; AVX:       # BB#0: # %entry
109; AVX-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
110; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
111; AVX-NEXT:    retq
112entry:
113  %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 >
114  ret <4 x i32> %A
115}
116
117define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind  {
118; SSE-LABEL: mul_v2i64c:
119; SSE:       # BB#0: # %entry
120; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [117,117]
121; SSE-NEXT:    movdqa %xmm0, %xmm2
122; SSE-NEXT:    pmuludq %xmm1, %xmm2
123; SSE-NEXT:    psrlq $32, %xmm0
124; SSE-NEXT:    pmuludq %xmm1, %xmm0
125; SSE-NEXT:    psllq $32, %xmm0
126; SSE-NEXT:    paddq %xmm2, %xmm0
127; SSE-NEXT:    retq
128;
129; AVX-LABEL: mul_v2i64c:
130; AVX:       # BB#0: # %entry
131; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [117,117]
132; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
133; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
134; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
135; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
136; AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
137; AVX-NEXT:    retq
138entry:
139  %A = mul <2 x i64> %i, < i64 117, i64 117 >
140  ret <2 x i64> %A
141}
142
143define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind  {
144; SSE2-LABEL: mul_v16i8:
145; SSE2:       # BB#0: # %entry
146; SSE2-NEXT:    movdqa %xmm1, %xmm2
147; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
148; SSE2-NEXT:    psraw $8, %xmm2
149; SSE2-NEXT:    movdqa %xmm0, %xmm3
150; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
151; SSE2-NEXT:    psraw $8, %xmm3
152; SSE2-NEXT:    pmullw %xmm2, %xmm3
153; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
154; SSE2-NEXT:    pand %xmm2, %xmm3
155; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
156; SSE2-NEXT:    psraw $8, %xmm1
157; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
158; SSE2-NEXT:    psraw $8, %xmm0
159; SSE2-NEXT:    pmullw %xmm1, %xmm0
160; SSE2-NEXT:    pand %xmm2, %xmm0
161; SSE2-NEXT:    packuswb %xmm3, %xmm0
162; SSE2-NEXT:    retq
163;
164; SSE41-LABEL: mul_v16i8:
165; SSE41:       # BB#0: # %entry
166; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
167; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
168; SSE41-NEXT:    pmullw %xmm3, %xmm2
169; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
170; SSE41-NEXT:    pand %xmm3, %xmm2
171; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
172; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
173; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
174; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
175; SSE41-NEXT:    pmullw %xmm1, %xmm0
176; SSE41-NEXT:    pand %xmm3, %xmm0
177; SSE41-NEXT:    packuswb %xmm0, %xmm2
178; SSE41-NEXT:    movdqa %xmm2, %xmm0
179; SSE41-NEXT:    retq
180;
181; AVX2-LABEL: mul_v16i8:
182; AVX2:       # BB#0: # %entry
183; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
184; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
185; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
186; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
187; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
188; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
189; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
190; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
191; AVX2-NEXT:    vzeroupper
192; AVX2-NEXT:    retq
193;
194; AVX512F-LABEL: mul_v16i8:
195; AVX512F:       # BB#0: # %entry
196; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
197; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
198; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
199; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
200; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
201; AVX512F-NEXT:    retq
202;
203; AVX512BW-LABEL: mul_v16i8:
204; AVX512BW:       # BB#0: # %entry
205; AVX512BW-NEXT:    vpmovsxbw %xmm1, %ymm1
206; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
207; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
208; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
209; AVX512BW-NEXT:    retq
210entry:
211  %A = mul <16 x i8> %i, %j
212  ret <16 x i8> %A
213}
214
215define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind  {
216; SSE-LABEL: mul_v8i16:
217; SSE:       # BB#0: # %entry
218; SSE-NEXT:    pmullw %xmm1, %xmm0
219; SSE-NEXT:    retq
220;
221; AVX-LABEL: mul_v8i16:
222; AVX:       # BB#0: # %entry
223; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
224; AVX-NEXT:    retq
225entry:
226  %A = mul <8 x i16> %i, %j
227  ret <8 x i16> %A
228}
229
230define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind  {
231; SSE2-LABEL: mul_v4i32:
232; SSE2:       # BB#0: # %entry
233; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
234; SSE2-NEXT:    pmuludq %xmm1, %xmm0
235; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
236; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
237; SSE2-NEXT:    pmuludq %xmm2, %xmm1
238; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
239; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
240; SSE2-NEXT:    retq
241;
242; SSE41-LABEL: mul_v4i32:
243; SSE41:       # BB#0: # %entry
244; SSE41-NEXT:    pmulld %xmm1, %xmm0
245; SSE41-NEXT:    retq
246;
247; AVX-LABEL: mul_v4i32:
248; AVX:       # BB#0: # %entry
249; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
250; AVX-NEXT:    retq
251entry:
252  %A = mul <4 x i32> %i, %j
253  ret <4 x i32> %A
254}
255
256define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind  {
257; SSE-LABEL: mul_v2i64:
258; SSE:       # BB#0: # %entry
259; SSE-NEXT:    movdqa %xmm0, %xmm2
260; SSE-NEXT:    pmuludq %xmm1, %xmm2
261; SSE-NEXT:    movdqa %xmm1, %xmm3
262; SSE-NEXT:    psrlq $32, %xmm3
263; SSE-NEXT:    pmuludq %xmm0, %xmm3
264; SSE-NEXT:    psllq $32, %xmm3
265; SSE-NEXT:    paddq %xmm3, %xmm2
266; SSE-NEXT:    psrlq $32, %xmm0
267; SSE-NEXT:    pmuludq %xmm1, %xmm0
268; SSE-NEXT:    psllq $32, %xmm0
269; SSE-NEXT:    paddq %xmm2, %xmm0
270; SSE-NEXT:    retq
271;
272; AVX-LABEL: mul_v2i64:
273; AVX:       # BB#0: # %entry
274; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
275; AVX-NEXT:    vpsrlq $32, %xmm1, %xmm3
276; AVX-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
277; AVX-NEXT:    vpsllq $32, %xmm3, %xmm3
278; AVX-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
279; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
280; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
281; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
282; AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
283; AVX-NEXT:    retq
284entry:
285  %A = mul <2 x i64> %i, %j
286  ret <2 x i64> %A
287}
288
289declare void @foo()
290
291define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind  {
292; SSE2-LABEL: mul_v4i32spill:
293; SSE2:       # BB#0: # %entry
294; SSE2-NEXT:    subq $40, %rsp
295; SSE2-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
296; SSE2-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
297; SSE2-NEXT:    callq foo
298; SSE2-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
299; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
300; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
301; SSE2-NEXT:    pmuludq %xmm2, %xmm0
302; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
303; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
304; SSE2-NEXT:    pmuludq %xmm1, %xmm2
305; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
306; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
307; SSE2-NEXT:    addq $40, %rsp
308; SSE2-NEXT:    retq
309;
310; SSE41-LABEL: mul_v4i32spill:
311; SSE41:       # BB#0: # %entry
312; SSE41-NEXT:    subq $40, %rsp
313; SSE41-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
314; SSE41-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
315; SSE41-NEXT:    callq foo
316; SSE41-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
317; SSE41-NEXT:    pmulld {{[0-9]+}}(%rsp), %xmm0 # 16-byte Folded Reload
318; SSE41-NEXT:    addq $40, %rsp
319; SSE41-NEXT:    retq
320;
321; AVX-LABEL: mul_v4i32spill:
322; AVX:       # BB#0: # %entry
323; AVX-NEXT:    subq $40, %rsp
324; AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
325; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
326; AVX-NEXT:    callq foo
327; AVX-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
328; AVX-NEXT:    vpmulld {{[0-9]+}}(%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
329; AVX-NEXT:    addq $40, %rsp
330; AVX-NEXT:    retq
331entry:
332  ; Use a call to force spills.
333  call void @foo()
334  %A = mul <4 x i32> %i, %j
335  ret <4 x i32> %A
336}
337
338define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind  {
339; SSE-LABEL: mul_v2i64spill:
340; SSE:       # BB#0: # %entry
341; SSE-NEXT:    subq $40, %rsp
342; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
343; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
344; SSE-NEXT:    callq foo
345; SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
346; SSE-NEXT:    movdqa %xmm0, %xmm2
347; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload
348; SSE-NEXT:    pmuludq %xmm3, %xmm2
349; SSE-NEXT:    movdqa %xmm3, %xmm1
350; SSE-NEXT:    psrlq $32, %xmm1
351; SSE-NEXT:    pmuludq %xmm0, %xmm1
352; SSE-NEXT:    psllq $32, %xmm1
353; SSE-NEXT:    paddq %xmm1, %xmm2
354; SSE-NEXT:    psrlq $32, %xmm0
355; SSE-NEXT:    pmuludq %xmm3, %xmm0
356; SSE-NEXT:    psllq $32, %xmm0
357; SSE-NEXT:    paddq %xmm2, %xmm0
358; SSE-NEXT:    addq $40, %rsp
359; SSE-NEXT:    retq
360;
361; AVX-LABEL: mul_v2i64spill:
362; AVX:       # BB#0: # %entry
363; AVX-NEXT:    subq $40, %rsp
364; AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
365; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
366; AVX-NEXT:    callq foo
367; AVX-NEXT:    vmovdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
368; AVX-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
369; AVX-NEXT:    vpmuludq %xmm2, %xmm3, %xmm0
370; AVX-NEXT:    vpsrlq $32, %xmm2, %xmm1
371; AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
372; AVX-NEXT:    vpsllq $32, %xmm1, %xmm1
373; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
374; AVX-NEXT:    vpsrlq $32, %xmm3, %xmm1
375; AVX-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
376; AVX-NEXT:    vpsllq $32, %xmm1, %xmm1
377; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
378; AVX-NEXT:    addq $40, %rsp
379; AVX-NEXT:    retq
380entry:
381  ; Use a call to force spills.
382  call void @foo()
383  %A = mul <2 x i64> %i, %j
384  ret <2 x i64> %A
385}
386
387define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind  {
388; SSE2-LABEL: mul_v32i8c:
389; SSE2:       # BB#0: # %entry
390; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
391; SSE2-NEXT:    psraw $8, %xmm2
392; SSE2-NEXT:    movdqa %xmm0, %xmm3
393; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
394; SSE2-NEXT:    psraw $8, %xmm3
395; SSE2-NEXT:    pmullw %xmm2, %xmm3
396; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
397; SSE2-NEXT:    pand %xmm4, %xmm3
398; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
399; SSE2-NEXT:    psraw $8, %xmm0
400; SSE2-NEXT:    pmullw %xmm2, %xmm0
401; SSE2-NEXT:    pand %xmm4, %xmm0
402; SSE2-NEXT:    packuswb %xmm3, %xmm0
403; SSE2-NEXT:    movdqa %xmm1, %xmm3
404; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
405; SSE2-NEXT:    psraw $8, %xmm3
406; SSE2-NEXT:    pmullw %xmm2, %xmm3
407; SSE2-NEXT:    pand %xmm4, %xmm3
408; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
409; SSE2-NEXT:    psraw $8, %xmm1
410; SSE2-NEXT:    pmullw %xmm2, %xmm1
411; SSE2-NEXT:    pand %xmm4, %xmm1
412; SSE2-NEXT:    packuswb %xmm3, %xmm1
413; SSE2-NEXT:    retq
414;
415; SSE41-LABEL: mul_v32i8c:
416; SSE41:       # BB#0: # %entry
417; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
418; SSE41-NEXT:    pmovsxbw {{.*}}(%rip), %xmm4
419; SSE41-NEXT:    pmullw %xmm4, %xmm2
420; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
421; SSE41-NEXT:    pand %xmm5, %xmm2
422; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
423; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
424; SSE41-NEXT:    pmullw %xmm4, %xmm0
425; SSE41-NEXT:    pand %xmm5, %xmm0
426; SSE41-NEXT:    packuswb %xmm0, %xmm2
427; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
428; SSE41-NEXT:    pmullw %xmm4, %xmm3
429; SSE41-NEXT:    pand %xmm5, %xmm3
430; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
431; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
432; SSE41-NEXT:    pmullw %xmm4, %xmm0
433; SSE41-NEXT:    pand %xmm5, %xmm0
434; SSE41-NEXT:    packuswb %xmm0, %xmm3
435; SSE41-NEXT:    movdqa %xmm2, %xmm0
436; SSE41-NEXT:    movdqa %xmm3, %xmm1
437; SSE41-NEXT:    retq
438;
439; AVX2-LABEL: mul_v32i8c:
440; AVX2:       # BB#0: # %entry
441; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
442; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
443; AVX2-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm2
444; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
445; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
446; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
447; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
448; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
449; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
450; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
451; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
452; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
453; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
454; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
455; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
456; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
457; AVX2-NEXT:    retq
458;
459; AVX512F-LABEL: mul_v32i8c:
460; AVX512F:       # BB#0: # %entry
461; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
462; AVX512F-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm2
463; AVX512F-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
464; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
465; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
466; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
467; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
468; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
469; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
470; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
471; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
472; AVX512F-NEXT:    retq
473;
474; AVX512BW-LABEL: mul_v32i8c:
475; AVX512BW:       # BB#0: # %entry
476; AVX512BW-NEXT:    vmovaps {{.*#+}} ymm1 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
477; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
478; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
479; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
480; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
481; AVX512BW-NEXT:    retq
482entry:
483  %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
484  ret <32 x i8> %A
485}
486
487define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind  {
488; SSE-LABEL: mul_v16i16c:
489; SSE:       # BB#0: # %entry
490; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
491; SSE-NEXT:    pmullw %xmm2, %xmm0
492; SSE-NEXT:    pmullw %xmm2, %xmm1
493; SSE-NEXT:    retq
494;
495; AVX-LABEL: mul_v16i16c:
496; AVX:       # BB#0: # %entry
497; AVX-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
498; AVX-NEXT:    retq
499entry:
500  %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
501  ret <16 x i16> %A
502}
503
504define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind  {
505; SSE2-LABEL: mul_v8i32c:
506; SSE2:       # BB#0: # %entry
507; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
508; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
509; SSE2-NEXT:    pmuludq %xmm2, %xmm0
510; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
511; SSE2-NEXT:    pmuludq %xmm2, %xmm3
512; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
513; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
514; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
515; SSE2-NEXT:    pmuludq %xmm2, %xmm1
516; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
517; SSE2-NEXT:    pmuludq %xmm2, %xmm3
518; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
519; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
520; SSE2-NEXT:    retq
521;
522; SSE41-LABEL: mul_v8i32c:
523; SSE41:       # BB#0: # %entry
524; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [117,117,117,117]
525; SSE41-NEXT:    pmulld %xmm2, %xmm0
526; SSE41-NEXT:    pmulld %xmm2, %xmm1
527; SSE41-NEXT:    retq
528;
529; AVX-LABEL: mul_v8i32c:
530; AVX:       # BB#0: # %entry
531; AVX-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
532; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
533; AVX-NEXT:    retq
534entry:
535  %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 >
536  ret <8 x i32> %A
537}
538
539define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind  {
540; SSE-LABEL: mul_v4i64c:
541; SSE:       # BB#0: # %entry
542; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [117,117]
543; SSE-NEXT:    movdqa %xmm0, %xmm3
544; SSE-NEXT:    pmuludq %xmm2, %xmm3
545; SSE-NEXT:    psrlq $32, %xmm0
546; SSE-NEXT:    pmuludq %xmm2, %xmm0
547; SSE-NEXT:    psllq $32, %xmm0
548; SSE-NEXT:    paddq %xmm3, %xmm0
549; SSE-NEXT:    movdqa %xmm1, %xmm3
550; SSE-NEXT:    pmuludq %xmm2, %xmm3
551; SSE-NEXT:    psrlq $32, %xmm1
552; SSE-NEXT:    pmuludq %xmm2, %xmm1
553; SSE-NEXT:    psllq $32, %xmm1
554; SSE-NEXT:    paddq %xmm3, %xmm1
555; SSE-NEXT:    retq
556;
557; AVX-LABEL: mul_v4i64c:
558; AVX:       # BB#0: # %entry
559; AVX-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
560; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
561; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm0
562; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
563; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
564; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
565; AVX-NEXT:    retq
566entry:
567  %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 >
568  ret <4 x i64> %A
569}
570
571define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind  {
572; SSE2-LABEL: mul_v32i8:
573; SSE2:       # BB#0: # %entry
574; SSE2-NEXT:    movdqa %xmm2, %xmm4
575; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
576; SSE2-NEXT:    psraw $8, %xmm4
577; SSE2-NEXT:    movdqa %xmm0, %xmm5
578; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
579; SSE2-NEXT:    psraw $8, %xmm5
580; SSE2-NEXT:    pmullw %xmm4, %xmm5
581; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
582; SSE2-NEXT:    pand %xmm4, %xmm5
583; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
584; SSE2-NEXT:    psraw $8, %xmm2
585; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
586; SSE2-NEXT:    psraw $8, %xmm0
587; SSE2-NEXT:    pmullw %xmm2, %xmm0
588; SSE2-NEXT:    pand %xmm4, %xmm0
589; SSE2-NEXT:    packuswb %xmm5, %xmm0
590; SSE2-NEXT:    movdqa %xmm3, %xmm2
591; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
592; SSE2-NEXT:    psraw $8, %xmm2
593; SSE2-NEXT:    movdqa %xmm1, %xmm5
594; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
595; SSE2-NEXT:    psraw $8, %xmm5
596; SSE2-NEXT:    pmullw %xmm2, %xmm5
597; SSE2-NEXT:    pand %xmm4, %xmm5
598; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
599; SSE2-NEXT:    psraw $8, %xmm3
600; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
601; SSE2-NEXT:    psraw $8, %xmm1
602; SSE2-NEXT:    pmullw %xmm3, %xmm1
603; SSE2-NEXT:    pand %xmm4, %xmm1
604; SSE2-NEXT:    packuswb %xmm5, %xmm1
605; SSE2-NEXT:    retq
606;
607; SSE41-LABEL: mul_v32i8:
608; SSE41:       # BB#0: # %entry
609; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
610; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
611; SSE41-NEXT:    pmullw %xmm5, %xmm4
612; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
613; SSE41-NEXT:    pand %xmm5, %xmm4
614; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
615; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
616; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
617; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
618; SSE41-NEXT:    pmullw %xmm2, %xmm0
619; SSE41-NEXT:    pand %xmm5, %xmm0
620; SSE41-NEXT:    packuswb %xmm0, %xmm4
621; SSE41-NEXT:    pmovsxbw %xmm3, %xmm0
622; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
623; SSE41-NEXT:    pmullw %xmm0, %xmm2
624; SSE41-NEXT:    pand %xmm5, %xmm2
625; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
626; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
627; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
628; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
629; SSE41-NEXT:    pmullw %xmm0, %xmm1
630; SSE41-NEXT:    pand %xmm5, %xmm1
631; SSE41-NEXT:    packuswb %xmm1, %xmm2
632; SSE41-NEXT:    movdqa %xmm4, %xmm0
633; SSE41-NEXT:    movdqa %xmm2, %xmm1
634; SSE41-NEXT:    retq
635;
636; AVX2-LABEL: mul_v32i8:
637; AVX2:       # BB#0: # %entry
638; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
639; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
640; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
641; AVX2-NEXT:    vpmovsxbw %xmm3, %ymm3
642; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
643; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
644; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
645; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
646; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
647; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
648; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
649; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
650; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
651; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
652; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
653; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
654; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
655; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
656; AVX2-NEXT:    retq
657;
658; AVX512F-LABEL: mul_v32i8:
659; AVX512F:       # BB#0: # %entry
660; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm2
661; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm3
662; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
663; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
664; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
665; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
666; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
667; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
668; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
669; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
670; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
671; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
672; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
673; AVX512F-NEXT:    retq
674;
675; AVX512BW-LABEL: mul_v32i8:
676; AVX512BW:       # BB#0: # %entry
677; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
678; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
679; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
680; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
681; AVX512BW-NEXT:    retq
682entry:
683  %A = mul <32 x i8> %i, %j
684  ret <32 x i8> %A
685}
686
687define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind  {
688; SSE-LABEL: mul_v16i16:
689; SSE:       # BB#0: # %entry
690; SSE-NEXT:    pmullw %xmm2, %xmm0
691; SSE-NEXT:    pmullw %xmm3, %xmm1
692; SSE-NEXT:    retq
693;
694; AVX-LABEL: mul_v16i16:
695; AVX:       # BB#0: # %entry
696; AVX-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
697; AVX-NEXT:    retq
698entry:
699  %A = mul <16 x i16> %i, %j
700  ret <16 x i16> %A
701}
702
703define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind  {
704; SSE2-LABEL: mul_v8i32:
705; SSE2:       # BB#0: # %entry
706; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
707; SSE2-NEXT:    pmuludq %xmm2, %xmm0
708; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
709; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
710; SSE2-NEXT:    pmuludq %xmm4, %xmm2
711; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
712; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
713; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
714; SSE2-NEXT:    pmuludq %xmm3, %xmm1
715; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
716; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
717; SSE2-NEXT:    pmuludq %xmm2, %xmm3
718; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
719; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
720; SSE2-NEXT:    retq
721;
722; SSE41-LABEL: mul_v8i32:
723; SSE41:       # BB#0: # %entry
724; SSE41-NEXT:    pmulld %xmm2, %xmm0
725; SSE41-NEXT:    pmulld %xmm3, %xmm1
726; SSE41-NEXT:    retq
727;
728; AVX-LABEL: mul_v8i32:
729; AVX:       # BB#0: # %entry
730; AVX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
731; AVX-NEXT:    retq
732entry:
733  %A = mul <8 x i32> %i, %j
734  ret <8 x i32> %A
735}
736
737define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind  {
738; SSE-LABEL: mul_v4i64:
739; SSE:       # BB#0: # %entry
740; SSE-NEXT:    movdqa %xmm0, %xmm4
741; SSE-NEXT:    pmuludq %xmm2, %xmm4
742; SSE-NEXT:    movdqa %xmm2, %xmm5
743; SSE-NEXT:    psrlq $32, %xmm5
744; SSE-NEXT:    pmuludq %xmm0, %xmm5
745; SSE-NEXT:    psllq $32, %xmm5
746; SSE-NEXT:    paddq %xmm5, %xmm4
747; SSE-NEXT:    psrlq $32, %xmm0
748; SSE-NEXT:    pmuludq %xmm2, %xmm0
749; SSE-NEXT:    psllq $32, %xmm0
750; SSE-NEXT:    paddq %xmm4, %xmm0
751; SSE-NEXT:    movdqa %xmm1, %xmm2
752; SSE-NEXT:    pmuludq %xmm3, %xmm2
753; SSE-NEXT:    movdqa %xmm3, %xmm4
754; SSE-NEXT:    psrlq $32, %xmm4
755; SSE-NEXT:    pmuludq %xmm1, %xmm4
756; SSE-NEXT:    psllq $32, %xmm4
757; SSE-NEXT:    paddq %xmm4, %xmm2
758; SSE-NEXT:    psrlq $32, %xmm1
759; SSE-NEXT:    pmuludq %xmm3, %xmm1
760; SSE-NEXT:    psllq $32, %xmm1
761; SSE-NEXT:    paddq %xmm2, %xmm1
762; SSE-NEXT:    retq
763;
764; AVX-LABEL: mul_v4i64:
765; AVX:       # BB#0: # %entry
766; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
767; AVX-NEXT:    vpsrlq $32, %ymm1, %ymm3
768; AVX-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3
769; AVX-NEXT:    vpsllq $32, %ymm3, %ymm3
770; AVX-NEXT:    vpaddq %ymm3, %ymm2, %ymm2
771; AVX-NEXT:    vpsrlq $32, %ymm0, %ymm0
772; AVX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
773; AVX-NEXT:    vpsllq $32, %ymm0, %ymm0
774; AVX-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
775; AVX-NEXT:    retq
776entry:
777  %A = mul <4 x i64> %i, %j
778  ret <4 x i64> %A
779}
780
781define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
782; SSE2-LABEL: mul_v64i8c:
783; SSE2:       # BB#0: # %entry
784; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
785; SSE2-NEXT:    psraw $8, %xmm4
786; SSE2-NEXT:    movdqa %xmm0, %xmm6
787; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
788; SSE2-NEXT:    psraw $8, %xmm6
789; SSE2-NEXT:    pmullw %xmm4, %xmm6
790; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
791; SSE2-NEXT:    pand %xmm5, %xmm6
792; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
793; SSE2-NEXT:    psraw $8, %xmm0
794; SSE2-NEXT:    pmullw %xmm4, %xmm0
795; SSE2-NEXT:    pand %xmm5, %xmm0
796; SSE2-NEXT:    packuswb %xmm6, %xmm0
797; SSE2-NEXT:    movdqa %xmm1, %xmm6
798; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
799; SSE2-NEXT:    psraw $8, %xmm6
800; SSE2-NEXT:    pmullw %xmm4, %xmm6
801; SSE2-NEXT:    pand %xmm5, %xmm6
802; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
803; SSE2-NEXT:    psraw $8, %xmm1
804; SSE2-NEXT:    pmullw %xmm4, %xmm1
805; SSE2-NEXT:    pand %xmm5, %xmm1
806; SSE2-NEXT:    packuswb %xmm6, %xmm1
807; SSE2-NEXT:    movdqa %xmm2, %xmm6
808; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
809; SSE2-NEXT:    psraw $8, %xmm6
810; SSE2-NEXT:    pmullw %xmm4, %xmm6
811; SSE2-NEXT:    pand %xmm5, %xmm6
812; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
813; SSE2-NEXT:    psraw $8, %xmm2
814; SSE2-NEXT:    pmullw %xmm4, %xmm2
815; SSE2-NEXT:    pand %xmm5, %xmm2
816; SSE2-NEXT:    packuswb %xmm6, %xmm2
817; SSE2-NEXT:    movdqa %xmm3, %xmm6
818; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
819; SSE2-NEXT:    psraw $8, %xmm6
820; SSE2-NEXT:    pmullw %xmm4, %xmm6
821; SSE2-NEXT:    pand %xmm5, %xmm6
822; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
823; SSE2-NEXT:    psraw $8, %xmm3
824; SSE2-NEXT:    pmullw %xmm4, %xmm3
825; SSE2-NEXT:    pand %xmm5, %xmm3
826; SSE2-NEXT:    packuswb %xmm6, %xmm3
827; SSE2-NEXT:    retq
828;
829; SSE41-LABEL: mul_v64i8c:
830; SSE41:       # BB#0: # %entry
831; SSE41-NEXT:    movdqa %xmm1, %xmm4
832; SSE41-NEXT:    movdqa %xmm0, %xmm1
833; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
834; SSE41-NEXT:    pmovsxbw {{.*}}(%rip), %xmm6
835; SSE41-NEXT:    pmullw %xmm6, %xmm0
836; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
837; SSE41-NEXT:    pand %xmm7, %xmm0
838; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
839; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
840; SSE41-NEXT:    pmullw %xmm6, %xmm1
841; SSE41-NEXT:    pand %xmm7, %xmm1
842; SSE41-NEXT:    packuswb %xmm1, %xmm0
843; SSE41-NEXT:    pmovsxbw %xmm4, %xmm1
844; SSE41-NEXT:    pmullw %xmm6, %xmm1
845; SSE41-NEXT:    pand %xmm7, %xmm1
846; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
847; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
848; SSE41-NEXT:    pmullw %xmm6, %xmm4
849; SSE41-NEXT:    pand %xmm7, %xmm4
850; SSE41-NEXT:    packuswb %xmm4, %xmm1
851; SSE41-NEXT:    pmovsxbw %xmm2, %xmm4
852; SSE41-NEXT:    pmullw %xmm6, %xmm4
853; SSE41-NEXT:    pand %xmm7, %xmm4
854; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
855; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
856; SSE41-NEXT:    pmullw %xmm6, %xmm2
857; SSE41-NEXT:    pand %xmm7, %xmm2
858; SSE41-NEXT:    packuswb %xmm2, %xmm4
859; SSE41-NEXT:    pmovsxbw %xmm3, %xmm5
860; SSE41-NEXT:    pmullw %xmm6, %xmm5
861; SSE41-NEXT:    pand %xmm7, %xmm5
862; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
863; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
864; SSE41-NEXT:    pmullw %xmm6, %xmm2
865; SSE41-NEXT:    pand %xmm7, %xmm2
866; SSE41-NEXT:    packuswb %xmm2, %xmm5
867; SSE41-NEXT:    movdqa %xmm4, %xmm2
868; SSE41-NEXT:    movdqa %xmm5, %xmm3
869; SSE41-NEXT:    retq
870;
871; AVX2-LABEL: mul_v64i8c:
872; AVX2:       # BB#0: # %entry
873; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
874; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
875; AVX2-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm3
876; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
877; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
878; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
879; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
880; AVX2-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
881; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
882; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
883; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
884; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
885; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
886; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
887; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
888; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
889; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
890; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
891; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
892; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
893; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
894; AVX2-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
895; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
896; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
897; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
898; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
899; AVX2-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
900; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
901; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
902; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
903; AVX2-NEXT:    retq
904;
905; AVX512F-LABEL: mul_v64i8c:
906; AVX512F:       # BB#0: # %entry
907; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm2
908; AVX512F-NEXT:    vpmovsxbw {{.*}}(%rip), %ymm3
909; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
910; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
911; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
912; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
913; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
914; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
915; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
916; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
917; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
918; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm2
919; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
920; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
921; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
922; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
923; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
924; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
925; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
926; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
927; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
928; AVX512F-NEXT:    retq
929;
930; AVX512BW-LABEL: mul_v64i8c:
931; AVX512BW:       # BB#0: # %entry
932; AVX512BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
933; AVX512BW-NEXT:    vpextrb $1, %xmm1, %eax
934; AVX512BW-NEXT:    movb $117, %cl
935; AVX512BW-NEXT:    mulb %cl
936; AVX512BW-NEXT:    movzbl %al, %edx
937; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
938; AVX512BW-NEXT:    mulb %cl
939; AVX512BW-NEXT:    movzbl %al, %eax
940; AVX512BW-NEXT:    vmovd %eax, %xmm2
941; AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
942; AVX512BW-NEXT:    vpextrb $2, %xmm1, %eax
943; AVX512BW-NEXT:    mulb %cl
944; AVX512BW-NEXT:    movzbl %al, %eax
945; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
946; AVX512BW-NEXT:    vpextrb $3, %xmm1, %eax
947; AVX512BW-NEXT:    mulb %cl
948; AVX512BW-NEXT:    movzbl %al, %eax
949; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
950; AVX512BW-NEXT:    vpextrb $4, %xmm1, %eax
951; AVX512BW-NEXT:    mulb %cl
952; AVX512BW-NEXT:    movzbl %al, %eax
953; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
954; AVX512BW-NEXT:    vpextrb $5, %xmm1, %eax
955; AVX512BW-NEXT:    mulb %cl
956; AVX512BW-NEXT:    movzbl %al, %eax
957; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
958; AVX512BW-NEXT:    vpextrb $6, %xmm1, %eax
959; AVX512BW-NEXT:    mulb %cl
960; AVX512BW-NEXT:    movzbl %al, %eax
961; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
962; AVX512BW-NEXT:    vpextrb $7, %xmm1, %eax
963; AVX512BW-NEXT:    mulb %cl
964; AVX512BW-NEXT:    movzbl %al, %eax
965; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
966; AVX512BW-NEXT:    vpextrb $8, %xmm1, %eax
967; AVX512BW-NEXT:    mulb %cl
968; AVX512BW-NEXT:    movzbl %al, %eax
969; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
970; AVX512BW-NEXT:    vpextrb $9, %xmm1, %eax
971; AVX512BW-NEXT:    mulb %cl
972; AVX512BW-NEXT:    movzbl %al, %eax
973; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
974; AVX512BW-NEXT:    vpextrb $10, %xmm1, %eax
975; AVX512BW-NEXT:    mulb %cl
976; AVX512BW-NEXT:    movzbl %al, %eax
977; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
978; AVX512BW-NEXT:    vpextrb $11, %xmm1, %eax
979; AVX512BW-NEXT:    mulb %cl
980; AVX512BW-NEXT:    movzbl %al, %eax
981; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
982; AVX512BW-NEXT:    vpextrb $12, %xmm1, %eax
983; AVX512BW-NEXT:    mulb %cl
984; AVX512BW-NEXT:    movzbl %al, %eax
985; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
986; AVX512BW-NEXT:    vpextrb $13, %xmm1, %eax
987; AVX512BW-NEXT:    mulb %cl
988; AVX512BW-NEXT:    movzbl %al, %eax
989; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
990; AVX512BW-NEXT:    vpextrb $14, %xmm1, %eax
991; AVX512BW-NEXT:    mulb %cl
992; AVX512BW-NEXT:    movzbl %al, %eax
993; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
994; AVX512BW-NEXT:    vpextrb $15, %xmm1, %eax
995; AVX512BW-NEXT:    mulb %cl
996; AVX512BW-NEXT:    movzbl %al, %eax
997; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
998; AVX512BW-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
999; AVX512BW-NEXT:    vpextrb $1, %xmm2, %eax
1000; AVX512BW-NEXT:    mulb %cl
1001; AVX512BW-NEXT:    movzbl %al, %edx
1002; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
1003; AVX512BW-NEXT:    mulb %cl
1004; AVX512BW-NEXT:    movzbl %al, %eax
1005; AVX512BW-NEXT:    vmovd %eax, %xmm3
1006; AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm3, %xmm3
1007; AVX512BW-NEXT:    vpextrb $2, %xmm2, %eax
1008; AVX512BW-NEXT:    mulb %cl
1009; AVX512BW-NEXT:    movzbl %al, %eax
1010; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm3, %xmm3
1011; AVX512BW-NEXT:    vpextrb $3, %xmm2, %eax
1012; AVX512BW-NEXT:    mulb %cl
1013; AVX512BW-NEXT:    movzbl %al, %eax
1014; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm3, %xmm3
1015; AVX512BW-NEXT:    vpextrb $4, %xmm2, %eax
1016; AVX512BW-NEXT:    mulb %cl
1017; AVX512BW-NEXT:    movzbl %al, %eax
1018; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm3, %xmm3
1019; AVX512BW-NEXT:    vpextrb $5, %xmm2, %eax
1020; AVX512BW-NEXT:    mulb %cl
1021; AVX512BW-NEXT:    movzbl %al, %eax
1022; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm3, %xmm3
1023; AVX512BW-NEXT:    vpextrb $6, %xmm2, %eax
1024; AVX512BW-NEXT:    mulb %cl
1025; AVX512BW-NEXT:    movzbl %al, %eax
1026; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm3, %xmm3
1027; AVX512BW-NEXT:    vpextrb $7, %xmm2, %eax
1028; AVX512BW-NEXT:    mulb %cl
1029; AVX512BW-NEXT:    movzbl %al, %eax
1030; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm3, %xmm3
1031; AVX512BW-NEXT:    vpextrb $8, %xmm2, %eax
1032; AVX512BW-NEXT:    mulb %cl
1033; AVX512BW-NEXT:    movzbl %al, %eax
1034; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm3, %xmm3
1035; AVX512BW-NEXT:    vpextrb $9, %xmm2, %eax
1036; AVX512BW-NEXT:    mulb %cl
1037; AVX512BW-NEXT:    movzbl %al, %eax
1038; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm3, %xmm3
1039; AVX512BW-NEXT:    vpextrb $10, %xmm2, %eax
1040; AVX512BW-NEXT:    mulb %cl
1041; AVX512BW-NEXT:    movzbl %al, %eax
1042; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm3, %xmm3
1043; AVX512BW-NEXT:    vpextrb $11, %xmm2, %eax
1044; AVX512BW-NEXT:    mulb %cl
1045; AVX512BW-NEXT:    movzbl %al, %eax
1046; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm3, %xmm3
1047; AVX512BW-NEXT:    vpextrb $12, %xmm2, %eax
1048; AVX512BW-NEXT:    mulb %cl
1049; AVX512BW-NEXT:    movzbl %al, %eax
1050; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm3, %xmm3
1051; AVX512BW-NEXT:    vpextrb $13, %xmm2, %eax
1052; AVX512BW-NEXT:    mulb %cl
1053; AVX512BW-NEXT:    movzbl %al, %eax
1054; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm3, %xmm3
1055; AVX512BW-NEXT:    vpextrb $14, %xmm2, %eax
1056; AVX512BW-NEXT:    mulb %cl
1057; AVX512BW-NEXT:    movzbl %al, %eax
1058; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm3, %xmm3
1059; AVX512BW-NEXT:    vpextrb $15, %xmm2, %eax
1060; AVX512BW-NEXT:    mulb %cl
1061; AVX512BW-NEXT:    movzbl %al, %eax
1062; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm3, %xmm2
1063; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
1064; AVX512BW-NEXT:    vextracti32x4 $1, %zmm0, %xmm2
1065; AVX512BW-NEXT:    vpextrb $1, %xmm2, %eax
1066; AVX512BW-NEXT:    mulb %cl
1067; AVX512BW-NEXT:    movzbl %al, %edx
1068; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
1069; AVX512BW-NEXT:    mulb %cl
1070; AVX512BW-NEXT:    movzbl %al, %eax
1071; AVX512BW-NEXT:    vmovd %eax, %xmm3
1072; AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm3, %xmm3
1073; AVX512BW-NEXT:    vpextrb $2, %xmm2, %eax
1074; AVX512BW-NEXT:    mulb %cl
1075; AVX512BW-NEXT:    movzbl %al, %eax
1076; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm3, %xmm3
1077; AVX512BW-NEXT:    vpextrb $3, %xmm2, %eax
1078; AVX512BW-NEXT:    mulb %cl
1079; AVX512BW-NEXT:    movzbl %al, %eax
1080; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm3, %xmm3
1081; AVX512BW-NEXT:    vpextrb $4, %xmm2, %eax
1082; AVX512BW-NEXT:    mulb %cl
1083; AVX512BW-NEXT:    movzbl %al, %eax
1084; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm3, %xmm3
1085; AVX512BW-NEXT:    vpextrb $5, %xmm2, %eax
1086; AVX512BW-NEXT:    mulb %cl
1087; AVX512BW-NEXT:    movzbl %al, %eax
1088; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm3, %xmm3
1089; AVX512BW-NEXT:    vpextrb $6, %xmm2, %eax
1090; AVX512BW-NEXT:    mulb %cl
1091; AVX512BW-NEXT:    movzbl %al, %eax
1092; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm3, %xmm3
1093; AVX512BW-NEXT:    vpextrb $7, %xmm2, %eax
1094; AVX512BW-NEXT:    mulb %cl
1095; AVX512BW-NEXT:    movzbl %al, %eax
1096; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm3, %xmm3
1097; AVX512BW-NEXT:    vpextrb $8, %xmm2, %eax
1098; AVX512BW-NEXT:    mulb %cl
1099; AVX512BW-NEXT:    movzbl %al, %eax
1100; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm3, %xmm3
1101; AVX512BW-NEXT:    vpextrb $9, %xmm2, %eax
1102; AVX512BW-NEXT:    mulb %cl
1103; AVX512BW-NEXT:    movzbl %al, %eax
1104; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm3, %xmm3
1105; AVX512BW-NEXT:    vpextrb $10, %xmm2, %eax
1106; AVX512BW-NEXT:    mulb %cl
1107; AVX512BW-NEXT:    movzbl %al, %eax
1108; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm3, %xmm3
1109; AVX512BW-NEXT:    vpextrb $11, %xmm2, %eax
1110; AVX512BW-NEXT:    mulb %cl
1111; AVX512BW-NEXT:    movzbl %al, %eax
1112; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm3, %xmm3
1113; AVX512BW-NEXT:    vpextrb $12, %xmm2, %eax
1114; AVX512BW-NEXT:    mulb %cl
1115; AVX512BW-NEXT:    movzbl %al, %eax
1116; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm3, %xmm3
1117; AVX512BW-NEXT:    vpextrb $13, %xmm2, %eax
1118; AVX512BW-NEXT:    mulb %cl
1119; AVX512BW-NEXT:    movzbl %al, %eax
1120; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm3, %xmm3
1121; AVX512BW-NEXT:    vpextrb $14, %xmm2, %eax
1122; AVX512BW-NEXT:    mulb %cl
1123; AVX512BW-NEXT:    movzbl %al, %eax
1124; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm3, %xmm3
1125; AVX512BW-NEXT:    vpextrb $15, %xmm2, %eax
1126; AVX512BW-NEXT:    mulb %cl
1127; AVX512BW-NEXT:    movzbl %al, %eax
1128; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm3, %xmm2
1129; AVX512BW-NEXT:    vpextrb $1, %xmm0, %eax
1130; AVX512BW-NEXT:    mulb %cl
1131; AVX512BW-NEXT:    movzbl %al, %edx
1132; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
1133; AVX512BW-NEXT:    mulb %cl
1134; AVX512BW-NEXT:    movzbl %al, %eax
1135; AVX512BW-NEXT:    vmovd %eax, %xmm3
1136; AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm3, %xmm3
1137; AVX512BW-NEXT:    vpextrb $2, %xmm0, %eax
1138; AVX512BW-NEXT:    mulb %cl
1139; AVX512BW-NEXT:    movzbl %al, %eax
1140; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm3, %xmm3
1141; AVX512BW-NEXT:    vpextrb $3, %xmm0, %eax
1142; AVX512BW-NEXT:    mulb %cl
1143; AVX512BW-NEXT:    movzbl %al, %eax
1144; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm3, %xmm3
1145; AVX512BW-NEXT:    vpextrb $4, %xmm0, %eax
1146; AVX512BW-NEXT:    mulb %cl
1147; AVX512BW-NEXT:    movzbl %al, %eax
1148; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm3, %xmm3
1149; AVX512BW-NEXT:    vpextrb $5, %xmm0, %eax
1150; AVX512BW-NEXT:    mulb %cl
1151; AVX512BW-NEXT:    movzbl %al, %eax
1152; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm3, %xmm3
1153; AVX512BW-NEXT:    vpextrb $6, %xmm0, %eax
1154; AVX512BW-NEXT:    mulb %cl
1155; AVX512BW-NEXT:    movzbl %al, %eax
1156; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm3, %xmm3
1157; AVX512BW-NEXT:    vpextrb $7, %xmm0, %eax
1158; AVX512BW-NEXT:    mulb %cl
1159; AVX512BW-NEXT:    movzbl %al, %eax
1160; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm3, %xmm3
1161; AVX512BW-NEXT:    vpextrb $8, %xmm0, %eax
1162; AVX512BW-NEXT:    mulb %cl
1163; AVX512BW-NEXT:    movzbl %al, %eax
1164; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm3, %xmm3
1165; AVX512BW-NEXT:    vpextrb $9, %xmm0, %eax
1166; AVX512BW-NEXT:    mulb %cl
1167; AVX512BW-NEXT:    movzbl %al, %eax
1168; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm3, %xmm3
1169; AVX512BW-NEXT:    vpextrb $10, %xmm0, %eax
1170; AVX512BW-NEXT:    mulb %cl
1171; AVX512BW-NEXT:    movzbl %al, %eax
1172; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm3, %xmm3
1173; AVX512BW-NEXT:    vpextrb $11, %xmm0, %eax
1174; AVX512BW-NEXT:    mulb %cl
1175; AVX512BW-NEXT:    movzbl %al, %eax
1176; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm3, %xmm3
1177; AVX512BW-NEXT:    vpextrb $12, %xmm0, %eax
1178; AVX512BW-NEXT:    mulb %cl
1179; AVX512BW-NEXT:    movzbl %al, %eax
1180; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm3, %xmm3
1181; AVX512BW-NEXT:    vpextrb $13, %xmm0, %eax
1182; AVX512BW-NEXT:    mulb %cl
1183; AVX512BW-NEXT:    movzbl %al, %eax
1184; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm3, %xmm3
1185; AVX512BW-NEXT:    vpextrb $14, %xmm0, %eax
1186; AVX512BW-NEXT:    mulb %cl
1187; AVX512BW-NEXT:    movzbl %al, %eax
1188; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm3, %xmm3
1189; AVX512BW-NEXT:    vpextrb $15, %xmm0, %eax
1190; AVX512BW-NEXT:    mulb %cl
1191; AVX512BW-NEXT:    movzbl %al, %eax
1192; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm3, %xmm0
1193; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
1194; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
1195; AVX512BW-NEXT:    retq
1196entry:
1197  %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
1198  ret <64 x i8> %A
1199}
1200
1201define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
1202; SSE2-LABEL: mul_v64i8:
1203; SSE2:       # BB#0: # %entry
1204; SSE2-NEXT:    movdqa %xmm4, %xmm8
1205; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1206; SSE2-NEXT:    psraw $8, %xmm8
1207; SSE2-NEXT:    movdqa %xmm0, %xmm9
1208; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1209; SSE2-NEXT:    psraw $8, %xmm9
1210; SSE2-NEXT:    pmullw %xmm8, %xmm9
1211; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
1212; SSE2-NEXT:    pand %xmm8, %xmm9
1213; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1214; SSE2-NEXT:    psraw $8, %xmm4
1215; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1216; SSE2-NEXT:    psraw $8, %xmm0
1217; SSE2-NEXT:    pmullw %xmm4, %xmm0
1218; SSE2-NEXT:    pand %xmm8, %xmm0
1219; SSE2-NEXT:    packuswb %xmm9, %xmm0
1220; SSE2-NEXT:    movdqa %xmm5, %xmm9
1221; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1222; SSE2-NEXT:    psraw $8, %xmm9
1223; SSE2-NEXT:    movdqa %xmm1, %xmm4
1224; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1225; SSE2-NEXT:    psraw $8, %xmm4
1226; SSE2-NEXT:    pmullw %xmm9, %xmm4
1227; SSE2-NEXT:    pand %xmm8, %xmm4
1228; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1229; SSE2-NEXT:    psraw $8, %xmm5
1230; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1231; SSE2-NEXT:    psraw $8, %xmm1
1232; SSE2-NEXT:    pmullw %xmm5, %xmm1
1233; SSE2-NEXT:    pand %xmm8, %xmm1
1234; SSE2-NEXT:    packuswb %xmm4, %xmm1
1235; SSE2-NEXT:    movdqa %xmm6, %xmm4
1236; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1237; SSE2-NEXT:    psraw $8, %xmm4
1238; SSE2-NEXT:    movdqa %xmm2, %xmm5
1239; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1240; SSE2-NEXT:    psraw $8, %xmm5
1241; SSE2-NEXT:    pmullw %xmm4, %xmm5
1242; SSE2-NEXT:    pand %xmm8, %xmm5
1243; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1244; SSE2-NEXT:    psraw $8, %xmm6
1245; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1246; SSE2-NEXT:    psraw $8, %xmm2
1247; SSE2-NEXT:    pmullw %xmm6, %xmm2
1248; SSE2-NEXT:    pand %xmm8, %xmm2
1249; SSE2-NEXT:    packuswb %xmm5, %xmm2
1250; SSE2-NEXT:    movdqa %xmm7, %xmm4
1251; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1252; SSE2-NEXT:    psraw $8, %xmm4
1253; SSE2-NEXT:    movdqa %xmm3, %xmm5
1254; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1255; SSE2-NEXT:    psraw $8, %xmm5
1256; SSE2-NEXT:    pmullw %xmm4, %xmm5
1257; SSE2-NEXT:    pand %xmm8, %xmm5
1258; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1259; SSE2-NEXT:    psraw $8, %xmm7
1260; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1261; SSE2-NEXT:    psraw $8, %xmm3
1262; SSE2-NEXT:    pmullw %xmm7, %xmm3
1263; SSE2-NEXT:    pand %xmm8, %xmm3
1264; SSE2-NEXT:    packuswb %xmm5, %xmm3
1265; SSE2-NEXT:    retq
1266;
1267; SSE41-LABEL: mul_v64i8:
1268; SSE41:       # BB#0: # %entry
1269; SSE41-NEXT:    movdqa %xmm1, %xmm8
1270; SSE41-NEXT:    movdqa %xmm0, %xmm1
1271; SSE41-NEXT:    pmovsxbw %xmm4, %xmm9
1272; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
1273; SSE41-NEXT:    pmullw %xmm9, %xmm0
1274; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
1275; SSE41-NEXT:    pand %xmm9, %xmm0
1276; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
1277; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
1278; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
1279; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
1280; SSE41-NEXT:    pmullw %xmm4, %xmm1
1281; SSE41-NEXT:    pand %xmm9, %xmm1
1282; SSE41-NEXT:    packuswb %xmm1, %xmm0
1283; SSE41-NEXT:    pmovsxbw %xmm5, %xmm4
1284; SSE41-NEXT:    pmovsxbw %xmm8, %xmm1
1285; SSE41-NEXT:    pmullw %xmm4, %xmm1
1286; SSE41-NEXT:    pand %xmm9, %xmm1
1287; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
1288; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
1289; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[2,3,0,1]
1290; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
1291; SSE41-NEXT:    pmullw %xmm4, %xmm5
1292; SSE41-NEXT:    pand %xmm9, %xmm5
1293; SSE41-NEXT:    packuswb %xmm5, %xmm1
1294; SSE41-NEXT:    pmovsxbw %xmm6, %xmm5
1295; SSE41-NEXT:    pmovsxbw %xmm2, %xmm4
1296; SSE41-NEXT:    pmullw %xmm5, %xmm4
1297; SSE41-NEXT:    pand %xmm9, %xmm4
1298; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[2,3,0,1]
1299; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
1300; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1301; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
1302; SSE41-NEXT:    pmullw %xmm5, %xmm2
1303; SSE41-NEXT:    pand %xmm9, %xmm2
1304; SSE41-NEXT:    packuswb %xmm2, %xmm4
1305; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
1306; SSE41-NEXT:    pmovsxbw %xmm3, %xmm5
1307; SSE41-NEXT:    pmullw %xmm2, %xmm5
1308; SSE41-NEXT:    pand %xmm9, %xmm5
1309; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[2,3,0,1]
1310; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
1311; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
1312; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
1313; SSE41-NEXT:    pmullw %xmm2, %xmm3
1314; SSE41-NEXT:    pand %xmm9, %xmm3
1315; SSE41-NEXT:    packuswb %xmm3, %xmm5
1316; SSE41-NEXT:    movdqa %xmm4, %xmm2
1317; SSE41-NEXT:    movdqa %xmm5, %xmm3
1318; SSE41-NEXT:    retq
1319;
1320; AVX2-LABEL: mul_v64i8:
1321; AVX2:       # BB#0: # %entry
1322; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
1323; AVX2-NEXT:    vpmovsxbw %xmm4, %ymm4
1324; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
1325; AVX2-NEXT:    vpmovsxbw %xmm5, %ymm5
1326; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
1327; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
1328; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1329; AVX2-NEXT:    vpshufb %xmm4, %xmm6, %xmm6
1330; AVX2-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
1331; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
1332; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
1333; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
1334; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
1335; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
1336; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
1337; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
1338; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1339; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm0
1340; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm2
1341; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
1342; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
1343; AVX2-NEXT:    vpmovsxbw %xmm5, %ymm5
1344; AVX2-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
1345; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
1346; AVX2-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
1347; AVX2-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
1348; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
1349; AVX2-NEXT:    vpmovsxbw %xmm3, %ymm3
1350; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
1351; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1352; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
1353; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
1354; AVX2-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
1355; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
1356; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
1357; AVX2-NEXT:    retq
1358;
1359; AVX512F-LABEL: mul_v64i8:
1360; AVX512F:       # BB#0: # %entry
1361; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm4
1362; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm5
1363; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
1364; AVX512F-NEXT:    vpmovsxwd %ymm4, %zmm4
1365; AVX512F-NEXT:    vpmovdb %zmm4, %xmm4
1366; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm2
1367; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
1368; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
1369; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
1370; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
1371; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
1372; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
1373; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
1374; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm2
1375; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm4
1376; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
1377; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
1378; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
1379; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm3
1380; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
1381; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
1382; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
1383; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
1384; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
1385; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
1386; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
1387; AVX512F-NEXT:    retq
1388;
1389; AVX512BW-LABEL: mul_v64i8:
1390; AVX512BW:       # BB#0: # %entry
1391; AVX512BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
1392; AVX512BW-NEXT:    vpextrb $1, %xmm2, %eax
1393; AVX512BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
1394; AVX512BW-NEXT:    vpextrb $1, %xmm3, %ecx
1395; AVX512BW-NEXT:    mulb %cl
1396; AVX512BW-NEXT:    movzbl %al, %ecx
1397; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
1398; AVX512BW-NEXT:    vpextrb $0, %xmm3, %edx
1399; AVX512BW-NEXT:    mulb %dl
1400; AVX512BW-NEXT:    movzbl %al, %eax
1401; AVX512BW-NEXT:    vmovd %eax, %xmm4
1402; AVX512BW-NEXT:    vpinsrb $1, %ecx, %xmm4, %xmm4
1403; AVX512BW-NEXT:    vpextrb $2, %xmm2, %eax
1404; AVX512BW-NEXT:    vpextrb $2, %xmm3, %ecx
1405; AVX512BW-NEXT:    mulb %cl
1406; AVX512BW-NEXT:    movzbl %al, %eax
1407; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm4, %xmm4
1408; AVX512BW-NEXT:    vpextrb $3, %xmm2, %eax
1409; AVX512BW-NEXT:    vpextrb $3, %xmm3, %ecx
1410; AVX512BW-NEXT:    mulb %cl
1411; AVX512BW-NEXT:    movzbl %al, %eax
1412; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm4, %xmm4
1413; AVX512BW-NEXT:    vpextrb $4, %xmm2, %eax
1414; AVX512BW-NEXT:    vpextrb $4, %xmm3, %ecx
1415; AVX512BW-NEXT:    mulb %cl
1416; AVX512BW-NEXT:    movzbl %al, %eax
1417; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm4, %xmm4
1418; AVX512BW-NEXT:    vpextrb $5, %xmm2, %eax
1419; AVX512BW-NEXT:    vpextrb $5, %xmm3, %ecx
1420; AVX512BW-NEXT:    mulb %cl
1421; AVX512BW-NEXT:    movzbl %al, %eax
1422; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm4, %xmm4
1423; AVX512BW-NEXT:    vpextrb $6, %xmm2, %eax
1424; AVX512BW-NEXT:    vpextrb $6, %xmm3, %ecx
1425; AVX512BW-NEXT:    mulb %cl
1426; AVX512BW-NEXT:    movzbl %al, %eax
1427; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm4, %xmm4
1428; AVX512BW-NEXT:    vpextrb $7, %xmm2, %eax
1429; AVX512BW-NEXT:    vpextrb $7, %xmm3, %ecx
1430; AVX512BW-NEXT:    mulb %cl
1431; AVX512BW-NEXT:    movzbl %al, %eax
1432; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm4, %xmm4
1433; AVX512BW-NEXT:    vpextrb $8, %xmm2, %eax
1434; AVX512BW-NEXT:    vpextrb $8, %xmm3, %ecx
1435; AVX512BW-NEXT:    mulb %cl
1436; AVX512BW-NEXT:    movzbl %al, %eax
1437; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm4, %xmm4
1438; AVX512BW-NEXT:    vpextrb $9, %xmm2, %eax
1439; AVX512BW-NEXT:    vpextrb $9, %xmm3, %ecx
1440; AVX512BW-NEXT:    mulb %cl
1441; AVX512BW-NEXT:    movzbl %al, %eax
1442; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm4, %xmm4
1443; AVX512BW-NEXT:    vpextrb $10, %xmm2, %eax
1444; AVX512BW-NEXT:    vpextrb $10, %xmm3, %ecx
1445; AVX512BW-NEXT:    mulb %cl
1446; AVX512BW-NEXT:    movzbl %al, %eax
1447; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm4, %xmm4
1448; AVX512BW-NEXT:    vpextrb $11, %xmm2, %eax
1449; AVX512BW-NEXT:    vpextrb $11, %xmm3, %ecx
1450; AVX512BW-NEXT:    mulb %cl
1451; AVX512BW-NEXT:    movzbl %al, %eax
1452; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm4, %xmm4
1453; AVX512BW-NEXT:    vpextrb $12, %xmm2, %eax
1454; AVX512BW-NEXT:    vpextrb $12, %xmm3, %ecx
1455; AVX512BW-NEXT:    mulb %cl
1456; AVX512BW-NEXT:    movzbl %al, %eax
1457; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm4, %xmm4
1458; AVX512BW-NEXT:    vpextrb $13, %xmm2, %eax
1459; AVX512BW-NEXT:    vpextrb $13, %xmm3, %ecx
1460; AVX512BW-NEXT:    mulb %cl
1461; AVX512BW-NEXT:    movzbl %al, %eax
1462; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm4, %xmm4
1463; AVX512BW-NEXT:    vpextrb $14, %xmm2, %eax
1464; AVX512BW-NEXT:    vpextrb $14, %xmm3, %ecx
1465; AVX512BW-NEXT:    mulb %cl
1466; AVX512BW-NEXT:    movzbl %al, %eax
1467; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm4, %xmm4
1468; AVX512BW-NEXT:    vpextrb $15, %xmm2, %eax
1469; AVX512BW-NEXT:    vpextrb $15, %xmm3, %ecx
1470; AVX512BW-NEXT:    mulb %cl
1471; AVX512BW-NEXT:    movzbl %al, %eax
1472; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm4, %xmm2
1473; AVX512BW-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
1474; AVX512BW-NEXT:    vpextrb $1, %xmm3, %eax
1475; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
1476; AVX512BW-NEXT:    vpextrb $1, %xmm4, %ecx
1477; AVX512BW-NEXT:    mulb %cl
1478; AVX512BW-NEXT:    movzbl %al, %ecx
1479; AVX512BW-NEXT:    vpextrb $0, %xmm3, %eax
1480; AVX512BW-NEXT:    vpextrb $0, %xmm4, %edx
1481; AVX512BW-NEXT:    mulb %dl
1482; AVX512BW-NEXT:    movzbl %al, %eax
1483; AVX512BW-NEXT:    vmovd %eax, %xmm5
1484; AVX512BW-NEXT:    vpinsrb $1, %ecx, %xmm5, %xmm5
1485; AVX512BW-NEXT:    vpextrb $2, %xmm3, %eax
1486; AVX512BW-NEXT:    vpextrb $2, %xmm4, %ecx
1487; AVX512BW-NEXT:    mulb %cl
1488; AVX512BW-NEXT:    movzbl %al, %eax
1489; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm5, %xmm5
1490; AVX512BW-NEXT:    vpextrb $3, %xmm3, %eax
1491; AVX512BW-NEXT:    vpextrb $3, %xmm4, %ecx
1492; AVX512BW-NEXT:    mulb %cl
1493; AVX512BW-NEXT:    movzbl %al, %eax
1494; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm5, %xmm5
1495; AVX512BW-NEXT:    vpextrb $4, %xmm3, %eax
1496; AVX512BW-NEXT:    vpextrb $4, %xmm4, %ecx
1497; AVX512BW-NEXT:    mulb %cl
1498; AVX512BW-NEXT:    movzbl %al, %eax
1499; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm5, %xmm5
1500; AVX512BW-NEXT:    vpextrb $5, %xmm3, %eax
1501; AVX512BW-NEXT:    vpextrb $5, %xmm4, %ecx
1502; AVX512BW-NEXT:    mulb %cl
1503; AVX512BW-NEXT:    movzbl %al, %eax
1504; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm5, %xmm5
1505; AVX512BW-NEXT:    vpextrb $6, %xmm3, %eax
1506; AVX512BW-NEXT:    vpextrb $6, %xmm4, %ecx
1507; AVX512BW-NEXT:    mulb %cl
1508; AVX512BW-NEXT:    movzbl %al, %eax
1509; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm5, %xmm5
1510; AVX512BW-NEXT:    vpextrb $7, %xmm3, %eax
1511; AVX512BW-NEXT:    vpextrb $7, %xmm4, %ecx
1512; AVX512BW-NEXT:    mulb %cl
1513; AVX512BW-NEXT:    movzbl %al, %eax
1514; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm5, %xmm5
1515; AVX512BW-NEXT:    vpextrb $8, %xmm3, %eax
1516; AVX512BW-NEXT:    vpextrb $8, %xmm4, %ecx
1517; AVX512BW-NEXT:    mulb %cl
1518; AVX512BW-NEXT:    movzbl %al, %eax
1519; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm5, %xmm5
1520; AVX512BW-NEXT:    vpextrb $9, %xmm3, %eax
1521; AVX512BW-NEXT:    vpextrb $9, %xmm4, %ecx
1522; AVX512BW-NEXT:    mulb %cl
1523; AVX512BW-NEXT:    movzbl %al, %eax
1524; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm5, %xmm5
1525; AVX512BW-NEXT:    vpextrb $10, %xmm3, %eax
1526; AVX512BW-NEXT:    vpextrb $10, %xmm4, %ecx
1527; AVX512BW-NEXT:    mulb %cl
1528; AVX512BW-NEXT:    movzbl %al, %eax
1529; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm5, %xmm5
1530; AVX512BW-NEXT:    vpextrb $11, %xmm3, %eax
1531; AVX512BW-NEXT:    vpextrb $11, %xmm4, %ecx
1532; AVX512BW-NEXT:    mulb %cl
1533; AVX512BW-NEXT:    movzbl %al, %eax
1534; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm5, %xmm5
1535; AVX512BW-NEXT:    vpextrb $12, %xmm3, %eax
1536; AVX512BW-NEXT:    vpextrb $12, %xmm4, %ecx
1537; AVX512BW-NEXT:    mulb %cl
1538; AVX512BW-NEXT:    movzbl %al, %eax
1539; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm5, %xmm5
1540; AVX512BW-NEXT:    vpextrb $13, %xmm3, %eax
1541; AVX512BW-NEXT:    vpextrb $13, %xmm4, %ecx
1542; AVX512BW-NEXT:    mulb %cl
1543; AVX512BW-NEXT:    movzbl %al, %eax
1544; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm5, %xmm5
1545; AVX512BW-NEXT:    vpextrb $14, %xmm3, %eax
1546; AVX512BW-NEXT:    vpextrb $14, %xmm4, %ecx
1547; AVX512BW-NEXT:    mulb %cl
1548; AVX512BW-NEXT:    movzbl %al, %eax
1549; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm5, %xmm5
1550; AVX512BW-NEXT:    vpextrb $15, %xmm3, %eax
1551; AVX512BW-NEXT:    vpextrb $15, %xmm4, %ecx
1552; AVX512BW-NEXT:    mulb %cl
1553; AVX512BW-NEXT:    movzbl %al, %eax
1554; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm5, %xmm3
1555; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
1556; AVX512BW-NEXT:    vextracti32x4 $1, %zmm0, %xmm3
1557; AVX512BW-NEXT:    vpextrb $1, %xmm3, %eax
1558; AVX512BW-NEXT:    vextracti32x4 $1, %zmm1, %xmm4
1559; AVX512BW-NEXT:    vpextrb $1, %xmm4, %ecx
1560; AVX512BW-NEXT:    mulb %cl
1561; AVX512BW-NEXT:    movzbl %al, %ecx
1562; AVX512BW-NEXT:    vpextrb $0, %xmm3, %eax
1563; AVX512BW-NEXT:    vpextrb $0, %xmm4, %edx
1564; AVX512BW-NEXT:    mulb %dl
1565; AVX512BW-NEXT:    movzbl %al, %eax
1566; AVX512BW-NEXT:    vmovd %eax, %xmm5
1567; AVX512BW-NEXT:    vpinsrb $1, %ecx, %xmm5, %xmm5
1568; AVX512BW-NEXT:    vpextrb $2, %xmm3, %eax
1569; AVX512BW-NEXT:    vpextrb $2, %xmm4, %ecx
1570; AVX512BW-NEXT:    mulb %cl
1571; AVX512BW-NEXT:    movzbl %al, %eax
1572; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm5, %xmm5
1573; AVX512BW-NEXT:    vpextrb $3, %xmm3, %eax
1574; AVX512BW-NEXT:    vpextrb $3, %xmm4, %ecx
1575; AVX512BW-NEXT:    mulb %cl
1576; AVX512BW-NEXT:    movzbl %al, %eax
1577; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm5, %xmm5
1578; AVX512BW-NEXT:    vpextrb $4, %xmm3, %eax
1579; AVX512BW-NEXT:    vpextrb $4, %xmm4, %ecx
1580; AVX512BW-NEXT:    mulb %cl
1581; AVX512BW-NEXT:    movzbl %al, %eax
1582; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm5, %xmm5
1583; AVX512BW-NEXT:    vpextrb $5, %xmm3, %eax
1584; AVX512BW-NEXT:    vpextrb $5, %xmm4, %ecx
1585; AVX512BW-NEXT:    mulb %cl
1586; AVX512BW-NEXT:    movzbl %al, %eax
1587; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm5, %xmm5
1588; AVX512BW-NEXT:    vpextrb $6, %xmm3, %eax
1589; AVX512BW-NEXT:    vpextrb $6, %xmm4, %ecx
1590; AVX512BW-NEXT:    mulb %cl
1591; AVX512BW-NEXT:    movzbl %al, %eax
1592; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm5, %xmm5
1593; AVX512BW-NEXT:    vpextrb $7, %xmm3, %eax
1594; AVX512BW-NEXT:    vpextrb $7, %xmm4, %ecx
1595; AVX512BW-NEXT:    mulb %cl
1596; AVX512BW-NEXT:    movzbl %al, %eax
1597; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm5, %xmm5
1598; AVX512BW-NEXT:    vpextrb $8, %xmm3, %eax
1599; AVX512BW-NEXT:    vpextrb $8, %xmm4, %ecx
1600; AVX512BW-NEXT:    mulb %cl
1601; AVX512BW-NEXT:    movzbl %al, %eax
1602; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm5, %xmm5
1603; AVX512BW-NEXT:    vpextrb $9, %xmm3, %eax
1604; AVX512BW-NEXT:    vpextrb $9, %xmm4, %ecx
1605; AVX512BW-NEXT:    mulb %cl
1606; AVX512BW-NEXT:    movzbl %al, %eax
1607; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm5, %xmm5
1608; AVX512BW-NEXT:    vpextrb $10, %xmm3, %eax
1609; AVX512BW-NEXT:    vpextrb $10, %xmm4, %ecx
1610; AVX512BW-NEXT:    mulb %cl
1611; AVX512BW-NEXT:    movzbl %al, %eax
1612; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm5, %xmm5
1613; AVX512BW-NEXT:    vpextrb $11, %xmm3, %eax
1614; AVX512BW-NEXT:    vpextrb $11, %xmm4, %ecx
1615; AVX512BW-NEXT:    mulb %cl
1616; AVX512BW-NEXT:    movzbl %al, %eax
1617; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm5, %xmm5
1618; AVX512BW-NEXT:    vpextrb $12, %xmm3, %eax
1619; AVX512BW-NEXT:    vpextrb $12, %xmm4, %ecx
1620; AVX512BW-NEXT:    mulb %cl
1621; AVX512BW-NEXT:    movzbl %al, %eax
1622; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm5, %xmm5
1623; AVX512BW-NEXT:    vpextrb $13, %xmm3, %eax
1624; AVX512BW-NEXT:    vpextrb $13, %xmm4, %ecx
1625; AVX512BW-NEXT:    mulb %cl
1626; AVX512BW-NEXT:    movzbl %al, %eax
1627; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm5, %xmm5
1628; AVX512BW-NEXT:    vpextrb $14, %xmm3, %eax
1629; AVX512BW-NEXT:    vpextrb $14, %xmm4, %ecx
1630; AVX512BW-NEXT:    mulb %cl
1631; AVX512BW-NEXT:    movzbl %al, %eax
1632; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm5, %xmm5
1633; AVX512BW-NEXT:    vpextrb $15, %xmm3, %eax
1634; AVX512BW-NEXT:    vpextrb $15, %xmm4, %ecx
1635; AVX512BW-NEXT:    mulb %cl
1636; AVX512BW-NEXT:    movzbl %al, %eax
1637; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm5, %xmm3
1638; AVX512BW-NEXT:    vpextrb $1, %xmm0, %eax
1639; AVX512BW-NEXT:    vpextrb $1, %xmm1, %ecx
1640; AVX512BW-NEXT:    mulb %cl
1641; AVX512BW-NEXT:    movzbl %al, %ecx
1642; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
1643; AVX512BW-NEXT:    vpextrb $0, %xmm1, %edx
1644; AVX512BW-NEXT:    mulb %dl
1645; AVX512BW-NEXT:    movzbl %al, %eax
1646; AVX512BW-NEXT:    vmovd %eax, %xmm4
1647; AVX512BW-NEXT:    vpinsrb $1, %ecx, %xmm4, %xmm4
1648; AVX512BW-NEXT:    vpextrb $2, %xmm0, %eax
1649; AVX512BW-NEXT:    vpextrb $2, %xmm1, %ecx
1650; AVX512BW-NEXT:    mulb %cl
1651; AVX512BW-NEXT:    movzbl %al, %eax
1652; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm4, %xmm4
1653; AVX512BW-NEXT:    vpextrb $3, %xmm0, %eax
1654; AVX512BW-NEXT:    vpextrb $3, %xmm1, %ecx
1655; AVX512BW-NEXT:    mulb %cl
1656; AVX512BW-NEXT:    movzbl %al, %eax
1657; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm4, %xmm4
1658; AVX512BW-NEXT:    vpextrb $4, %xmm0, %eax
1659; AVX512BW-NEXT:    vpextrb $4, %xmm1, %ecx
1660; AVX512BW-NEXT:    mulb %cl
1661; AVX512BW-NEXT:    movzbl %al, %eax
1662; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm4, %xmm4
1663; AVX512BW-NEXT:    vpextrb $5, %xmm0, %eax
1664; AVX512BW-NEXT:    vpextrb $5, %xmm1, %ecx
1665; AVX512BW-NEXT:    mulb %cl
1666; AVX512BW-NEXT:    movzbl %al, %eax
1667; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm4, %xmm4
1668; AVX512BW-NEXT:    vpextrb $6, %xmm0, %eax
1669; AVX512BW-NEXT:    vpextrb $6, %xmm1, %ecx
1670; AVX512BW-NEXT:    mulb %cl
1671; AVX512BW-NEXT:    movzbl %al, %eax
1672; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm4, %xmm4
1673; AVX512BW-NEXT:    vpextrb $7, %xmm0, %eax
1674; AVX512BW-NEXT:    vpextrb $7, %xmm1, %ecx
1675; AVX512BW-NEXT:    mulb %cl
1676; AVX512BW-NEXT:    movzbl %al, %eax
1677; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm4, %xmm4
1678; AVX512BW-NEXT:    vpextrb $8, %xmm0, %eax
1679; AVX512BW-NEXT:    vpextrb $8, %xmm1, %ecx
1680; AVX512BW-NEXT:    mulb %cl
1681; AVX512BW-NEXT:    movzbl %al, %eax
1682; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm4, %xmm4
1683; AVX512BW-NEXT:    vpextrb $9, %xmm0, %eax
1684; AVX512BW-NEXT:    vpextrb $9, %xmm1, %ecx
1685; AVX512BW-NEXT:    mulb %cl
1686; AVX512BW-NEXT:    movzbl %al, %eax
1687; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm4, %xmm4
1688; AVX512BW-NEXT:    vpextrb $10, %xmm0, %eax
1689; AVX512BW-NEXT:    vpextrb $10, %xmm1, %ecx
1690; AVX512BW-NEXT:    mulb %cl
1691; AVX512BW-NEXT:    movzbl %al, %eax
1692; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm4, %xmm4
1693; AVX512BW-NEXT:    vpextrb $11, %xmm0, %eax
1694; AVX512BW-NEXT:    vpextrb $11, %xmm1, %ecx
1695; AVX512BW-NEXT:    mulb %cl
1696; AVX512BW-NEXT:    movzbl %al, %eax
1697; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm4, %xmm4
1698; AVX512BW-NEXT:    vpextrb $12, %xmm0, %eax
1699; AVX512BW-NEXT:    vpextrb $12, %xmm1, %ecx
1700; AVX512BW-NEXT:    mulb %cl
1701; AVX512BW-NEXT:    movzbl %al, %eax
1702; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm4, %xmm4
1703; AVX512BW-NEXT:    vpextrb $13, %xmm0, %eax
1704; AVX512BW-NEXT:    vpextrb $13, %xmm1, %ecx
1705; AVX512BW-NEXT:    mulb %cl
1706; AVX512BW-NEXT:    movzbl %al, %eax
1707; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm4, %xmm4
1708; AVX512BW-NEXT:    vpextrb $14, %xmm0, %eax
1709; AVX512BW-NEXT:    vpextrb $14, %xmm1, %ecx
1710; AVX512BW-NEXT:    mulb %cl
1711; AVX512BW-NEXT:    movzbl %al, %eax
1712; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm4, %xmm4
1713; AVX512BW-NEXT:    vpextrb $15, %xmm0, %eax
1714; AVX512BW-NEXT:    vpextrb $15, %xmm1, %ecx
1715; AVX512BW-NEXT:    mulb %cl
1716; AVX512BW-NEXT:    movzbl %al, %eax
1717; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm4, %xmm0
1718; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
1719; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
1720; AVX512BW-NEXT:    retq
1721entry:
1722  %A = mul <64 x i8> %i, %j
1723  ret <64 x i8> %A
1724}
1725
1726