1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
9;
10; Just two 32-bit runs to make sure we do reasonable things there.
11; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X32-SSE,X32-SSE2
12; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=X32-SSE,X32-SSE41
13
14define <8 x i16> @sext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
15; SSE2-LABEL: sext_16i8_to_8i16:
16; SSE2:       # %bb.0: # %entry
17; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
18; SSE2-NEXT:    psraw $8, %xmm0
19; SSE2-NEXT:    retq
20;
21; SSSE3-LABEL: sext_16i8_to_8i16:
22; SSSE3:       # %bb.0: # %entry
23; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
24; SSSE3-NEXT:    psraw $8, %xmm0
25; SSSE3-NEXT:    retq
26;
27; SSE41-LABEL: sext_16i8_to_8i16:
28; SSE41:       # %bb.0: # %entry
29; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
30; SSE41-NEXT:    retq
31;
32; AVX-LABEL: sext_16i8_to_8i16:
33; AVX:       # %bb.0: # %entry
34; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
35; AVX-NEXT:    retq
36;
37; X32-SSE2-LABEL: sext_16i8_to_8i16:
38; X32-SSE2:       # %bb.0: # %entry
39; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
40; X32-SSE2-NEXT:    psraw $8, %xmm0
41; X32-SSE2-NEXT:    retl
42;
43; X32-SSE41-LABEL: sext_16i8_to_8i16:
44; X32-SSE41:       # %bb.0: # %entry
45; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
46; X32-SSE41-NEXT:    retl
47entry:
48  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
49  %C = sext <8 x i8> %B to <8 x i16>
50  ret <8 x i16> %C
51}
52
53define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ssp {
54; SSE2-LABEL: sext_16i8_to_16i16:
55; SSE2:       # %bb.0: # %entry
56; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
57; SSE2-NEXT:    psraw $8, %xmm2
58; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
59; SSE2-NEXT:    psraw $8, %xmm1
60; SSE2-NEXT:    movdqa %xmm2, %xmm0
61; SSE2-NEXT:    retq
62;
63; SSSE3-LABEL: sext_16i8_to_16i16:
64; SSSE3:       # %bb.0: # %entry
65; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
66; SSSE3-NEXT:    psraw $8, %xmm2
67; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
68; SSSE3-NEXT:    psraw $8, %xmm1
69; SSSE3-NEXT:    movdqa %xmm2, %xmm0
70; SSSE3-NEXT:    retq
71;
72; SSE41-LABEL: sext_16i8_to_16i16:
73; SSE41:       # %bb.0: # %entry
74; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
75; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
76; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
77; SSE41-NEXT:    movdqa %xmm2, %xmm0
78; SSE41-NEXT:    retq
79;
80; AVX1-LABEL: sext_16i8_to_16i16:
81; AVX1:       # %bb.0: # %entry
82; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
83; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
84; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
85; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
86; AVX1-NEXT:    retq
87;
88; AVX2-LABEL: sext_16i8_to_16i16:
89; AVX2:       # %bb.0: # %entry
90; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
91; AVX2-NEXT:    retq
92;
93; AVX512-LABEL: sext_16i8_to_16i16:
94; AVX512:       # %bb.0: # %entry
95; AVX512-NEXT:    vpmovsxbw %xmm0, %ymm0
96; AVX512-NEXT:    retq
97;
98; X32-SSE2-LABEL: sext_16i8_to_16i16:
99; X32-SSE2:       # %bb.0: # %entry
100; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
101; X32-SSE2-NEXT:    psraw $8, %xmm2
102; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
103; X32-SSE2-NEXT:    psraw $8, %xmm1
104; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
105; X32-SSE2-NEXT:    retl
106;
107; X32-SSE41-LABEL: sext_16i8_to_16i16:
108; X32-SSE41:       # %bb.0: # %entry
109; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
110; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
111; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
112; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
113; X32-SSE41-NEXT:    retl
114entry:
115  %B = sext <16 x i8> %A to <16 x i16>
116  ret <16 x i16> %B
117}
118
119define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ssp {
120; SSE2-LABEL: sext_32i8_to_32i16:
121; SSE2:       # %bb.0: # %entry
122; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
123; SSE2-NEXT:    psraw $8, %xmm4
124; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
125; SSE2-NEXT:    psraw $8, %xmm5
126; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
127; SSE2-NEXT:    psraw $8, %xmm2
128; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
129; SSE2-NEXT:    psraw $8, %xmm3
130; SSE2-NEXT:    movdqa %xmm4, %xmm0
131; SSE2-NEXT:    movdqa %xmm5, %xmm1
132; SSE2-NEXT:    retq
133;
134; SSSE3-LABEL: sext_32i8_to_32i16:
135; SSSE3:       # %bb.0: # %entry
136; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
137; SSSE3-NEXT:    psraw $8, %xmm4
138; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
139; SSSE3-NEXT:    psraw $8, %xmm5
140; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
141; SSSE3-NEXT:    psraw $8, %xmm2
142; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
143; SSSE3-NEXT:    psraw $8, %xmm3
144; SSSE3-NEXT:    movdqa %xmm4, %xmm0
145; SSSE3-NEXT:    movdqa %xmm5, %xmm1
146; SSSE3-NEXT:    retq
147;
148; SSE41-LABEL: sext_32i8_to_32i16:
149; SSE41:       # %bb.0: # %entry
150; SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
151; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
152; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
153; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
154; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
155; SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
156; SSE41-NEXT:    movdqa %xmm5, %xmm0
157; SSE41-NEXT:    movdqa %xmm4, %xmm1
158; SSE41-NEXT:    retq
159;
160; AVX1-LABEL: sext_32i8_to_32i16:
161; AVX1:       # %bb.0: # %entry
162; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
163; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
164; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
165; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
166; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
167; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
168; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
169; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
170; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
171; AVX1-NEXT:    vmovaps %ymm2, %ymm0
172; AVX1-NEXT:    retq
173;
174; AVX2-LABEL: sext_32i8_to_32i16:
175; AVX2:       # %bb.0: # %entry
176; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm2
177; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
178; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm1
179; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
180; AVX2-NEXT:    retq
181;
182; AVX512F-LABEL: sext_32i8_to_32i16:
183; AVX512F:       # %bb.0: # %entry
184; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
185; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
186; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
187; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
188; AVX512F-NEXT:    retq
189;
190; AVX512BW-LABEL: sext_32i8_to_32i16:
191; AVX512BW:       # %bb.0: # %entry
192; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
193; AVX512BW-NEXT:    retq
194;
195; X32-SSE2-LABEL: sext_32i8_to_32i16:
196; X32-SSE2:       # %bb.0: # %entry
197; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
198; X32-SSE2-NEXT:    psraw $8, %xmm4
199; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
200; X32-SSE2-NEXT:    psraw $8, %xmm5
201; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
202; X32-SSE2-NEXT:    psraw $8, %xmm2
203; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
204; X32-SSE2-NEXT:    psraw $8, %xmm3
205; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
206; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
207; X32-SSE2-NEXT:    retl
208;
209; X32-SSE41-LABEL: sext_32i8_to_32i16:
210; X32-SSE41:       # %bb.0: # %entry
211; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
212; X32-SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
213; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
214; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
215; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
216; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
217; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
218; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
219; X32-SSE41-NEXT:    retl
220entry:
221  %B = sext <32 x i8> %A to <32 x i16>
222  ret <32 x i16> %B
223}
224
225define <4 x i32> @sext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
226; SSE2-LABEL: sext_16i8_to_4i32:
227; SSE2:       # %bb.0: # %entry
228; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
229; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
230; SSE2-NEXT:    psrad $24, %xmm0
231; SSE2-NEXT:    retq
232;
233; SSSE3-LABEL: sext_16i8_to_4i32:
234; SSSE3:       # %bb.0: # %entry
235; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
236; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
237; SSSE3-NEXT:    psrad $24, %xmm0
238; SSSE3-NEXT:    retq
239;
240; SSE41-LABEL: sext_16i8_to_4i32:
241; SSE41:       # %bb.0: # %entry
242; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
243; SSE41-NEXT:    retq
244;
245; AVX-LABEL: sext_16i8_to_4i32:
246; AVX:       # %bb.0: # %entry
247; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
248; AVX-NEXT:    retq
249;
250; X32-SSE2-LABEL: sext_16i8_to_4i32:
251; X32-SSE2:       # %bb.0: # %entry
252; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
253; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
254; X32-SSE2-NEXT:    psrad $24, %xmm0
255; X32-SSE2-NEXT:    retl
256;
257; X32-SSE41-LABEL: sext_16i8_to_4i32:
258; X32-SSE41:       # %bb.0: # %entry
259; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
260; X32-SSE41-NEXT:    retl
261entry:
262  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
263  %C = sext <4 x i8> %B to <4 x i32>
264  ret <4 x i32> %C
265}
266
267define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
268; SSE2-LABEL: sext_16i8_to_8i32:
269; SSE2:       # %bb.0: # %entry
270; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
271; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
272; SSE2-NEXT:    psrad $24, %xmm0
273; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
274; SSE2-NEXT:    psrad $24, %xmm1
275; SSE2-NEXT:    retq
276;
277; SSSE3-LABEL: sext_16i8_to_8i32:
278; SSSE3:       # %bb.0: # %entry
279; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
280; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
281; SSSE3-NEXT:    psrad $24, %xmm0
282; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
283; SSSE3-NEXT:    psrad $24, %xmm1
284; SSSE3-NEXT:    retq
285;
286; SSE41-LABEL: sext_16i8_to_8i32:
287; SSE41:       # %bb.0: # %entry
288; SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
289; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
290; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
291; SSE41-NEXT:    movdqa %xmm2, %xmm0
292; SSE41-NEXT:    retq
293;
294; AVX1-LABEL: sext_16i8_to_8i32:
295; AVX1:       # %bb.0: # %entry
296; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
297; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
298; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
299; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
300; AVX1-NEXT:    retq
301;
302; AVX2-LABEL: sext_16i8_to_8i32:
303; AVX2:       # %bb.0: # %entry
304; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
305; AVX2-NEXT:    retq
306;
307; AVX512-LABEL: sext_16i8_to_8i32:
308; AVX512:       # %bb.0: # %entry
309; AVX512-NEXT:    vpmovsxbd %xmm0, %ymm0
310; AVX512-NEXT:    retq
311;
312; X32-SSE2-LABEL: sext_16i8_to_8i32:
313; X32-SSE2:       # %bb.0: # %entry
314; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
315; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
316; X32-SSE2-NEXT:    psrad $24, %xmm0
317; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
318; X32-SSE2-NEXT:    psrad $24, %xmm1
319; X32-SSE2-NEXT:    retl
320;
321; X32-SSE41-LABEL: sext_16i8_to_8i32:
322; X32-SSE41:       # %bb.0: # %entry
323; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
324; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
325; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
326; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
327; X32-SSE41-NEXT:    retl
328entry:
329  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
330  %C = sext <8 x i8> %B to <8 x i32>
331  ret <8 x i32> %C
332}
333
334define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ssp {
335; SSE2-LABEL: sext_16i8_to_16i32:
336; SSE2:       # %bb.0: # %entry
337; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
338; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
339; SSE2-NEXT:    psrad $24, %xmm4
340; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
341; SSE2-NEXT:    psrad $24, %xmm1
342; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
343; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
344; SSE2-NEXT:    psrad $24, %xmm2
345; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
346; SSE2-NEXT:    psrad $24, %xmm3
347; SSE2-NEXT:    movdqa %xmm4, %xmm0
348; SSE2-NEXT:    retq
349;
350; SSSE3-LABEL: sext_16i8_to_16i32:
351; SSSE3:       # %bb.0: # %entry
352; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
353; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
354; SSSE3-NEXT:    psrad $24, %xmm4
355; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
356; SSSE3-NEXT:    psrad $24, %xmm1
357; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
358; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
359; SSSE3-NEXT:    psrad $24, %xmm2
360; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
361; SSSE3-NEXT:    psrad $24, %xmm3
362; SSSE3-NEXT:    movdqa %xmm4, %xmm0
363; SSSE3-NEXT:    retq
364;
365; SSE41-LABEL: sext_16i8_to_16i32:
366; SSE41:       # %bb.0: # %entry
367; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
368; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
369; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
370; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
371; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
372; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
373; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
374; SSE41-NEXT:    movdqa %xmm4, %xmm0
375; SSE41-NEXT:    retq
376;
377; AVX1-LABEL: sext_16i8_to_16i32:
378; AVX1:       # %bb.0: # %entry
379; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
380; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
381; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
382; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
383; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
384; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
385; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
386; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
387; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
388; AVX1-NEXT:    vmovaps %ymm2, %ymm0
389; AVX1-NEXT:    retq
390;
391; AVX2-LABEL: sext_16i8_to_16i32:
392; AVX2:       # %bb.0: # %entry
393; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm2
394; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
395; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm1
396; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
397; AVX2-NEXT:    retq
398;
399; AVX512-LABEL: sext_16i8_to_16i32:
400; AVX512:       # %bb.0: # %entry
401; AVX512-NEXT:    vpmovsxbd %xmm0, %zmm0
402; AVX512-NEXT:    retq
403;
404; X32-SSE2-LABEL: sext_16i8_to_16i32:
405; X32-SSE2:       # %bb.0: # %entry
406; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
407; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
408; X32-SSE2-NEXT:    psrad $24, %xmm4
409; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
410; X32-SSE2-NEXT:    psrad $24, %xmm1
411; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
412; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
413; X32-SSE2-NEXT:    psrad $24, %xmm2
414; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
415; X32-SSE2-NEXT:    psrad $24, %xmm3
416; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
417; X32-SSE2-NEXT:    retl
418;
419; X32-SSE41-LABEL: sext_16i8_to_16i32:
420; X32-SSE41:       # %bb.0: # %entry
421; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
422; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
423; X32-SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
424; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
425; X32-SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
426; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
427; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
428; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
429; X32-SSE41-NEXT:    retl
430entry:
431  %B = sext <16 x i8> %A to <16 x i32>
432  ret <16 x i32> %B
433}
434
435define <2 x i64> @sext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
436; SSE2-LABEL: sext_16i8_to_2i64:
437; SSE2:       # %bb.0: # %entry
438; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
439; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
440; SSE2-NEXT:    pxor %xmm1, %xmm1
441; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
442; SSE2-NEXT:    psrad $24, %xmm0
443; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
444; SSE2-NEXT:    retq
445;
446; SSSE3-LABEL: sext_16i8_to_2i64:
447; SSSE3:       # %bb.0: # %entry
448; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
449; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
450; SSSE3-NEXT:    pxor %xmm1, %xmm1
451; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
452; SSSE3-NEXT:    psrad $24, %xmm0
453; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
454; SSSE3-NEXT:    retq
455;
456; SSE41-LABEL: sext_16i8_to_2i64:
457; SSE41:       # %bb.0: # %entry
458; SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
459; SSE41-NEXT:    retq
460;
461; AVX-LABEL: sext_16i8_to_2i64:
462; AVX:       # %bb.0: # %entry
463; AVX-NEXT:    vpmovsxbq %xmm0, %xmm0
464; AVX-NEXT:    retq
465;
466; X32-SSE2-LABEL: sext_16i8_to_2i64:
467; X32-SSE2:       # %bb.0: # %entry
468; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
469; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
470; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
471; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
472; X32-SSE2-NEXT:    psrad $24, %xmm0
473; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
474; X32-SSE2-NEXT:    retl
475;
476; X32-SSE41-LABEL: sext_16i8_to_2i64:
477; X32-SSE41:       # %bb.0: # %entry
478; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
479; X32-SSE41-NEXT:    retl
480entry:
481  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
482  %C = sext <2 x i8> %B to <2 x i64>
483  ret <2 x i64> %C
484}
485
486define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
487; SSE2-LABEL: sext_16i8_to_4i64:
488; SSE2:       # %bb.0: # %entry
489; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
490; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
491; SSE2-NEXT:    psrad $24, %xmm1
492; SSE2-NEXT:    pxor %xmm2, %xmm2
493; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
494; SSE2-NEXT:    movdqa %xmm1, %xmm0
495; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
496; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
497; SSE2-NEXT:    retq
498;
499; SSSE3-LABEL: sext_16i8_to_4i64:
500; SSSE3:       # %bb.0: # %entry
501; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
502; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
503; SSSE3-NEXT:    psrad $24, %xmm1
504; SSSE3-NEXT:    pxor %xmm2, %xmm2
505; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
506; SSSE3-NEXT:    movdqa %xmm1, %xmm0
507; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
508; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
509; SSSE3-NEXT:    retq
510;
511; SSE41-LABEL: sext_16i8_to_4i64:
512; SSE41:       # %bb.0: # %entry
513; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
514; SSE41-NEXT:    psrld $16, %xmm0
515; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
516; SSE41-NEXT:    movdqa %xmm2, %xmm0
517; SSE41-NEXT:    retq
518;
519; AVX1-LABEL: sext_16i8_to_4i64:
520; AVX1:       # %bb.0: # %entry
521; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
522; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
523; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
524; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
525; AVX1-NEXT:    retq
526;
527; AVX2-LABEL: sext_16i8_to_4i64:
528; AVX2:       # %bb.0: # %entry
529; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm0
530; AVX2-NEXT:    retq
531;
532; AVX512-LABEL: sext_16i8_to_4i64:
533; AVX512:       # %bb.0: # %entry
534; AVX512-NEXT:    vpmovsxbq %xmm0, %ymm0
535; AVX512-NEXT:    retq
536;
537; X32-SSE2-LABEL: sext_16i8_to_4i64:
538; X32-SSE2:       # %bb.0: # %entry
539; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
540; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
541; X32-SSE2-NEXT:    psrad $24, %xmm1
542; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
543; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
544; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
545; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
546; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
547; X32-SSE2-NEXT:    retl
548;
549; X32-SSE41-LABEL: sext_16i8_to_4i64:
550; X32-SSE41:       # %bb.0: # %entry
551; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
552; X32-SSE41-NEXT:    psrld $16, %xmm0
553; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
554; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
555; X32-SSE41-NEXT:    retl
556entry:
557  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
558  %C = sext <4 x i8> %B to <4 x i64>
559  ret <4 x i64> %C
560}
561
562define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
563; SSE2-LABEL: sext_16i8_to_8i64:
564; SSE2:       # %bb.0: # %entry
565; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
566; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
567; SSE2-NEXT:    psrad $24, %xmm1
568; SSE2-NEXT:    pxor %xmm4, %xmm4
569; SSE2-NEXT:    pxor %xmm3, %xmm3
570; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
571; SSE2-NEXT:    movdqa %xmm1, %xmm0
572; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
573; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
574; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
575; SSE2-NEXT:    psrad $24, %xmm3
576; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
577; SSE2-NEXT:    movdqa %xmm3, %xmm2
578; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
579; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
580; SSE2-NEXT:    retq
581;
582; SSSE3-LABEL: sext_16i8_to_8i64:
583; SSSE3:       # %bb.0: # %entry
584; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
585; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
586; SSSE3-NEXT:    psrad $24, %xmm1
587; SSSE3-NEXT:    pxor %xmm4, %xmm4
588; SSSE3-NEXT:    pxor %xmm3, %xmm3
589; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
590; SSSE3-NEXT:    movdqa %xmm1, %xmm0
591; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
592; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
593; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
594; SSSE3-NEXT:    psrad $24, %xmm3
595; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
596; SSSE3-NEXT:    movdqa %xmm3, %xmm2
597; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
598; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
599; SSSE3-NEXT:    retq
600;
601; SSE41-LABEL: sext_16i8_to_8i64:
602; SSE41:       # %bb.0: # %entry
603; SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
604; SSE41-NEXT:    movdqa %xmm0, %xmm1
605; SSE41-NEXT:    psrld $16, %xmm1
606; SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
607; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
608; SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
609; SSE41-NEXT:    psrlq $48, %xmm0
610; SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
611; SSE41-NEXT:    movdqa %xmm4, %xmm0
612; SSE41-NEXT:    retq
613;
614; AVX1-LABEL: sext_16i8_to_8i64:
615; AVX1:       # %bb.0: # %entry
616; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
617; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
618; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
619; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
620; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
621; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
622; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
623; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
624; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
625; AVX1-NEXT:    vmovaps %ymm2, %ymm0
626; AVX1-NEXT:    retq
627;
628; AVX2-LABEL: sext_16i8_to_8i64:
629; AVX2:       # %bb.0: # %entry
630; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm2
631; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
632; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm1
633; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
634; AVX2-NEXT:    retq
635;
636; AVX512-LABEL: sext_16i8_to_8i64:
637; AVX512:       # %bb.0: # %entry
638; AVX512-NEXT:    vpmovsxbq %xmm0, %zmm0
639; AVX512-NEXT:    retq
640;
641; X32-SSE2-LABEL: sext_16i8_to_8i64:
642; X32-SSE2:       # %bb.0: # %entry
643; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
644; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
645; X32-SSE2-NEXT:    psrad $24, %xmm1
646; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
647; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
648; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
649; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
650; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
651; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
652; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
653; X32-SSE2-NEXT:    psrad $24, %xmm3
654; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
655; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
656; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
657; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
658; X32-SSE2-NEXT:    retl
659;
660; X32-SSE41-LABEL: sext_16i8_to_8i64:
661; X32-SSE41:       # %bb.0: # %entry
662; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
663; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
664; X32-SSE41-NEXT:    psrld $16, %xmm1
665; X32-SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
666; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
667; X32-SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
668; X32-SSE41-NEXT:    psrlq $48, %xmm0
669; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
670; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
671; X32-SSE41-NEXT:    retl
672entry:
673  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
674  %C = sext <8 x i8> %B to <8 x i64>
675  ret <8 x i64> %C
676}
677
678define <4 x i32> @sext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
679; SSE2-LABEL: sext_8i16_to_4i32:
680; SSE2:       # %bb.0: # %entry
681; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
682; SSE2-NEXT:    psrad $16, %xmm0
683; SSE2-NEXT:    retq
684;
685; SSSE3-LABEL: sext_8i16_to_4i32:
686; SSSE3:       # %bb.0: # %entry
687; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
688; SSSE3-NEXT:    psrad $16, %xmm0
689; SSSE3-NEXT:    retq
690;
691; SSE41-LABEL: sext_8i16_to_4i32:
692; SSE41:       # %bb.0: # %entry
693; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
694; SSE41-NEXT:    retq
695;
696; AVX-LABEL: sext_8i16_to_4i32:
697; AVX:       # %bb.0: # %entry
698; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
699; AVX-NEXT:    retq
700;
701; X32-SSE2-LABEL: sext_8i16_to_4i32:
702; X32-SSE2:       # %bb.0: # %entry
703; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
704; X32-SSE2-NEXT:    psrad $16, %xmm0
705; X32-SSE2-NEXT:    retl
706;
707; X32-SSE41-LABEL: sext_8i16_to_4i32:
708; X32-SSE41:       # %bb.0: # %entry
709; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
710; X32-SSE41-NEXT:    retl
711entry:
712  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
713  %C = sext <4 x i16> %B to <4 x i32>
714  ret <4 x i32> %C
715}
716
717define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
718; SSE2-LABEL: sext_8i16_to_8i32:
719; SSE2:       # %bb.0: # %entry
720; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
721; SSE2-NEXT:    psrad $16, %xmm2
722; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
723; SSE2-NEXT:    psrad $16, %xmm1
724; SSE2-NEXT:    movdqa %xmm2, %xmm0
725; SSE2-NEXT:    retq
726;
727; SSSE3-LABEL: sext_8i16_to_8i32:
728; SSSE3:       # %bb.0: # %entry
729; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
730; SSSE3-NEXT:    psrad $16, %xmm2
731; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
732; SSSE3-NEXT:    psrad $16, %xmm1
733; SSSE3-NEXT:    movdqa %xmm2, %xmm0
734; SSSE3-NEXT:    retq
735;
736; SSE41-LABEL: sext_8i16_to_8i32:
737; SSE41:       # %bb.0: # %entry
738; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
739; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
740; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
741; SSE41-NEXT:    movdqa %xmm2, %xmm0
742; SSE41-NEXT:    retq
743;
744; AVX1-LABEL: sext_8i16_to_8i32:
745; AVX1:       # %bb.0: # %entry
746; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
747; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
748; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
749; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
750; AVX1-NEXT:    retq
751;
752; AVX2-LABEL: sext_8i16_to_8i32:
753; AVX2:       # %bb.0: # %entry
754; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
755; AVX2-NEXT:    retq
756;
757; AVX512-LABEL: sext_8i16_to_8i32:
758; AVX512:       # %bb.0: # %entry
759; AVX512-NEXT:    vpmovsxwd %xmm0, %ymm0
760; AVX512-NEXT:    retq
761;
762; X32-SSE2-LABEL: sext_8i16_to_8i32:
763; X32-SSE2:       # %bb.0: # %entry
764; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
765; X32-SSE2-NEXT:    psrad $16, %xmm2
766; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
767; X32-SSE2-NEXT:    psrad $16, %xmm1
768; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
769; X32-SSE2-NEXT:    retl
770;
771; X32-SSE41-LABEL: sext_8i16_to_8i32:
772; X32-SSE41:       # %bb.0: # %entry
773; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
774; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
775; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
776; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
777; X32-SSE41-NEXT:    retl
778entry:
779  %B = sext <8 x i16> %A to <8 x i32>
780  ret <8 x i32> %B
781}
782
783define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone ssp {
784; SSE2-LABEL: sext_16i16_to_16i32:
785; SSE2:       # %bb.0: # %entry
786; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
787; SSE2-NEXT:    psrad $16, %xmm4
788; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
789; SSE2-NEXT:    psrad $16, %xmm5
790; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
791; SSE2-NEXT:    psrad $16, %xmm2
792; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
793; SSE2-NEXT:    psrad $16, %xmm3
794; SSE2-NEXT:    movdqa %xmm4, %xmm0
795; SSE2-NEXT:    movdqa %xmm5, %xmm1
796; SSE2-NEXT:    retq
797;
798; SSSE3-LABEL: sext_16i16_to_16i32:
799; SSSE3:       # %bb.0: # %entry
800; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
801; SSSE3-NEXT:    psrad $16, %xmm4
802; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
803; SSSE3-NEXT:    psrad $16, %xmm5
804; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
805; SSSE3-NEXT:    psrad $16, %xmm2
806; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
807; SSSE3-NEXT:    psrad $16, %xmm3
808; SSSE3-NEXT:    movdqa %xmm4, %xmm0
809; SSSE3-NEXT:    movdqa %xmm5, %xmm1
810; SSSE3-NEXT:    retq
811;
812; SSE41-LABEL: sext_16i16_to_16i32:
813; SSE41:       # %bb.0: # %entry
814; SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
815; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
816; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
817; SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
818; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
819; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
820; SSE41-NEXT:    movdqa %xmm5, %xmm0
821; SSE41-NEXT:    movdqa %xmm4, %xmm1
822; SSE41-NEXT:    retq
823;
824; AVX1-LABEL: sext_16i16_to_16i32:
825; AVX1:       # %bb.0: # %entry
826; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
827; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
828; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
829; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
830; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
831; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
832; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
833; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
834; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
835; AVX1-NEXT:    vmovaps %ymm2, %ymm0
836; AVX1-NEXT:    retq
837;
838; AVX2-LABEL: sext_16i16_to_16i32:
839; AVX2:       # %bb.0: # %entry
840; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm2
841; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
842; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm1
843; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
844; AVX2-NEXT:    retq
845;
846; AVX512-LABEL: sext_16i16_to_16i32:
847; AVX512:       # %bb.0: # %entry
848; AVX512-NEXT:    vpmovsxwd %ymm0, %zmm0
849; AVX512-NEXT:    retq
850;
851; X32-SSE2-LABEL: sext_16i16_to_16i32:
852; X32-SSE2:       # %bb.0: # %entry
853; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
854; X32-SSE2-NEXT:    psrad $16, %xmm4
855; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
856; X32-SSE2-NEXT:    psrad $16, %xmm5
857; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
858; X32-SSE2-NEXT:    psrad $16, %xmm2
859; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
860; X32-SSE2-NEXT:    psrad $16, %xmm3
861; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
862; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
863; X32-SSE2-NEXT:    retl
864;
865; X32-SSE41-LABEL: sext_16i16_to_16i32:
866; X32-SSE41:       # %bb.0: # %entry
867; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
868; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
869; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
870; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
871; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
872; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
873; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
874; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
875; X32-SSE41-NEXT:    retl
876entry:
877  %B = sext <16 x i16> %A to <16 x i32>
878  ret <16 x i32> %B
879}
880
881define <2 x i64> @sext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
882; SSE2-LABEL: sext_8i16_to_2i64:
883; SSE2:       # %bb.0: # %entry
884; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
885; SSE2-NEXT:    pxor %xmm1, %xmm1
886; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
887; SSE2-NEXT:    psrad $16, %xmm0
888; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
889; SSE2-NEXT:    retq
890;
891; SSSE3-LABEL: sext_8i16_to_2i64:
892; SSSE3:       # %bb.0: # %entry
893; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
894; SSSE3-NEXT:    pxor %xmm1, %xmm1
895; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
896; SSSE3-NEXT:    psrad $16, %xmm0
897; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
898; SSSE3-NEXT:    retq
899;
900; SSE41-LABEL: sext_8i16_to_2i64:
901; SSE41:       # %bb.0: # %entry
902; SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
903; SSE41-NEXT:    retq
904;
905; AVX-LABEL: sext_8i16_to_2i64:
906; AVX:       # %bb.0: # %entry
907; AVX-NEXT:    vpmovsxwq %xmm0, %xmm0
908; AVX-NEXT:    retq
909;
910; X32-SSE2-LABEL: sext_8i16_to_2i64:
911; X32-SSE2:       # %bb.0: # %entry
912; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
913; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
914; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
915; X32-SSE2-NEXT:    psrad $16, %xmm0
916; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
917; X32-SSE2-NEXT:    retl
918;
919; X32-SSE41-LABEL: sext_8i16_to_2i64:
920; X32-SSE41:       # %bb.0: # %entry
921; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
922; X32-SSE41-NEXT:    retl
923entry:
924  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
925  %C = sext <2 x i16> %B to <2 x i64>
926  ret <2 x i64> %C
927}
928
929define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
930; SSE2-LABEL: sext_8i16_to_4i64:
931; SSE2:       # %bb.0: # %entry
932; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
933; SSE2-NEXT:    psrad $16, %xmm1
934; SSE2-NEXT:    pxor %xmm2, %xmm2
935; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
936; SSE2-NEXT:    movdqa %xmm1, %xmm0
937; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
938; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
939; SSE2-NEXT:    retq
940;
941; SSSE3-LABEL: sext_8i16_to_4i64:
942; SSSE3:       # %bb.0: # %entry
943; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
944; SSSE3-NEXT:    psrad $16, %xmm1
945; SSSE3-NEXT:    pxor %xmm2, %xmm2
946; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
947; SSSE3-NEXT:    movdqa %xmm1, %xmm0
948; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
949; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
950; SSSE3-NEXT:    retq
951;
952; SSE41-LABEL: sext_8i16_to_4i64:
953; SSE41:       # %bb.0: # %entry
954; SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
955; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
956; SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
957; SSE41-NEXT:    movdqa %xmm2, %xmm0
958; SSE41-NEXT:    retq
959;
960; AVX1-LABEL: sext_8i16_to_4i64:
961; AVX1:       # %bb.0: # %entry
962; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
963; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
964; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
965; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
966; AVX1-NEXT:    retq
967;
968; AVX2-LABEL: sext_8i16_to_4i64:
969; AVX2:       # %bb.0: # %entry
970; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
971; AVX2-NEXT:    retq
972;
973; AVX512-LABEL: sext_8i16_to_4i64:
974; AVX512:       # %bb.0: # %entry
975; AVX512-NEXT:    vpmovsxwq %xmm0, %ymm0
976; AVX512-NEXT:    retq
977;
978; X32-SSE2-LABEL: sext_8i16_to_4i64:
979; X32-SSE2:       # %bb.0: # %entry
980; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
981; X32-SSE2-NEXT:    psrad $16, %xmm1
982; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
983; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
984; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
985; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
986; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
987; X32-SSE2-NEXT:    retl
988;
989; X32-SSE41-LABEL: sext_8i16_to_4i64:
990; X32-SSE41:       # %bb.0: # %entry
991; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
992; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
993; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
994; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
995; X32-SSE41-NEXT:    retl
996entry:
997  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
998  %C = sext <4 x i16> %B to <4 x i64>
999  ret <4 x i64> %C
1000}
1001
1002define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp {
1003; SSE2-LABEL: sext_8i16_to_8i64:
1004; SSE2:       # %bb.0: # %entry
1005; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1006; SSE2-NEXT:    psrad $16, %xmm1
1007; SSE2-NEXT:    pxor %xmm5, %xmm5
1008; SSE2-NEXT:    pxor %xmm2, %xmm2
1009; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1010; SSE2-NEXT:    movdqa %xmm1, %xmm4
1011; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1012; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1013; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1014; SSE2-NEXT:    psrad $16, %xmm3
1015; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1016; SSE2-NEXT:    movdqa %xmm3, %xmm2
1017; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1018; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1019; SSE2-NEXT:    movdqa %xmm4, %xmm0
1020; SSE2-NEXT:    retq
1021;
1022; SSSE3-LABEL: sext_8i16_to_8i64:
1023; SSSE3:       # %bb.0: # %entry
1024; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1025; SSSE3-NEXT:    psrad $16, %xmm1
1026; SSSE3-NEXT:    pxor %xmm5, %xmm5
1027; SSSE3-NEXT:    pxor %xmm2, %xmm2
1028; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1029; SSSE3-NEXT:    movdqa %xmm1, %xmm4
1030; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1031; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1032; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1033; SSSE3-NEXT:    psrad $16, %xmm3
1034; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
1035; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1036; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1037; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1038; SSSE3-NEXT:    movdqa %xmm4, %xmm0
1039; SSSE3-NEXT:    retq
1040;
1041; SSE41-LABEL: sext_8i16_to_8i64:
1042; SSE41:       # %bb.0: # %entry
1043; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1044; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1045; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1046; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1047; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1048; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1049; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1050; SSE41-NEXT:    movdqa %xmm4, %xmm0
1051; SSE41-NEXT:    retq
1052;
1053; AVX1-LABEL: sext_8i16_to_8i64:
1054; AVX1:       # %bb.0: # %entry
1055; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
1056; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
1057; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
1058; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1059; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1060; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
1061; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
1062; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
1063; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1064; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1065; AVX1-NEXT:    retq
1066;
1067; AVX2-LABEL: sext_8i16_to_8i64:
1068; AVX2:       # %bb.0: # %entry
1069; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm2
1070; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1071; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm1
1072; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1073; AVX2-NEXT:    retq
1074;
1075; AVX512-LABEL: sext_8i16_to_8i64:
1076; AVX512:       # %bb.0: # %entry
1077; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
1078; AVX512-NEXT:    retq
1079;
1080; X32-SSE2-LABEL: sext_8i16_to_8i64:
1081; X32-SSE2:       # %bb.0: # %entry
1082; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1083; X32-SSE2-NEXT:    psrad $16, %xmm1
1084; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1085; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1086; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1087; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
1088; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1089; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1090; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1091; X32-SSE2-NEXT:    psrad $16, %xmm3
1092; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1093; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
1094; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1095; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1096; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
1097; X32-SSE2-NEXT:    retl
1098;
1099; X32-SSE41-LABEL: sext_8i16_to_8i64:
1100; X32-SSE41:       # %bb.0: # %entry
1101; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1102; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1103; X32-SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1104; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1105; X32-SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1106; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1107; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1108; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
1109; X32-SSE41-NEXT:    retl
1110entry:
1111  %B = sext <8 x i16> %A to <8 x i64>
1112  ret <8 x i64> %B
1113}
1114
1115define <2 x i64> @sext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1116; SSE2-LABEL: sext_4i32_to_2i64:
1117; SSE2:       # %bb.0: # %entry
1118; SSE2-NEXT:    pxor %xmm1, %xmm1
1119; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1120; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1121; SSE2-NEXT:    retq
1122;
1123; SSSE3-LABEL: sext_4i32_to_2i64:
1124; SSSE3:       # %bb.0: # %entry
1125; SSSE3-NEXT:    pxor %xmm1, %xmm1
1126; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1127; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1128; SSSE3-NEXT:    retq
1129;
1130; SSE41-LABEL: sext_4i32_to_2i64:
1131; SSE41:       # %bb.0: # %entry
1132; SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1133; SSE41-NEXT:    retq
1134;
1135; AVX-LABEL: sext_4i32_to_2i64:
1136; AVX:       # %bb.0: # %entry
1137; AVX-NEXT:    vpmovsxdq %xmm0, %xmm0
1138; AVX-NEXT:    retq
1139;
1140; X32-SSE2-LABEL: sext_4i32_to_2i64:
1141; X32-SSE2:       # %bb.0: # %entry
1142; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1143; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1144; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1145; X32-SSE2-NEXT:    retl
1146;
1147; X32-SSE41-LABEL: sext_4i32_to_2i64:
1148; X32-SSE41:       # %bb.0: # %entry
1149; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1150; X32-SSE41-NEXT:    retl
1151entry:
1152  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
1153  %C = sext <2 x i32> %B to <2 x i64>
1154  ret <2 x i64> %C
1155}
1156
1157define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1158; SSE2-LABEL: sext_4i32_to_4i64:
1159; SSE2:       # %bb.0: # %entry
1160; SSE2-NEXT:    pxor %xmm2, %xmm2
1161; SSE2-NEXT:    pxor %xmm3, %xmm3
1162; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1163; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1164; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1165; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1166; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1167; SSE2-NEXT:    retq
1168;
1169; SSSE3-LABEL: sext_4i32_to_4i64:
1170; SSSE3:       # %bb.0: # %entry
1171; SSSE3-NEXT:    pxor %xmm2, %xmm2
1172; SSSE3-NEXT:    pxor %xmm3, %xmm3
1173; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1174; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1175; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1176; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1177; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1178; SSSE3-NEXT:    retq
1179;
1180; SSE41-LABEL: sext_4i32_to_4i64:
1181; SSE41:       # %bb.0: # %entry
1182; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1183; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1184; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1185; SSE41-NEXT:    movdqa %xmm2, %xmm0
1186; SSE41-NEXT:    retq
1187;
1188; AVX1-LABEL: sext_4i32_to_4i64:
1189; AVX1:       # %bb.0: # %entry
1190; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1191; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1192; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1193; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1194; AVX1-NEXT:    retq
1195;
1196; AVX2-LABEL: sext_4i32_to_4i64:
1197; AVX2:       # %bb.0: # %entry
1198; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
1199; AVX2-NEXT:    retq
1200;
1201; AVX512-LABEL: sext_4i32_to_4i64:
1202; AVX512:       # %bb.0: # %entry
1203; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
1204; AVX512-NEXT:    retq
1205;
1206; X32-SSE2-LABEL: sext_4i32_to_4i64:
1207; X32-SSE2:       # %bb.0: # %entry
1208; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1209; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1210; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1211; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1212; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1213; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1214; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1215; X32-SSE2-NEXT:    retl
1216;
1217; X32-SSE41-LABEL: sext_4i32_to_4i64:
1218; X32-SSE41:       # %bb.0: # %entry
1219; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1220; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1221; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1222; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
1223; X32-SSE41-NEXT:    retl
1224entry:
1225  %B = sext <4 x i32> %A to <4 x i64>
1226  ret <4 x i64> %B
1227}
1228
1229define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp {
1230; SSE2-LABEL: sext_8i32_to_8i64:
1231; SSE2:       # %bb.0: # %entry
1232; SSE2-NEXT:    movdqa %xmm1, %xmm2
1233; SSE2-NEXT:    pxor %xmm4, %xmm4
1234; SSE2-NEXT:    pxor %xmm3, %xmm3
1235; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1236; SSE2-NEXT:    pxor %xmm5, %xmm5
1237; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1238; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1239; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1240; SSE2-NEXT:    pxor %xmm3, %xmm3
1241; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1242; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1243; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1244; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1245; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1246; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1247; SSE2-NEXT:    retq
1248;
1249; SSSE3-LABEL: sext_8i32_to_8i64:
1250; SSSE3:       # %bb.0: # %entry
1251; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1252; SSSE3-NEXT:    pxor %xmm4, %xmm4
1253; SSSE3-NEXT:    pxor %xmm3, %xmm3
1254; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1255; SSSE3-NEXT:    pxor %xmm5, %xmm5
1256; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
1257; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1258; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1259; SSSE3-NEXT:    pxor %xmm3, %xmm3
1260; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
1261; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1262; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1263; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1264; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
1265; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1266; SSSE3-NEXT:    retq
1267;
1268; SSE41-LABEL: sext_8i32_to_8i64:
1269; SSE41:       # %bb.0: # %entry
1270; SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1271; SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1272; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1273; SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1274; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1275; SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1276; SSE41-NEXT:    movdqa %xmm5, %xmm0
1277; SSE41-NEXT:    movdqa %xmm4, %xmm1
1278; SSE41-NEXT:    retq
1279;
1280; AVX1-LABEL: sext_8i32_to_8i64:
1281; AVX1:       # %bb.0: # %entry
1282; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1283; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1284; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
1285; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1286; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1287; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1288; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1289; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1290; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1291; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1292; AVX1-NEXT:    retq
1293;
1294; AVX2-LABEL: sext_8i32_to_8i64:
1295; AVX2:       # %bb.0: # %entry
1296; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm2
1297; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1298; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm1
1299; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1300; AVX2-NEXT:    retq
1301;
1302; AVX512-LABEL: sext_8i32_to_8i64:
1303; AVX512:       # %bb.0: # %entry
1304; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
1305; AVX512-NEXT:    retq
1306;
1307; X32-SSE2-LABEL: sext_8i32_to_8i64:
1308; X32-SSE2:       # %bb.0: # %entry
1309; X32-SSE2-NEXT:    movdqa %xmm1, %xmm2
1310; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
1311; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1312; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1313; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1314; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1315; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1316; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1317; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1318; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1319; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1320; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1321; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1322; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1323; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1324; X32-SSE2-NEXT:    retl
1325;
1326; X32-SSE41-LABEL: sext_8i32_to_8i64:
1327; X32-SSE41:       # %bb.0: # %entry
1328; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1329; X32-SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1330; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1331; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1332; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1333; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1334; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
1335; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
1336; X32-SSE41-NEXT:    retl
1337entry:
1338  %B = sext <8 x i32> %A to <8 x i64>
1339  ret <8 x i64> %B
1340}
1341
1342define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) {
1343; SSE-LABEL: load_sext_2i1_to_2i64:
1344; SSE:       # %bb.0: # %entry
1345; SSE-NEXT:    movzbl (%rdi), %eax
1346; SSE-NEXT:    movq %rax, %rcx
1347; SSE-NEXT:    shlq $62, %rcx
1348; SSE-NEXT:    movq %rcx, %xmm0
1349; SSE-NEXT:    shlq $63, %rax
1350; SSE-NEXT:    movq %rax, %xmm1
1351; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1352; SSE-NEXT:    psrad $31, %xmm1
1353; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
1354; SSE-NEXT:    retq
1355;
1356; AVX1-LABEL: load_sext_2i1_to_2i64:
1357; AVX1:       # %bb.0: # %entry
1358; AVX1-NEXT:    movzbl (%rdi), %eax
1359; AVX1-NEXT:    movq %rax, %rcx
1360; AVX1-NEXT:    shlq $62, %rcx
1361; AVX1-NEXT:    vmovq %rcx, %xmm0
1362; AVX1-NEXT:    shlq $63, %rax
1363; AVX1-NEXT:    vmovq %rax, %xmm1
1364; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1365; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1366; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
1367; AVX1-NEXT:    retq
1368;
1369; AVX2-LABEL: load_sext_2i1_to_2i64:
1370; AVX2:       # %bb.0: # %entry
1371; AVX2-NEXT:    movzbl (%rdi), %eax
1372; AVX2-NEXT:    movq %rax, %rcx
1373; AVX2-NEXT:    shlq $62, %rcx
1374; AVX2-NEXT:    vmovq %rcx, %xmm0
1375; AVX2-NEXT:    shlq $63, %rax
1376; AVX2-NEXT:    vmovq %rax, %xmm1
1377; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1378; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1379; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
1380; AVX2-NEXT:    retq
1381;
1382; AVX512-LABEL: load_sext_2i1_to_2i64:
1383; AVX512:       # %bb.0: # %entry
1384; AVX512-NEXT:    kmovw (%rdi), %k1
1385; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1386; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1387; AVX512-NEXT:    vzeroupper
1388; AVX512-NEXT:    retq
1389;
1390; X32-SSE2-LABEL: load_sext_2i1_to_2i64:
1391; X32-SSE2:       # %bb.0: # %entry
1392; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1393; X32-SSE2-NEXT:    movzbl (%eax), %eax
1394; X32-SSE2-NEXT:    movl %eax, %ecx
1395; X32-SSE2-NEXT:    shll $30, %ecx
1396; X32-SSE2-NEXT:    movd %ecx, %xmm0
1397; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]
1398; X32-SSE2-NEXT:    shll $31, %eax
1399; X32-SSE2-NEXT:    movd %eax, %xmm0
1400; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
1401; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1402; X32-SSE2-NEXT:    psrad $31, %xmm0
1403; X32-SSE2-NEXT:    retl
1404;
1405; X32-SSE41-LABEL: load_sext_2i1_to_2i64:
1406; X32-SSE41:       # %bb.0: # %entry
1407; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1408; X32-SSE41-NEXT:    movzbl (%eax), %eax
1409; X32-SSE41-NEXT:    movl %eax, %ecx
1410; X32-SSE41-NEXT:    shll $31, %ecx
1411; X32-SSE41-NEXT:    movd %ecx, %xmm0
1412; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1413; X32-SSE41-NEXT:    shll $30, %eax
1414; X32-SSE41-NEXT:    pinsrd $2, %eax, %xmm0
1415; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1416; X32-SSE41-NEXT:    psrad $31, %xmm0
1417; X32-SSE41-NEXT:    retl
1418entry:
1419 %X = load <2 x i1>, <2 x i1>* %ptr
1420 %Y = sext <2 x i1> %X to <2 x i64>
1421 ret <2 x i64> %Y
1422}
1423
1424define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) {
1425; SSE2-LABEL: load_sext_2i8_to_2i64:
1426; SSE2:       # %bb.0: # %entry
1427; SSE2-NEXT:    movzwl (%rdi), %eax
1428; SSE2-NEXT:    movd %eax, %xmm0
1429; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1430; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1431; SSE2-NEXT:    pxor %xmm1, %xmm1
1432; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1433; SSE2-NEXT:    psrad $24, %xmm0
1434; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1435; SSE2-NEXT:    retq
1436;
1437; SSSE3-LABEL: load_sext_2i8_to_2i64:
1438; SSSE3:       # %bb.0: # %entry
1439; SSSE3-NEXT:    movzwl (%rdi), %eax
1440; SSSE3-NEXT:    movd %eax, %xmm0
1441; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1442; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1443; SSSE3-NEXT:    pxor %xmm1, %xmm1
1444; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1445; SSSE3-NEXT:    psrad $24, %xmm0
1446; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1447; SSSE3-NEXT:    retq
1448;
1449; SSE41-LABEL: load_sext_2i8_to_2i64:
1450; SSE41:       # %bb.0: # %entry
1451; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
1452; SSE41-NEXT:    retq
1453;
1454; AVX-LABEL: load_sext_2i8_to_2i64:
1455; AVX:       # %bb.0: # %entry
1456; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
1457; AVX-NEXT:    retq
1458;
1459; X32-SSE2-LABEL: load_sext_2i8_to_2i64:
1460; X32-SSE2:       # %bb.0: # %entry
1461; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1462; X32-SSE2-NEXT:    movzwl (%eax), %eax
1463; X32-SSE2-NEXT:    movd %eax, %xmm0
1464; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1465; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1466; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1467; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1468; X32-SSE2-NEXT:    psrad $24, %xmm0
1469; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1470; X32-SSE2-NEXT:    retl
1471;
1472; X32-SSE41-LABEL: load_sext_2i8_to_2i64:
1473; X32-SSE41:       # %bb.0: # %entry
1474; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1475; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
1476; X32-SSE41-NEXT:    retl
1477entry:
1478 %X = load <2 x i8>, <2 x i8>* %ptr
1479 %Y = sext <2 x i8> %X to <2 x i64>
1480 ret <2 x i64> %Y
1481}
1482
1483define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) {
1484; SSE2-LABEL: load_sext_4i1_to_4i32:
1485; SSE2:       # %bb.0: # %entry
1486; SSE2-NEXT:    movl (%rdi), %eax
1487; SSE2-NEXT:    movq %rax, %rcx
1488; SSE2-NEXT:    shlq $60, %rcx
1489; SSE2-NEXT:    sarq $63, %rcx
1490; SSE2-NEXT:    movd %ecx, %xmm0
1491; SSE2-NEXT:    movq %rax, %rcx
1492; SSE2-NEXT:    shlq $61, %rcx
1493; SSE2-NEXT:    sarq $63, %rcx
1494; SSE2-NEXT:    movd %ecx, %xmm1
1495; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1496; SSE2-NEXT:    movq %rax, %rcx
1497; SSE2-NEXT:    shlq $62, %rcx
1498; SSE2-NEXT:    sarq $63, %rcx
1499; SSE2-NEXT:    movd %ecx, %xmm2
1500; SSE2-NEXT:    shlq $63, %rax
1501; SSE2-NEXT:    sarq $63, %rax
1502; SSE2-NEXT:    movd %eax, %xmm0
1503; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1504; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1505; SSE2-NEXT:    retq
1506;
1507; SSSE3-LABEL: load_sext_4i1_to_4i32:
1508; SSSE3:       # %bb.0: # %entry
1509; SSSE3-NEXT:    movl (%rdi), %eax
1510; SSSE3-NEXT:    movq %rax, %rcx
1511; SSSE3-NEXT:    shlq $60, %rcx
1512; SSSE3-NEXT:    sarq $63, %rcx
1513; SSSE3-NEXT:    movd %ecx, %xmm0
1514; SSSE3-NEXT:    movq %rax, %rcx
1515; SSSE3-NEXT:    shlq $61, %rcx
1516; SSSE3-NEXT:    sarq $63, %rcx
1517; SSSE3-NEXT:    movd %ecx, %xmm1
1518; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1519; SSSE3-NEXT:    movq %rax, %rcx
1520; SSSE3-NEXT:    shlq $62, %rcx
1521; SSSE3-NEXT:    sarq $63, %rcx
1522; SSSE3-NEXT:    movd %ecx, %xmm2
1523; SSSE3-NEXT:    shlq $63, %rax
1524; SSSE3-NEXT:    sarq $63, %rax
1525; SSSE3-NEXT:    movd %eax, %xmm0
1526; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1527; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1528; SSSE3-NEXT:    retq
1529;
1530; SSE41-LABEL: load_sext_4i1_to_4i32:
1531; SSE41:       # %bb.0: # %entry
1532; SSE41-NEXT:    movl (%rdi), %eax
1533; SSE41-NEXT:    movq %rax, %rcx
1534; SSE41-NEXT:    shlq $62, %rcx
1535; SSE41-NEXT:    sarq $63, %rcx
1536; SSE41-NEXT:    movq %rax, %rdx
1537; SSE41-NEXT:    shlq $63, %rdx
1538; SSE41-NEXT:    sarq $63, %rdx
1539; SSE41-NEXT:    movd %edx, %xmm0
1540; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1541; SSE41-NEXT:    movq %rax, %rcx
1542; SSE41-NEXT:    shlq $61, %rcx
1543; SSE41-NEXT:    sarq $63, %rcx
1544; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1545; SSE41-NEXT:    shlq $60, %rax
1546; SSE41-NEXT:    sarq $63, %rax
1547; SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1548; SSE41-NEXT:    retq
1549;
1550; AVX1-LABEL: load_sext_4i1_to_4i32:
1551; AVX1:       # %bb.0: # %entry
1552; AVX1-NEXT:    movl (%rdi), %eax
1553; AVX1-NEXT:    movq %rax, %rcx
1554; AVX1-NEXT:    shlq $62, %rcx
1555; AVX1-NEXT:    sarq $63, %rcx
1556; AVX1-NEXT:    movq %rax, %rdx
1557; AVX1-NEXT:    shlq $63, %rdx
1558; AVX1-NEXT:    sarq $63, %rdx
1559; AVX1-NEXT:    vmovd %edx, %xmm0
1560; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1561; AVX1-NEXT:    movq %rax, %rcx
1562; AVX1-NEXT:    shlq $61, %rcx
1563; AVX1-NEXT:    sarq $63, %rcx
1564; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1565; AVX1-NEXT:    shlq $60, %rax
1566; AVX1-NEXT:    sarq $63, %rax
1567; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1568; AVX1-NEXT:    retq
1569;
1570; AVX2-LABEL: load_sext_4i1_to_4i32:
1571; AVX2:       # %bb.0: # %entry
1572; AVX2-NEXT:    movl (%rdi), %eax
1573; AVX2-NEXT:    movq %rax, %rcx
1574; AVX2-NEXT:    shlq $62, %rcx
1575; AVX2-NEXT:    sarq $63, %rcx
1576; AVX2-NEXT:    movq %rax, %rdx
1577; AVX2-NEXT:    shlq $63, %rdx
1578; AVX2-NEXT:    sarq $63, %rdx
1579; AVX2-NEXT:    vmovd %edx, %xmm0
1580; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1581; AVX2-NEXT:    movq %rax, %rcx
1582; AVX2-NEXT:    shlq $61, %rcx
1583; AVX2-NEXT:    sarq $63, %rcx
1584; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1585; AVX2-NEXT:    shlq $60, %rax
1586; AVX2-NEXT:    sarq $63, %rax
1587; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1588; AVX2-NEXT:    retq
1589;
1590; AVX512-LABEL: load_sext_4i1_to_4i32:
1591; AVX512:       # %bb.0: # %entry
1592; AVX512-NEXT:    kmovw (%rdi), %k1
1593; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1594; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1595; AVX512-NEXT:    vzeroupper
1596; AVX512-NEXT:    retq
1597;
1598; X32-SSE2-LABEL: load_sext_4i1_to_4i32:
1599; X32-SSE2:       # %bb.0: # %entry
1600; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1601; X32-SSE2-NEXT:    movl (%eax), %eax
1602; X32-SSE2-NEXT:    movl %eax, %ecx
1603; X32-SSE2-NEXT:    shll $28, %ecx
1604; X32-SSE2-NEXT:    movd %ecx, %xmm0
1605; X32-SSE2-NEXT:    movl %eax, %ecx
1606; X32-SSE2-NEXT:    shll $29, %ecx
1607; X32-SSE2-NEXT:    movd %ecx, %xmm1
1608; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1609; X32-SSE2-NEXT:    movl %eax, %ecx
1610; X32-SSE2-NEXT:    shll $30, %ecx
1611; X32-SSE2-NEXT:    movd %ecx, %xmm2
1612; X32-SSE2-NEXT:    shll $31, %eax
1613; X32-SSE2-NEXT:    movd %eax, %xmm0
1614; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1615; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1616; X32-SSE2-NEXT:    psrad $31, %xmm0
1617; X32-SSE2-NEXT:    retl
1618;
1619; X32-SSE41-LABEL: load_sext_4i1_to_4i32:
1620; X32-SSE41:       # %bb.0: # %entry
1621; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1622; X32-SSE41-NEXT:    movl (%eax), %eax
1623; X32-SSE41-NEXT:    movl %eax, %ecx
1624; X32-SSE41-NEXT:    shll $30, %ecx
1625; X32-SSE41-NEXT:    movl %eax, %edx
1626; X32-SSE41-NEXT:    shll $31, %edx
1627; X32-SSE41-NEXT:    movd %edx, %xmm0
1628; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1629; X32-SSE41-NEXT:    movl %eax, %ecx
1630; X32-SSE41-NEXT:    shll $29, %ecx
1631; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1632; X32-SSE41-NEXT:    shll $28, %eax
1633; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1634; X32-SSE41-NEXT:    psrad $31, %xmm0
1635; X32-SSE41-NEXT:    retl
1636entry:
1637 %X = load <4 x i1>, <4 x i1>* %ptr
1638 %Y = sext <4 x i1> %X to <4 x i32>
1639 ret <4 x i32> %Y
1640}
1641
1642define <4 x i32> @load_sext_4i8_to_4i32(<4 x i8> *%ptr) {
1643; SSE2-LABEL: load_sext_4i8_to_4i32:
1644; SSE2:       # %bb.0: # %entry
1645; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1646; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1647; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1648; SSE2-NEXT:    psrad $24, %xmm0
1649; SSE2-NEXT:    retq
1650;
1651; SSSE3-LABEL: load_sext_4i8_to_4i32:
1652; SSSE3:       # %bb.0: # %entry
1653; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1654; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1655; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1656; SSSE3-NEXT:    psrad $24, %xmm0
1657; SSSE3-NEXT:    retq
1658;
1659; SSE41-LABEL: load_sext_4i8_to_4i32:
1660; SSE41:       # %bb.0: # %entry
1661; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
1662; SSE41-NEXT:    retq
1663;
1664; AVX-LABEL: load_sext_4i8_to_4i32:
1665; AVX:       # %bb.0: # %entry
1666; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
1667; AVX-NEXT:    retq
1668;
1669; X32-SSE2-LABEL: load_sext_4i8_to_4i32:
1670; X32-SSE2:       # %bb.0: # %entry
1671; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1672; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1673; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1674; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1675; X32-SSE2-NEXT:    psrad $24, %xmm0
1676; X32-SSE2-NEXT:    retl
1677;
1678; X32-SSE41-LABEL: load_sext_4i8_to_4i32:
1679; X32-SSE41:       # %bb.0: # %entry
1680; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1681; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
1682; X32-SSE41-NEXT:    retl
1683entry:
1684 %X = load <4 x i8>, <4 x i8>* %ptr
1685 %Y = sext <4 x i8> %X to <4 x i32>
1686 ret <4 x i32> %Y
1687}
1688
1689define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
1690; SSE2-LABEL: load_sext_4i1_to_4i64:
1691; SSE2:       # %bb.0: # %entry
1692; SSE2-NEXT:    movl (%rdi), %eax
1693; SSE2-NEXT:    movl %eax, %ecx
1694; SSE2-NEXT:    shrl $3, %ecx
1695; SSE2-NEXT:    movd %ecx, %xmm0
1696; SSE2-NEXT:    movl %eax, %ecx
1697; SSE2-NEXT:    shrl $2, %ecx
1698; SSE2-NEXT:    movd %ecx, %xmm1
1699; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1700; SSE2-NEXT:    movd %eax, %xmm2
1701; SSE2-NEXT:    shrl %eax
1702; SSE2-NEXT:    movd %eax, %xmm0
1703; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1704; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1705; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
1706; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1707; SSE2-NEXT:    psllq $63, %xmm0
1708; SSE2-NEXT:    psrad $31, %xmm0
1709; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1710; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1711; SSE2-NEXT:    psllq $63, %xmm1
1712; SSE2-NEXT:    psrad $31, %xmm1
1713; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1714; SSE2-NEXT:    retq
1715;
1716; SSSE3-LABEL: load_sext_4i1_to_4i64:
1717; SSSE3:       # %bb.0: # %entry
1718; SSSE3-NEXT:    movl (%rdi), %eax
1719; SSSE3-NEXT:    movl %eax, %ecx
1720; SSSE3-NEXT:    shrl $3, %ecx
1721; SSSE3-NEXT:    movd %ecx, %xmm0
1722; SSSE3-NEXT:    movl %eax, %ecx
1723; SSSE3-NEXT:    shrl $2, %ecx
1724; SSSE3-NEXT:    movd %ecx, %xmm1
1725; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1726; SSSE3-NEXT:    movd %eax, %xmm2
1727; SSSE3-NEXT:    shrl %eax
1728; SSSE3-NEXT:    movd %eax, %xmm0
1729; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1730; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1731; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
1732; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1733; SSSE3-NEXT:    psllq $63, %xmm0
1734; SSSE3-NEXT:    psrad $31, %xmm0
1735; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1736; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1737; SSSE3-NEXT:    psllq $63, %xmm1
1738; SSSE3-NEXT:    psrad $31, %xmm1
1739; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1740; SSSE3-NEXT:    retq
1741;
1742; SSE41-LABEL: load_sext_4i1_to_4i64:
1743; SSE41:       # %bb.0: # %entry
1744; SSE41-NEXT:    movl (%rdi), %eax
1745; SSE41-NEXT:    movl %eax, %ecx
1746; SSE41-NEXT:    shrl %ecx
1747; SSE41-NEXT:    movd %eax, %xmm1
1748; SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1749; SSE41-NEXT:    movl %eax, %ecx
1750; SSE41-NEXT:    shrl $2, %ecx
1751; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1752; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1753; SSE41-NEXT:    shrl $3, %eax
1754; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1755; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
1756; SSE41-NEXT:    psllq $63, %xmm0
1757; SSE41-NEXT:    psrad $31, %xmm0
1758; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1759; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1760; SSE41-NEXT:    psllq $63, %xmm1
1761; SSE41-NEXT:    psrad $31, %xmm1
1762; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1763; SSE41-NEXT:    retq
1764;
1765; AVX1-LABEL: load_sext_4i1_to_4i64:
1766; AVX1:       # %bb.0: # %entry
1767; AVX1-NEXT:    movl (%rdi), %eax
1768; AVX1-NEXT:    movq %rax, %rcx
1769; AVX1-NEXT:    shlq $62, %rcx
1770; AVX1-NEXT:    sarq $63, %rcx
1771; AVX1-NEXT:    movq %rax, %rdx
1772; AVX1-NEXT:    shlq $63, %rdx
1773; AVX1-NEXT:    sarq $63, %rdx
1774; AVX1-NEXT:    vmovd %edx, %xmm0
1775; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1776; AVX1-NEXT:    movq %rax, %rcx
1777; AVX1-NEXT:    shlq $61, %rcx
1778; AVX1-NEXT:    sarq $63, %rcx
1779; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1780; AVX1-NEXT:    shlq $60, %rax
1781; AVX1-NEXT:    sarq $63, %rax
1782; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1783; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1784; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1785; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1786; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1787; AVX1-NEXT:    retq
1788;
1789; AVX2-LABEL: load_sext_4i1_to_4i64:
1790; AVX2:       # %bb.0: # %entry
1791; AVX2-NEXT:    movl (%rdi), %eax
1792; AVX2-NEXT:    movq %rax, %rcx
1793; AVX2-NEXT:    shlq $60, %rcx
1794; AVX2-NEXT:    vmovq %rcx, %xmm0
1795; AVX2-NEXT:    movq %rax, %rcx
1796; AVX2-NEXT:    shlq $61, %rcx
1797; AVX2-NEXT:    vmovq %rcx, %xmm1
1798; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1799; AVX2-NEXT:    movq %rax, %rcx
1800; AVX2-NEXT:    shlq $62, %rcx
1801; AVX2-NEXT:    vmovq %rcx, %xmm1
1802; AVX2-NEXT:    shlq $63, %rax
1803; AVX2-NEXT:    vmovq %rax, %xmm2
1804; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1805; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
1806; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1807; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm0
1808; AVX2-NEXT:    retq
1809;
1810; AVX512-LABEL: load_sext_4i1_to_4i64:
1811; AVX512:       # %bb.0: # %entry
1812; AVX512-NEXT:    kmovw (%rdi), %k1
1813; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1814; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1815; AVX512-NEXT:    retq
1816;
1817; X32-SSE2-LABEL: load_sext_4i1_to_4i64:
1818; X32-SSE2:       # %bb.0: # %entry
1819; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1820; X32-SSE2-NEXT:    movzbl (%eax), %eax
1821; X32-SSE2-NEXT:    movl %eax, %ecx
1822; X32-SSE2-NEXT:    shrl $3, %ecx
1823; X32-SSE2-NEXT:    movd %ecx, %xmm0
1824; X32-SSE2-NEXT:    movl %eax, %ecx
1825; X32-SSE2-NEXT:    shrl $2, %ecx
1826; X32-SSE2-NEXT:    movd %ecx, %xmm1
1827; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1828; X32-SSE2-NEXT:    movd %eax, %xmm2
1829; X32-SSE2-NEXT:    shrl %eax
1830; X32-SSE2-NEXT:    movd %eax, %xmm0
1831; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1832; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1833; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
1834; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1835; X32-SSE2-NEXT:    psllq $63, %xmm0
1836; X32-SSE2-NEXT:    psrad $31, %xmm0
1837; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1838; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1839; X32-SSE2-NEXT:    psllq $63, %xmm1
1840; X32-SSE2-NEXT:    psrad $31, %xmm1
1841; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1842; X32-SSE2-NEXT:    retl
1843;
1844; X32-SSE41-LABEL: load_sext_4i1_to_4i64:
1845; X32-SSE41:       # %bb.0: # %entry
1846; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1847; X32-SSE41-NEXT:    movzbl (%eax), %eax
1848; X32-SSE41-NEXT:    movl %eax, %ecx
1849; X32-SSE41-NEXT:    shrl %ecx
1850; X32-SSE41-NEXT:    movd %eax, %xmm1
1851; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1852; X32-SSE41-NEXT:    movl %eax, %ecx
1853; X32-SSE41-NEXT:    shrl $2, %ecx
1854; X32-SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1855; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1856; X32-SSE41-NEXT:    shrl $3, %eax
1857; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1858; X32-SSE41-NEXT:    pand {{\.LCPI.*}}, %xmm1
1859; X32-SSE41-NEXT:    psllq $63, %xmm0
1860; X32-SSE41-NEXT:    psrad $31, %xmm0
1861; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1862; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1863; X32-SSE41-NEXT:    psllq $63, %xmm1
1864; X32-SSE41-NEXT:    psrad $31, %xmm1
1865; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1866; X32-SSE41-NEXT:    retl
1867entry:
1868 %X = load <4 x i1>, <4 x i1>* %ptr
1869 %Y = sext <4 x i1> %X to <4 x i64>
1870 ret <4 x i64> %Y
1871}
1872
1873define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
1874; SSE2-LABEL: load_sext_4i8_to_4i64:
1875; SSE2:       # %bb.0: # %entry
1876; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1877; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1878; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1879; SSE2-NEXT:    psrad $24, %xmm1
1880; SSE2-NEXT:    pxor %xmm2, %xmm2
1881; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1882; SSE2-NEXT:    movdqa %xmm1, %xmm0
1883; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1884; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1885; SSE2-NEXT:    retq
1886;
1887; SSSE3-LABEL: load_sext_4i8_to_4i64:
1888; SSSE3:       # %bb.0: # %entry
1889; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1890; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1891; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1892; SSSE3-NEXT:    psrad $24, %xmm1
1893; SSSE3-NEXT:    pxor %xmm2, %xmm2
1894; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1895; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1896; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1897; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1898; SSSE3-NEXT:    retq
1899;
1900; SSE41-LABEL: load_sext_4i8_to_4i64:
1901; SSE41:       # %bb.0: # %entry
1902; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
1903; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
1904; SSE41-NEXT:    retq
1905;
1906; AVX1-LABEL: load_sext_4i8_to_4i64:
1907; AVX1:       # %bb.0: # %entry
1908; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
1909; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm1
1910; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1911; AVX1-NEXT:    retq
1912;
1913; AVX2-LABEL: load_sext_4i8_to_4i64:
1914; AVX2:       # %bb.0: # %entry
1915; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
1916; AVX2-NEXT:    retq
1917;
1918; AVX512-LABEL: load_sext_4i8_to_4i64:
1919; AVX512:       # %bb.0: # %entry
1920; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
1921; AVX512-NEXT:    retq
1922;
1923; X32-SSE2-LABEL: load_sext_4i8_to_4i64:
1924; X32-SSE2:       # %bb.0: # %entry
1925; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1926; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1927; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1928; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1929; X32-SSE2-NEXT:    psrad $24, %xmm1
1930; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1931; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1932; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
1933; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1934; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1935; X32-SSE2-NEXT:    retl
1936;
1937; X32-SSE41-LABEL: load_sext_4i8_to_4i64:
1938; X32-SSE41:       # %bb.0: # %entry
1939; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1940; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
1941; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
1942; X32-SSE41-NEXT:    retl
1943entry:
1944 %X = load <4 x i8>, <4 x i8>* %ptr
1945 %Y = sext <4 x i8> %X to <4 x i64>
1946 ret <4 x i64> %Y
1947}
1948
1949define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
1950; SSE2-LABEL: load_sext_4i8_to_4i64_extract:
1951; SSE2:       # %bb.0:
1952; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1953; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1954; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1955; SSE2-NEXT:    psrad $24, %xmm0
1956; SSE2-NEXT:    pxor %xmm1, %xmm1
1957; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1958; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1959; SSE2-NEXT:    retq
1960;
1961; SSSE3-LABEL: load_sext_4i8_to_4i64_extract:
1962; SSSE3:       # %bb.0:
1963; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1964; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1965; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1966; SSSE3-NEXT:    psrad $24, %xmm0
1967; SSSE3-NEXT:    pxor %xmm1, %xmm1
1968; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1969; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1970; SSSE3-NEXT:    retq
1971;
1972; SSE41-LABEL: load_sext_4i8_to_4i64_extract:
1973; SSE41:       # %bb.0:
1974; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm0
1975; SSE41-NEXT:    retq
1976;
1977; AVX1-LABEL: load_sext_4i8_to_4i64_extract:
1978; AVX1:       # %bb.0:
1979; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
1980; AVX1-NEXT:    retq
1981;
1982; AVX2-LABEL: load_sext_4i8_to_4i64_extract:
1983; AVX2:       # %bb.0:
1984; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
1985; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1986; AVX2-NEXT:    vzeroupper
1987; AVX2-NEXT:    retq
1988;
1989; AVX512-LABEL: load_sext_4i8_to_4i64_extract:
1990; AVX512:       # %bb.0:
1991; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
1992; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
1993; AVX512-NEXT:    vzeroupper
1994; AVX512-NEXT:    retq
1995;
1996; X32-SSE2-LABEL: load_sext_4i8_to_4i64_extract:
1997; X32-SSE2:       # %bb.0:
1998; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1999; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2000; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2001; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2002; X32-SSE2-NEXT:    psrad $24, %xmm0
2003; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
2004; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2005; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2006; X32-SSE2-NEXT:    retl
2007;
2008; X32-SSE41-LABEL: load_sext_4i8_to_4i64_extract:
2009; X32-SSE41:       # %bb.0:
2010; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2011; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm0
2012; X32-SSE41-NEXT:    retl
2013 %ld = load <4 x i8>, <4 x i8>* %ptr
2014 %sext = sext <4 x i8> %ld to <4 x i64>
2015 %extract = shufflevector <4 x i64> %sext, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
2016 ret <2 x i64> %extract
2017}
2018
2019define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
2020; SSE-LABEL: load_sext_8i1_to_8i16:
2021; SSE:       # %bb.0: # %entry
2022; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2023; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2024; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
2025; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
2026; SSE-NEXT:    pand %xmm1, %xmm0
2027; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
2028; SSE-NEXT:    retq
2029;
2030; AVX1-LABEL: load_sext_8i1_to_8i16:
2031; AVX1:       # %bb.0: # %entry
2032; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2033; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2034; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
2035; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
2036; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
2037; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2038; AVX1-NEXT:    retq
2039;
2040; AVX2-LABEL: load_sext_8i1_to_8i16:
2041; AVX2:       # %bb.0: # %entry
2042; AVX2-NEXT:    vpbroadcastb (%rdi), %xmm0
2043; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
2044; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
2045; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
2046; AVX2-NEXT:    retq
2047;
2048; AVX512F-LABEL: load_sext_8i1_to_8i16:
2049; AVX512F:       # %bb.0: # %entry
2050; AVX512F-NEXT:    kmovw (%rdi), %k1
2051; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2052; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
2053; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2054; AVX512F-NEXT:    vzeroupper
2055; AVX512F-NEXT:    retq
2056;
2057; AVX512BW-LABEL: load_sext_8i1_to_8i16:
2058; AVX512BW:       # %bb.0: # %entry
2059; AVX512BW-NEXT:    kmovw (%rdi), %k0
2060; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
2061; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
2062; AVX512BW-NEXT:    vzeroupper
2063; AVX512BW-NEXT:    retq
2064;
2065; X32-SSE-LABEL: load_sext_8i1_to_8i16:
2066; X32-SSE:       # %bb.0: # %entry
2067; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
2068; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2069; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2070; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
2071; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
2072; X32-SSE-NEXT:    pand %xmm1, %xmm0
2073; X32-SSE-NEXT:    pcmpeqw %xmm1, %xmm0
2074; X32-SSE-NEXT:    retl
2075entry:
2076 %X = load <8 x i1>, <8 x i1>* %ptr
2077 %Y = sext <8 x i1> %X to <8 x i16>
2078 ret <8 x i16> %Y
2079}
2080
2081define <8 x i16> @load_sext_8i8_to_8i16(<8 x i8> *%ptr) {
2082; SSE2-LABEL: load_sext_8i8_to_8i16:
2083; SSE2:       # %bb.0: # %entry
2084; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2085; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2086; SSE2-NEXT:    psraw $8, %xmm0
2087; SSE2-NEXT:    retq
2088;
2089; SSSE3-LABEL: load_sext_8i8_to_8i16:
2090; SSSE3:       # %bb.0: # %entry
2091; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2092; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2093; SSSE3-NEXT:    psraw $8, %xmm0
2094; SSSE3-NEXT:    retq
2095;
2096; SSE41-LABEL: load_sext_8i8_to_8i16:
2097; SSE41:       # %bb.0: # %entry
2098; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
2099; SSE41-NEXT:    retq
2100;
2101; AVX-LABEL: load_sext_8i8_to_8i16:
2102; AVX:       # %bb.0: # %entry
2103; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
2104; AVX-NEXT:    retq
2105;
2106; X32-SSE2-LABEL: load_sext_8i8_to_8i16:
2107; X32-SSE2:       # %bb.0: # %entry
2108; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2109; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2110; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2111; X32-SSE2-NEXT:    psraw $8, %xmm0
2112; X32-SSE2-NEXT:    retl
2113;
2114; X32-SSE41-LABEL: load_sext_8i8_to_8i16:
2115; X32-SSE41:       # %bb.0: # %entry
2116; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2117; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
2118; X32-SSE41-NEXT:    retl
2119entry:
2120 %X = load <8 x i8>, <8 x i8>* %ptr
2121 %Y = sext <8 x i8> %X to <8 x i16>
2122 ret <8 x i16> %Y
2123}
2124
2125define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
2126; SSE2-LABEL: load_sext_8i8_to_8i64:
2127; SSE2:       # %bb.0: # %entry
2128; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2129; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2130; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2131; SSE2-NEXT:    psrad $24, %xmm1
2132; SSE2-NEXT:    pxor %xmm4, %xmm4
2133; SSE2-NEXT:    pxor %xmm3, %xmm3
2134; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
2135; SSE2-NEXT:    movdqa %xmm1, %xmm0
2136; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2137; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2138; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2139; SSE2-NEXT:    psrad $24, %xmm3
2140; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
2141; SSE2-NEXT:    movdqa %xmm3, %xmm2
2142; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2143; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2144; SSE2-NEXT:    retq
2145;
2146; SSSE3-LABEL: load_sext_8i8_to_8i64:
2147; SSSE3:       # %bb.0: # %entry
2148; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2149; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2150; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2151; SSSE3-NEXT:    psrad $24, %xmm1
2152; SSSE3-NEXT:    pxor %xmm4, %xmm4
2153; SSSE3-NEXT:    pxor %xmm3, %xmm3
2154; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
2155; SSSE3-NEXT:    movdqa %xmm1, %xmm0
2156; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2157; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2158; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2159; SSSE3-NEXT:    psrad $24, %xmm3
2160; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
2161; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2162; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2163; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2164; SSSE3-NEXT:    retq
2165;
2166; SSE41-LABEL: load_sext_8i8_to_8i64:
2167; SSE41:       # %bb.0: # %entry
2168; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
2169; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
2170; SSE41-NEXT:    pmovsxbq 4(%rdi), %xmm2
2171; SSE41-NEXT:    pmovsxbq 6(%rdi), %xmm3
2172; SSE41-NEXT:    retq
2173;
2174; AVX1-LABEL: load_sext_8i8_to_8i64:
2175; AVX1:       # %bb.0: # %entry
2176; AVX1-NEXT:    vpmovsxbq 6(%rdi), %xmm1
2177; AVX1-NEXT:    vpmovsxbq 4(%rdi), %xmm2
2178; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
2179; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm3
2180; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
2181; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
2182; AVX1-NEXT:    retq
2183;
2184; AVX2-LABEL: load_sext_8i8_to_8i64:
2185; AVX2:       # %bb.0: # %entry
2186; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2187; AVX2-NEXT:    vpmovsxbq 4(%rdi), %ymm1
2188; AVX2-NEXT:    retq
2189;
2190; AVX512-LABEL: load_sext_8i8_to_8i64:
2191; AVX512:       # %bb.0: # %entry
2192; AVX512-NEXT:    vpmovsxbq (%rdi), %zmm0
2193; AVX512-NEXT:    retq
2194;
2195; X32-SSE2-LABEL: load_sext_8i8_to_8i64:
2196; X32-SSE2:       # %bb.0: # %entry
2197; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2198; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2199; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2200; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2201; X32-SSE2-NEXT:    psrad $24, %xmm1
2202; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
2203; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
2204; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
2205; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
2206; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2207; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2208; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2209; X32-SSE2-NEXT:    psrad $24, %xmm3
2210; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
2211; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
2212; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2213; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2214; X32-SSE2-NEXT:    retl
2215;
2216; X32-SSE41-LABEL: load_sext_8i8_to_8i64:
2217; X32-SSE41:       # %bb.0: # %entry
2218; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2219; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
2220; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
2221; X32-SSE41-NEXT:    pmovsxbq 4(%eax), %xmm2
2222; X32-SSE41-NEXT:    pmovsxbq 6(%eax), %xmm3
2223; X32-SSE41-NEXT:    retl
2224entry:
2225 %X = load <8 x i8>, <8 x i8>* %ptr
2226 %Y = sext <8 x i8> %X to <8 x i64>
2227 ret <8 x i64> %Y
2228}
2229
2230define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
2231; SSE-LABEL: load_sext_8i1_to_8i32:
2232; SSE:       # %bb.0: # %entry
2233; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2234; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
2235; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8]
2236; SSE-NEXT:    movdqa %xmm1, %xmm0
2237; SSE-NEXT:    pand %xmm2, %xmm0
2238; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
2239; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [16,32,64,128]
2240; SSE-NEXT:    pand %xmm2, %xmm1
2241; SSE-NEXT:    pcmpeqd %xmm2, %xmm1
2242; SSE-NEXT:    retq
2243;
2244; AVX1-LABEL: load_sext_8i1_to_8i32:
2245; AVX1:       # %bb.0: # %entry
2246; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
2247; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
2248; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
2249; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
2250; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
2251; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2252; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
2253; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
2254; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
2255; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
2256; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
2257; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
2258; AVX1-NEXT:    retq
2259;
2260; AVX2-LABEL: load_sext_8i1_to_8i32:
2261; AVX2:       # %bb.0: # %entry
2262; AVX2-NEXT:    vpbroadcastb (%rdi), %ymm0
2263; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128]
2264; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
2265; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0
2266; AVX2-NEXT:    retq
2267;
2268; AVX512-LABEL: load_sext_8i1_to_8i32:
2269; AVX512:       # %bb.0: # %entry
2270; AVX512-NEXT:    kmovw (%rdi), %k1
2271; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2272; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2273; AVX512-NEXT:    retq
2274;
2275; X32-SSE-LABEL: load_sext_8i1_to_8i32:
2276; X32-SSE:       # %bb.0: # %entry
2277; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
2278; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2279; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
2280; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8]
2281; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
2282; X32-SSE-NEXT:    pand %xmm2, %xmm0
2283; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm0
2284; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [16,32,64,128]
2285; X32-SSE-NEXT:    pand %xmm2, %xmm1
2286; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm1
2287; X32-SSE-NEXT:    retl
2288entry:
2289 %X = load <8 x i1>, <8 x i1>* %ptr
2290 %Y = sext <8 x i1> %X to <8 x i32>
2291 ret <8 x i32> %Y
2292}
2293
2294define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
2295; SSE2-LABEL: load_sext_8i8_to_8i32:
2296; SSE2:       # %bb.0: # %entry
2297; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2298; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2299; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2300; SSE2-NEXT:    psrad $24, %xmm0
2301; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2302; SSE2-NEXT:    psrad $24, %xmm1
2303; SSE2-NEXT:    retq
2304;
2305; SSSE3-LABEL: load_sext_8i8_to_8i32:
2306; SSSE3:       # %bb.0: # %entry
2307; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2308; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2309; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2310; SSSE3-NEXT:    psrad $24, %xmm0
2311; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2312; SSSE3-NEXT:    psrad $24, %xmm1
2313; SSSE3-NEXT:    retq
2314;
2315; SSE41-LABEL: load_sext_8i8_to_8i32:
2316; SSE41:       # %bb.0: # %entry
2317; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
2318; SSE41-NEXT:    pmovsxbd 4(%rdi), %xmm1
2319; SSE41-NEXT:    retq
2320;
2321; AVX1-LABEL: load_sext_8i8_to_8i32:
2322; AVX1:       # %bb.0: # %entry
2323; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm0
2324; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm1
2325; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2326; AVX1-NEXT:    retq
2327;
2328; AVX2-LABEL: load_sext_8i8_to_8i32:
2329; AVX2:       # %bb.0: # %entry
2330; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm0
2331; AVX2-NEXT:    retq
2332;
2333; AVX512-LABEL: load_sext_8i8_to_8i32:
2334; AVX512:       # %bb.0: # %entry
2335; AVX512-NEXT:    vpmovsxbd (%rdi), %ymm0
2336; AVX512-NEXT:    retq
2337;
2338; X32-SSE2-LABEL: load_sext_8i8_to_8i32:
2339; X32-SSE2:       # %bb.0: # %entry
2340; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2341; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2342; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2343; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2344; X32-SSE2-NEXT:    psrad $24, %xmm0
2345; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2346; X32-SSE2-NEXT:    psrad $24, %xmm1
2347; X32-SSE2-NEXT:    retl
2348;
2349; X32-SSE41-LABEL: load_sext_8i8_to_8i32:
2350; X32-SSE41:       # %bb.0: # %entry
2351; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2352; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
2353; X32-SSE41-NEXT:    pmovsxbd 4(%eax), %xmm1
2354; X32-SSE41-NEXT:    retl
2355entry:
2356 %X = load <8 x i8>, <8 x i8>* %ptr
2357 %Y = sext <8 x i8> %X to <8 x i32>
2358 ret <8 x i32> %Y
2359}
2360
2361define <16 x i8> @load_sext_16i1_to_16i8(<16 x i1> *%ptr) nounwind readnone {
2362; SSE2-LABEL: load_sext_16i1_to_16i8:
2363; SSE2:       # %bb.0: # %entry
2364; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2365; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2366; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,1,1,4,5,6,7]
2367; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
2368; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2369; SSE2-NEXT:    pand %xmm1, %xmm0
2370; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
2371; SSE2-NEXT:    retq
2372;
2373; SSSE3-LABEL: load_sext_16i1_to_16i8:
2374; SSSE3:       # %bb.0: # %entry
2375; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2376; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
2377; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2378; SSSE3-NEXT:    pand %xmm1, %xmm0
2379; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm0
2380; SSSE3-NEXT:    retq
2381;
2382; SSE41-LABEL: load_sext_16i1_to_16i8:
2383; SSE41:       # %bb.0: # %entry
2384; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2385; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
2386; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2387; SSE41-NEXT:    pand %xmm1, %xmm0
2388; SSE41-NEXT:    pcmpeqb %xmm1, %xmm0
2389; SSE41-NEXT:    retq
2390;
2391; AVX1-LABEL: load_sext_16i1_to_16i8:
2392; AVX1:       # %bb.0: # %entry
2393; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2394; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
2395; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = [9241421688590303745,9241421688590303745]
2396; AVX1-NEXT:    # xmm1 = mem[0,0]
2397; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
2398; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
2399; AVX1-NEXT:    retq
2400;
2401; AVX2-LABEL: load_sext_16i1_to_16i8:
2402; AVX2:       # %bb.0: # %entry
2403; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2404; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
2405; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [9241421688590303745,9241421688590303745]
2406; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
2407; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
2408; AVX2-NEXT:    retq
2409;
2410; AVX512F-LABEL: load_sext_16i1_to_16i8:
2411; AVX512F:       # %bb.0: # %entry
2412; AVX512F-NEXT:    kmovw (%rdi), %k1
2413; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2414; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2415; AVX512F-NEXT:    vzeroupper
2416; AVX512F-NEXT:    retq
2417;
2418; AVX512BW-LABEL: load_sext_16i1_to_16i8:
2419; AVX512BW:       # %bb.0: # %entry
2420; AVX512BW-NEXT:    kmovw (%rdi), %k0
2421; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
2422; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
2423; AVX512BW-NEXT:    vzeroupper
2424; AVX512BW-NEXT:    retq
2425;
2426; X32-SSE2-LABEL: load_sext_16i1_to_16i8:
2427; X32-SSE2:       # %bb.0: # %entry
2428; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2429; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2430; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2431; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,1,1,4,5,6,7]
2432; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
2433; X32-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2434; X32-SSE2-NEXT:    pand %xmm1, %xmm0
2435; X32-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
2436; X32-SSE2-NEXT:    retl
2437;
2438; X32-SSE41-LABEL: load_sext_16i1_to_16i8:
2439; X32-SSE41:       # %bb.0: # %entry
2440; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2441; X32-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2442; X32-SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
2443; X32-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2444; X32-SSE41-NEXT:    pand %xmm1, %xmm0
2445; X32-SSE41-NEXT:    pcmpeqb %xmm1, %xmm0
2446; X32-SSE41-NEXT:    retl
2447entry:
2448 %X = load <16 x i1>, <16 x i1>* %ptr
2449 %Y = sext <16 x i1> %X to <16 x i8>
2450 ret <16 x i8> %Y
2451}
2452
2453define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
2454; SSE-LABEL: load_sext_16i1_to_16i16:
2455; SSE:       # %bb.0: # %entry
2456; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2457; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2458; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
2459; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
2460; SSE-NEXT:    movdqa %xmm1, %xmm0
2461; SSE-NEXT:    pand %xmm2, %xmm0
2462; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
2463; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [256,512,1024,2048,4096,8192,16384,32768]
2464; SSE-NEXT:    pand %xmm2, %xmm1
2465; SSE-NEXT:    pcmpeqw %xmm2, %xmm1
2466; SSE-NEXT:    retq
2467;
2468; AVX1-LABEL: load_sext_16i1_to_16i16:
2469; AVX1:       # %bb.0: # %entry
2470; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2471; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2472; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
2473; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
2474; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
2475; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
2476; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2477; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
2478; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
2479; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
2480; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
2481; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
2482; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
2483; AVX1-NEXT:    retq
2484;
2485; AVX2-LABEL: load_sext_16i1_to_16i16:
2486; AVX2:       # %bb.0: # %entry
2487; AVX2-NEXT:    vpbroadcastw (%rdi), %ymm0
2488; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
2489; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
2490; AVX2-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
2491; AVX2-NEXT:    retq
2492;
2493; AVX512F-LABEL: load_sext_16i1_to_16i16:
2494; AVX512F:       # %bb.0: # %entry
2495; AVX512F-NEXT:    kmovw (%rdi), %k1
2496; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2497; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
2498; AVX512F-NEXT:    retq
2499;
2500; AVX512BW-LABEL: load_sext_16i1_to_16i16:
2501; AVX512BW:       # %bb.0: # %entry
2502; AVX512BW-NEXT:    kmovw (%rdi), %k0
2503; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
2504; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2505; AVX512BW-NEXT:    retq
2506;
2507; X32-SSE-LABEL: load_sext_16i1_to_16i16:
2508; X32-SSE:       # %bb.0: # %entry
2509; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
2510; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2511; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2512; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
2513; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
2514; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
2515; X32-SSE-NEXT:    pand %xmm2, %xmm0
2516; X32-SSE-NEXT:    pcmpeqw %xmm2, %xmm0
2517; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [256,512,1024,2048,4096,8192,16384,32768]
2518; X32-SSE-NEXT:    pand %xmm2, %xmm1
2519; X32-SSE-NEXT:    pcmpeqw %xmm2, %xmm1
2520; X32-SSE-NEXT:    retl
2521entry:
2522 %X = load <16 x i1>, <16 x i1>* %ptr
2523 %Y = sext <16 x i1> %X to <16 x i16>
2524 ret <16 x i16> %Y
2525}
2526
2527define <32 x i8> @load_sext_32i1_to_32i8(<32 x i1> *%ptr) nounwind readnone {
2528; SSE-LABEL: load_sext_32i1_to_32i8:
2529; SSE:       # %bb.0: # %entry
2530; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
2531; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2532; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,1,1,4,5,6,7]
2533; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
2534; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2535; SSE-NEXT:    pand %xmm2, %xmm0
2536; SSE-NEXT:    pcmpeqb %xmm2, %xmm0
2537; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,2,3,3,4,5,6,7]
2538; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
2539; SSE-NEXT:    pand %xmm2, %xmm1
2540; SSE-NEXT:    pcmpeqb %xmm2, %xmm1
2541; SSE-NEXT:    retq
2542;
2543; AVX1-LABEL: load_sext_32i1_to_32i8:
2544; AVX1:       # %bb.0: # %entry
2545; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2546; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2547; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,1,1,4,5,6,7]
2548; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,2,3,3,4,5,6,7]
2549; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2550; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]
2551; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
2552; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
2553; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
2554; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
2555; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
2556; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
2557; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
2558; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
2559; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
2560; AVX1-NEXT:    retq
2561;
2562; AVX2-LABEL: load_sext_32i1_to_32i8:
2563; AVX2:       # %bb.0: # %entry
2564; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2565; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
2566; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19]
2567; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
2568; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
2569; AVX2-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
2570; AVX2-NEXT:    retq
2571;
2572; AVX512F-LABEL: load_sext_32i1_to_32i8:
2573; AVX512F:       # %bb.0: # %entry
2574; AVX512F-NEXT:    kmovw (%rdi), %k1
2575; AVX512F-NEXT:    kmovw 2(%rdi), %k2
2576; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2577; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
2578; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
2579; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
2580; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
2581; AVX512F-NEXT:    retq
2582;
2583; AVX512BW-LABEL: load_sext_32i1_to_32i8:
2584; AVX512BW:       # %bb.0: # %entry
2585; AVX512BW-NEXT:    kmovd (%rdi), %k0
2586; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
2587; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2588; AVX512BW-NEXT:    retq
2589;
2590; X32-SSE-LABEL: load_sext_32i1_to_32i8:
2591; X32-SSE:       # %bb.0: # %entry
2592; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
2593; X32-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
2594; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2595; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,1,1,4,5,6,7]
2596; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
2597; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
2598; X32-SSE-NEXT:    pand %xmm2, %xmm0
2599; X32-SSE-NEXT:    pcmpeqb %xmm2, %xmm0
2600; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,2,3,3,4,5,6,7]
2601; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
2602; X32-SSE-NEXT:    pand %xmm2, %xmm1
2603; X32-SSE-NEXT:    pcmpeqb %xmm2, %xmm1
2604; X32-SSE-NEXT:    retl
2605entry:
2606 %X = load <32 x i1>, <32 x i1>* %ptr
2607 %Y = sext <32 x i1> %X to <32 x i8>
2608 ret <32 x i8> %Y
2609}
2610
2611define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
2612; SSE2-LABEL: load_sext_16i8_to_16i16:
2613; SSE2:       # %bb.0: # %entry
2614; SSE2-NEXT:    movdqa (%rdi), %xmm1
2615; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
2616; SSE2-NEXT:    psraw $8, %xmm0
2617; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2618; SSE2-NEXT:    psraw $8, %xmm1
2619; SSE2-NEXT:    retq
2620;
2621; SSSE3-LABEL: load_sext_16i8_to_16i16:
2622; SSSE3:       # %bb.0: # %entry
2623; SSSE3-NEXT:    movdqa (%rdi), %xmm1
2624; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
2625; SSSE3-NEXT:    psraw $8, %xmm0
2626; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2627; SSSE3-NEXT:    psraw $8, %xmm1
2628; SSSE3-NEXT:    retq
2629;
2630; SSE41-LABEL: load_sext_16i8_to_16i16:
2631; SSE41:       # %bb.0: # %entry
2632; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
2633; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
2634; SSE41-NEXT:    retq
2635;
2636; AVX1-LABEL: load_sext_16i8_to_16i16:
2637; AVX1:       # %bb.0: # %entry
2638; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm0
2639; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm1
2640; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2641; AVX1-NEXT:    retq
2642;
2643; AVX2-LABEL: load_sext_16i8_to_16i16:
2644; AVX2:       # %bb.0: # %entry
2645; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
2646; AVX2-NEXT:    retq
2647;
2648; AVX512-LABEL: load_sext_16i8_to_16i16:
2649; AVX512:       # %bb.0: # %entry
2650; AVX512-NEXT:    vpmovsxbw (%rdi), %ymm0
2651; AVX512-NEXT:    retq
2652;
2653; X32-SSE2-LABEL: load_sext_16i8_to_16i16:
2654; X32-SSE2:       # %bb.0: # %entry
2655; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2656; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
2657; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
2658; X32-SSE2-NEXT:    psraw $8, %xmm0
2659; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2660; X32-SSE2-NEXT:    psraw $8, %xmm1
2661; X32-SSE2-NEXT:    retl
2662;
2663; X32-SSE41-LABEL: load_sext_16i8_to_16i16:
2664; X32-SSE41:       # %bb.0: # %entry
2665; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2666; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
2667; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
2668; X32-SSE41-NEXT:    retl
2669entry:
2670 %X = load <16 x i8>, <16 x i8>* %ptr
2671 %Y = sext <16 x i8> %X to <16 x i16>
2672 ret <16 x i16> %Y
2673}
2674
2675define <2 x i64> @load_sext_2i16_to_2i64(<2 x i16> *%ptr) {
2676; SSE2-LABEL: load_sext_2i16_to_2i64:
2677; SSE2:       # %bb.0: # %entry
2678; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2679; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
2680; SSE2-NEXT:    pxor %xmm1, %xmm1
2681; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2682; SSE2-NEXT:    psrad $16, %xmm0
2683; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2684; SSE2-NEXT:    retq
2685;
2686; SSSE3-LABEL: load_sext_2i16_to_2i64:
2687; SSSE3:       # %bb.0: # %entry
2688; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2689; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
2690; SSSE3-NEXT:    pxor %xmm1, %xmm1
2691; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
2692; SSSE3-NEXT:    psrad $16, %xmm0
2693; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2694; SSSE3-NEXT:    retq
2695;
2696; SSE41-LABEL: load_sext_2i16_to_2i64:
2697; SSE41:       # %bb.0: # %entry
2698; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
2699; SSE41-NEXT:    retq
2700;
2701; AVX-LABEL: load_sext_2i16_to_2i64:
2702; AVX:       # %bb.0: # %entry
2703; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
2704; AVX-NEXT:    retq
2705;
2706; X32-SSE2-LABEL: load_sext_2i16_to_2i64:
2707; X32-SSE2:       # %bb.0: # %entry
2708; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2709; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2710; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
2711; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
2712; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2713; X32-SSE2-NEXT:    psrad $16, %xmm0
2714; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2715; X32-SSE2-NEXT:    retl
2716;
2717; X32-SSE41-LABEL: load_sext_2i16_to_2i64:
2718; X32-SSE41:       # %bb.0: # %entry
2719; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2720; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
2721; X32-SSE41-NEXT:    retl
2722entry:
2723 %X = load <2 x i16>, <2 x i16>* %ptr
2724 %Y = sext <2 x i16> %X to <2 x i64>
2725 ret <2 x i64> %Y
2726}
2727
2728define <4 x i32> @load_sext_4i16_to_4i32(<4 x i16> *%ptr) {
2729; SSE2-LABEL: load_sext_4i16_to_4i32:
2730; SSE2:       # %bb.0: # %entry
2731; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2732; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2733; SSE2-NEXT:    psrad $16, %xmm0
2734; SSE2-NEXT:    retq
2735;
2736; SSSE3-LABEL: load_sext_4i16_to_4i32:
2737; SSSE3:       # %bb.0: # %entry
2738; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2739; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2740; SSSE3-NEXT:    psrad $16, %xmm0
2741; SSSE3-NEXT:    retq
2742;
2743; SSE41-LABEL: load_sext_4i16_to_4i32:
2744; SSE41:       # %bb.0: # %entry
2745; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
2746; SSE41-NEXT:    retq
2747;
2748; AVX-LABEL: load_sext_4i16_to_4i32:
2749; AVX:       # %bb.0: # %entry
2750; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
2751; AVX-NEXT:    retq
2752;
2753; X32-SSE2-LABEL: load_sext_4i16_to_4i32:
2754; X32-SSE2:       # %bb.0: # %entry
2755; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2756; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2757; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2758; X32-SSE2-NEXT:    psrad $16, %xmm0
2759; X32-SSE2-NEXT:    retl
2760;
2761; X32-SSE41-LABEL: load_sext_4i16_to_4i32:
2762; X32-SSE41:       # %bb.0: # %entry
2763; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2764; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
2765; X32-SSE41-NEXT:    retl
2766entry:
2767 %X = load <4 x i16>, <4 x i16>* %ptr
2768 %Y = sext <4 x i16> %X to <4 x i32>
2769 ret <4 x i32> %Y
2770}
2771
2772define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
2773; SSE2-LABEL: load_sext_4i16_to_4i64:
2774; SSE2:       # %bb.0: # %entry
2775; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2776; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2777; SSE2-NEXT:    psrad $16, %xmm1
2778; SSE2-NEXT:    pxor %xmm2, %xmm2
2779; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
2780; SSE2-NEXT:    movdqa %xmm1, %xmm0
2781; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2782; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2783; SSE2-NEXT:    retq
2784;
2785; SSSE3-LABEL: load_sext_4i16_to_4i64:
2786; SSSE3:       # %bb.0: # %entry
2787; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2788; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2789; SSSE3-NEXT:    psrad $16, %xmm1
2790; SSSE3-NEXT:    pxor %xmm2, %xmm2
2791; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
2792; SSSE3-NEXT:    movdqa %xmm1, %xmm0
2793; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2794; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2795; SSSE3-NEXT:    retq
2796;
2797; SSE41-LABEL: load_sext_4i16_to_4i64:
2798; SSE41:       # %bb.0: # %entry
2799; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
2800; SSE41-NEXT:    pmovsxwq 4(%rdi), %xmm1
2801; SSE41-NEXT:    retq
2802;
2803; AVX1-LABEL: load_sext_4i16_to_4i64:
2804; AVX1:       # %bb.0: # %entry
2805; AVX1-NEXT:    vpmovsxwq 4(%rdi), %xmm0
2806; AVX1-NEXT:    vpmovsxwq (%rdi), %xmm1
2807; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2808; AVX1-NEXT:    retq
2809;
2810; AVX2-LABEL: load_sext_4i16_to_4i64:
2811; AVX2:       # %bb.0: # %entry
2812; AVX2-NEXT:    vpmovsxwq (%rdi), %ymm0
2813; AVX2-NEXT:    retq
2814;
2815; AVX512-LABEL: load_sext_4i16_to_4i64:
2816; AVX512:       # %bb.0: # %entry
2817; AVX512-NEXT:    vpmovsxwq (%rdi), %ymm0
2818; AVX512-NEXT:    retq
2819;
2820; X32-SSE2-LABEL: load_sext_4i16_to_4i64:
2821; X32-SSE2:       # %bb.0: # %entry
2822; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2823; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2824; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2825; X32-SSE2-NEXT:    psrad $16, %xmm1
2826; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
2827; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
2828; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
2829; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2830; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2831; X32-SSE2-NEXT:    retl
2832;
2833; X32-SSE41-LABEL: load_sext_4i16_to_4i64:
2834; X32-SSE41:       # %bb.0: # %entry
2835; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2836; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
2837; X32-SSE41-NEXT:    pmovsxwq 4(%eax), %xmm1
2838; X32-SSE41-NEXT:    retl
2839entry:
2840 %X = load <4 x i16>, <4 x i16>* %ptr
2841 %Y = sext <4 x i16> %X to <4 x i64>
2842 ret <4 x i64> %Y
2843}
2844
2845define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
2846; SSE2-LABEL: load_sext_8i16_to_8i32:
2847; SSE2:       # %bb.0: # %entry
2848; SSE2-NEXT:    movdqa (%rdi), %xmm1
2849; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2850; SSE2-NEXT:    psrad $16, %xmm0
2851; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2852; SSE2-NEXT:    psrad $16, %xmm1
2853; SSE2-NEXT:    retq
2854;
2855; SSSE3-LABEL: load_sext_8i16_to_8i32:
2856; SSSE3:       # %bb.0: # %entry
2857; SSSE3-NEXT:    movdqa (%rdi), %xmm1
2858; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2859; SSSE3-NEXT:    psrad $16, %xmm0
2860; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2861; SSSE3-NEXT:    psrad $16, %xmm1
2862; SSSE3-NEXT:    retq
2863;
2864; SSE41-LABEL: load_sext_8i16_to_8i32:
2865; SSE41:       # %bb.0: # %entry
2866; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
2867; SSE41-NEXT:    pmovsxwd 8(%rdi), %xmm1
2868; SSE41-NEXT:    retq
2869;
2870; AVX1-LABEL: load_sext_8i16_to_8i32:
2871; AVX1:       # %bb.0: # %entry
2872; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm0
2873; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm1
2874; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2875; AVX1-NEXT:    retq
2876;
2877; AVX2-LABEL: load_sext_8i16_to_8i32:
2878; AVX2:       # %bb.0: # %entry
2879; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm0
2880; AVX2-NEXT:    retq
2881;
2882; AVX512-LABEL: load_sext_8i16_to_8i32:
2883; AVX512:       # %bb.0: # %entry
2884; AVX512-NEXT:    vpmovsxwd (%rdi), %ymm0
2885; AVX512-NEXT:    retq
2886;
2887; X32-SSE2-LABEL: load_sext_8i16_to_8i32:
2888; X32-SSE2:       # %bb.0: # %entry
2889; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2890; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
2891; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2892; X32-SSE2-NEXT:    psrad $16, %xmm0
2893; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2894; X32-SSE2-NEXT:    psrad $16, %xmm1
2895; X32-SSE2-NEXT:    retl
2896;
2897; X32-SSE41-LABEL: load_sext_8i16_to_8i32:
2898; X32-SSE41:       # %bb.0: # %entry
2899; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2900; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
2901; X32-SSE41-NEXT:    pmovsxwd 8(%eax), %xmm1
2902; X32-SSE41-NEXT:    retl
2903entry:
2904 %X = load <8 x i16>, <8 x i16>* %ptr
2905 %Y = sext <8 x i16> %X to <8 x i32>
2906 ret <8 x i32> %Y
2907}
2908
2909define <2 x i64> @load_sext_2i32_to_2i64(<2 x i32> *%ptr) {
2910; SSE2-LABEL: load_sext_2i32_to_2i64:
2911; SSE2:       # %bb.0: # %entry
2912; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2913; SSE2-NEXT:    pxor %xmm1, %xmm1
2914; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2915; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2916; SSE2-NEXT:    retq
2917;
2918; SSSE3-LABEL: load_sext_2i32_to_2i64:
2919; SSSE3:       # %bb.0: # %entry
2920; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2921; SSSE3-NEXT:    pxor %xmm1, %xmm1
2922; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
2923; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2924; SSSE3-NEXT:    retq
2925;
2926; SSE41-LABEL: load_sext_2i32_to_2i64:
2927; SSE41:       # %bb.0: # %entry
2928; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
2929; SSE41-NEXT:    retq
2930;
2931; AVX-LABEL: load_sext_2i32_to_2i64:
2932; AVX:       # %bb.0: # %entry
2933; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
2934; AVX-NEXT:    retq
2935;
2936; X32-SSE2-LABEL: load_sext_2i32_to_2i64:
2937; X32-SSE2:       # %bb.0: # %entry
2938; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2939; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2940; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
2941; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2942; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2943; X32-SSE2-NEXT:    retl
2944;
2945; X32-SSE41-LABEL: load_sext_2i32_to_2i64:
2946; X32-SSE41:       # %bb.0: # %entry
2947; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2948; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
2949; X32-SSE41-NEXT:    retl
2950entry:
2951 %X = load <2 x i32>, <2 x i32>* %ptr
2952 %Y = sext <2 x i32> %X to <2 x i64>
2953 ret <2 x i64> %Y
2954}
2955
2956define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
2957; SSE2-LABEL: load_sext_4i32_to_4i64:
2958; SSE2:       # %bb.0: # %entry
2959; SSE2-NEXT:    movdqa (%rdi), %xmm0
2960; SSE2-NEXT:    pxor %xmm2, %xmm2
2961; SSE2-NEXT:    pxor %xmm3, %xmm3
2962; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
2963; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2964; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2965; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
2966; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2967; SSE2-NEXT:    retq
2968;
2969; SSSE3-LABEL: load_sext_4i32_to_4i64:
2970; SSSE3:       # %bb.0: # %entry
2971; SSSE3-NEXT:    movdqa (%rdi), %xmm0
2972; SSSE3-NEXT:    pxor %xmm2, %xmm2
2973; SSSE3-NEXT:    pxor %xmm3, %xmm3
2974; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
2975; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2976; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2977; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
2978; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2979; SSSE3-NEXT:    retq
2980;
2981; SSE41-LABEL: load_sext_4i32_to_4i64:
2982; SSE41:       # %bb.0: # %entry
2983; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
2984; SSE41-NEXT:    pmovsxdq 8(%rdi), %xmm1
2985; SSE41-NEXT:    retq
2986;
2987; AVX1-LABEL: load_sext_4i32_to_4i64:
2988; AVX1:       # %bb.0: # %entry
2989; AVX1-NEXT:    vpmovsxdq 8(%rdi), %xmm0
2990; AVX1-NEXT:    vpmovsxdq (%rdi), %xmm1
2991; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2992; AVX1-NEXT:    retq
2993;
2994; AVX2-LABEL: load_sext_4i32_to_4i64:
2995; AVX2:       # %bb.0: # %entry
2996; AVX2-NEXT:    vpmovsxdq (%rdi), %ymm0
2997; AVX2-NEXT:    retq
2998;
2999; AVX512-LABEL: load_sext_4i32_to_4i64:
3000; AVX512:       # %bb.0: # %entry
3001; AVX512-NEXT:    vpmovsxdq (%rdi), %ymm0
3002; AVX512-NEXT:    retq
3003;
3004; X32-SSE2-LABEL: load_sext_4i32_to_4i64:
3005; X32-SSE2:       # %bb.0: # %entry
3006; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3007; X32-SSE2-NEXT:    movdqa (%eax), %xmm0
3008; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
3009; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
3010; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3011; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
3012; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3013; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3014; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3015; X32-SSE2-NEXT:    retl
3016;
3017; X32-SSE41-LABEL: load_sext_4i32_to_4i64:
3018; X32-SSE41:       # %bb.0: # %entry
3019; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3020; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
3021; X32-SSE41-NEXT:    pmovsxdq 8(%eax), %xmm1
3022; X32-SSE41-NEXT:    retl
3023entry:
3024 %X = load <4 x i32>, <4 x i32>* %ptr
3025 %Y = sext <4 x i32> %X to <4 x i64>
3026 ret <4 x i64> %Y
3027}
3028
3029define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
3030; SSE2-LABEL: sext_2i8_to_i32:
3031; SSE2:       # %bb.0: # %entry
3032; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3033; SSE2-NEXT:    psraw $8, %xmm0
3034; SSE2-NEXT:    movd %xmm0, %eax
3035; SSE2-NEXT:    retq
3036;
3037; SSSE3-LABEL: sext_2i8_to_i32:
3038; SSSE3:       # %bb.0: # %entry
3039; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3040; SSSE3-NEXT:    psraw $8, %xmm0
3041; SSSE3-NEXT:    movd %xmm0, %eax
3042; SSSE3-NEXT:    retq
3043;
3044; SSE41-LABEL: sext_2i8_to_i32:
3045; SSE41:       # %bb.0: # %entry
3046; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
3047; SSE41-NEXT:    movd %xmm0, %eax
3048; SSE41-NEXT:    retq
3049;
3050; AVX-LABEL: sext_2i8_to_i32:
3051; AVX:       # %bb.0: # %entry
3052; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
3053; AVX-NEXT:    vmovd %xmm0, %eax
3054; AVX-NEXT:    retq
3055;
3056; X32-SSE2-LABEL: sext_2i8_to_i32:
3057; X32-SSE2:       # %bb.0: # %entry
3058; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3059; X32-SSE2-NEXT:    psraw $8, %xmm0
3060; X32-SSE2-NEXT:    movd %xmm0, %eax
3061; X32-SSE2-NEXT:    retl
3062;
3063; X32-SSE41-LABEL: sext_2i8_to_i32:
3064; X32-SSE41:       # %bb.0: # %entry
3065; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
3066; X32-SSE41-NEXT:    movd %xmm0, %eax
3067; X32-SSE41-NEXT:    retl
3068entry:
3069  %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
3070  %Ex = sext <2 x i8> %Shuf to <2 x i16>
3071  %Bc = bitcast <2 x i16> %Ex to i32
3072  ret i32 %Bc
3073}
3074
3075define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
3076; SSE2-LABEL: sext_4i1_to_4i64:
3077; SSE2:       # %bb.0:
3078; SSE2-NEXT:    pslld $31, %xmm0
3079; SSE2-NEXT:    psrad $31, %xmm0
3080; SSE2-NEXT:    pxor %xmm2, %xmm2
3081; SSE2-NEXT:    pxor %xmm3, %xmm3
3082; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3083; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
3084; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3085; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3086; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3087; SSE2-NEXT:    retq
3088;
3089; SSSE3-LABEL: sext_4i1_to_4i64:
3090; SSSE3:       # %bb.0:
3091; SSSE3-NEXT:    pslld $31, %xmm0
3092; SSSE3-NEXT:    psrad $31, %xmm0
3093; SSSE3-NEXT:    pxor %xmm2, %xmm2
3094; SSSE3-NEXT:    pxor %xmm3, %xmm3
3095; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
3096; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
3097; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3098; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
3099; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3100; SSSE3-NEXT:    retq
3101;
3102; SSE41-LABEL: sext_4i1_to_4i64:
3103; SSE41:       # %bb.0:
3104; SSE41-NEXT:    pslld $31, %xmm0
3105; SSE41-NEXT:    psrad $31, %xmm0
3106; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
3107; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
3108; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
3109; SSE41-NEXT:    movdqa %xmm2, %xmm0
3110; SSE41-NEXT:    retq
3111;
3112; AVX1-LABEL: sext_4i1_to_4i64:
3113; AVX1:       # %bb.0:
3114; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
3115; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
3116; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
3117; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
3118; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
3119; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
3120; AVX1-NEXT:    retq
3121;
3122; AVX2-LABEL: sext_4i1_to_4i64:
3123; AVX2:       # %bb.0:
3124; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
3125; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
3126; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
3127; AVX2-NEXT:    retq
3128;
3129; AVX512-LABEL: sext_4i1_to_4i64:
3130; AVX512:       # %bb.0:
3131; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
3132; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm0
3133; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
3134; AVX512-NEXT:    retq
3135;
3136; X32-SSE2-LABEL: sext_4i1_to_4i64:
3137; X32-SSE2:       # %bb.0:
3138; X32-SSE2-NEXT:    pslld $31, %xmm0
3139; X32-SSE2-NEXT:    psrad $31, %xmm0
3140; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
3141; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
3142; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
3143; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
3144; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3145; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3146; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3147; X32-SSE2-NEXT:    retl
3148;
3149; X32-SSE41-LABEL: sext_4i1_to_4i64:
3150; X32-SSE41:       # %bb.0:
3151; X32-SSE41-NEXT:    pslld $31, %xmm0
3152; X32-SSE41-NEXT:    psrad $31, %xmm0
3153; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
3154; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
3155; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
3156; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
3157; X32-SSE41-NEXT:    retl
3158  %extmask = sext <4 x i1> %mask to <4 x i64>
3159  ret <4 x i64> %extmask
3160}
3161
3162define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
3163; SSE2-LABEL: sext_4i8_to_4i64:
3164; SSE2:       # %bb.0:
3165; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3166; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3167; SSE2-NEXT:    psrad $24, %xmm1
3168; SSE2-NEXT:    pxor %xmm2, %xmm2
3169; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3170; SSE2-NEXT:    movdqa %xmm1, %xmm0
3171; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
3172; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3173; SSE2-NEXT:    retq
3174;
3175; SSSE3-LABEL: sext_4i8_to_4i64:
3176; SSSE3:       # %bb.0:
3177; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3178; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3179; SSSE3-NEXT:    psrad $24, %xmm1
3180; SSSE3-NEXT:    pxor %xmm2, %xmm2
3181; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
3182; SSSE3-NEXT:    movdqa %xmm1, %xmm0
3183; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
3184; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3185; SSSE3-NEXT:    retq
3186;
3187; SSE41-LABEL: sext_4i8_to_4i64:
3188; SSE41:       # %bb.0:
3189; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
3190; SSE41-NEXT:    psrld $16, %xmm0
3191; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
3192; SSE41-NEXT:    movdqa %xmm2, %xmm0
3193; SSE41-NEXT:    retq
3194;
3195; AVX1-LABEL: sext_4i8_to_4i64:
3196; AVX1:       # %bb.0:
3197; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
3198; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
3199; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
3200; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
3201; AVX1-NEXT:    retq
3202;
3203; AVX2-LABEL: sext_4i8_to_4i64:
3204; AVX2:       # %bb.0:
3205; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm0
3206; AVX2-NEXT:    retq
3207;
3208; AVX512-LABEL: sext_4i8_to_4i64:
3209; AVX512:       # %bb.0:
3210; AVX512-NEXT:    vpmovsxbq %xmm0, %ymm0
3211; AVX512-NEXT:    retq
3212;
3213; X32-SSE2-LABEL: sext_4i8_to_4i64:
3214; X32-SSE2:       # %bb.0:
3215; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3216; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3217; X32-SSE2-NEXT:    psrad $24, %xmm1
3218; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
3219; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
3220; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
3221; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
3222; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3223; X32-SSE2-NEXT:    retl
3224;
3225; X32-SSE41-LABEL: sext_4i8_to_4i64:
3226; X32-SSE41:       # %bb.0:
3227; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
3228; X32-SSE41-NEXT:    psrld $16, %xmm0
3229; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
3230; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
3231; X32-SSE41-NEXT:    retl
3232  %extmask = sext <4 x i8> %mask to <4 x i64>
3233  ret <4 x i64> %extmask
3234}
3235
3236define <32 x i8> @sext_32xi1_to_32xi8(<32 x i16> %c1, <32 x i16> %c2)nounwind {
3237; SSE-LABEL: sext_32xi1_to_32xi8:
3238; SSE:       # %bb.0:
3239; SSE-NEXT:    pcmpeqw %xmm5, %xmm1
3240; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
3241; SSE-NEXT:    packsswb %xmm1, %xmm0
3242; SSE-NEXT:    pcmpeqw %xmm7, %xmm3
3243; SSE-NEXT:    pcmpeqw %xmm6, %xmm2
3244; SSE-NEXT:    packsswb %xmm3, %xmm2
3245; SSE-NEXT:    movdqa %xmm2, %xmm1
3246; SSE-NEXT:    retq
3247;
3248; AVX1-LABEL: sext_32xi1_to_32xi8:
3249; AVX1:       # %bb.0:
3250; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
3251; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
3252; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm5, %xmm4
3253; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
3254; AVX1-NEXT:    vpacksswb %xmm4, %xmm1, %xmm1
3255; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
3256; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
3257; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm4, %xmm3
3258; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
3259; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
3260; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
3261; AVX1-NEXT:    retq
3262;
3263; AVX2-LABEL: sext_32xi1_to_32xi8:
3264; AVX2:       # %bb.0:
3265; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
3266; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
3267; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
3268; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
3269; AVX2-NEXT:    retq
3270;
3271; AVX512F-LABEL: sext_32xi1_to_32xi8:
3272; AVX512F:       # %bb.0:
3273; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
3274; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
3275; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
3276; AVX512F-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
3277; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
3278; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
3279; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
3280; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
3281; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
3282; AVX512F-NEXT:    retq
3283;
3284; AVX512BW-LABEL: sext_32xi1_to_32xi8:
3285; AVX512BW:       # %bb.0:
3286; AVX512BW-NEXT:    vpcmpeqw %zmm1, %zmm0, %k0
3287; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
3288; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
3289; AVX512BW-NEXT:    retq
3290;
3291; X32-SSE-LABEL: sext_32xi1_to_32xi8:
3292; X32-SSE:       # %bb.0:
3293; X32-SSE-NEXT:    pushl %ebp
3294; X32-SSE-NEXT:    movl %esp, %ebp
3295; X32-SSE-NEXT:    andl $-16, %esp
3296; X32-SSE-NEXT:    subl $16, %esp
3297; X32-SSE-NEXT:    movdqa 8(%ebp), %xmm3
3298; X32-SSE-NEXT:    pcmpeqw 40(%ebp), %xmm1
3299; X32-SSE-NEXT:    pcmpeqw 24(%ebp), %xmm0
3300; X32-SSE-NEXT:    packsswb %xmm1, %xmm0
3301; X32-SSE-NEXT:    pcmpeqw 72(%ebp), %xmm3
3302; X32-SSE-NEXT:    pcmpeqw 56(%ebp), %xmm2
3303; X32-SSE-NEXT:    packsswb %xmm3, %xmm2
3304; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
3305; X32-SSE-NEXT:    movl %ebp, %esp
3306; X32-SSE-NEXT:    popl %ebp
3307; X32-SSE-NEXT:    retl
3308  %a = icmp eq <32 x i16> %c1, %c2
3309  %b = sext <32 x i1> %a to <32 x i8>
3310  ret <32 x i8> %b
3311}
3312
3313define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
3314; SSE2-LABEL: sext_2i8_to_2i32:
3315; SSE2:       # %bb.0:
3316; SSE2-NEXT:    movzwl (%rdi), %eax
3317; SSE2-NEXT:    movd %eax, %xmm0
3318; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3319; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3320; SSE2-NEXT:    psrad $24, %xmm0
3321; SSE2-NEXT:    paddd %xmm0, %xmm0
3322; SSE2-NEXT:    retq
3323;
3324; SSSE3-LABEL: sext_2i8_to_2i32:
3325; SSSE3:       # %bb.0:
3326; SSSE3-NEXT:    movzwl (%rdi), %eax
3327; SSSE3-NEXT:    movd %eax, %xmm0
3328; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3329; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3330; SSSE3-NEXT:    psrad $24, %xmm0
3331; SSSE3-NEXT:    paddd %xmm0, %xmm0
3332; SSSE3-NEXT:    retq
3333;
3334; SSE41-LABEL: sext_2i8_to_2i32:
3335; SSE41:       # %bb.0:
3336; SSE41-NEXT:    movzwl (%rdi), %eax
3337; SSE41-NEXT:    movd %eax, %xmm0
3338; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
3339; SSE41-NEXT:    paddd %xmm0, %xmm0
3340; SSE41-NEXT:    retq
3341;
3342; AVX-LABEL: sext_2i8_to_2i32:
3343; AVX:       # %bb.0:
3344; AVX-NEXT:    movzwl (%rdi), %eax
3345; AVX-NEXT:    vmovd %eax, %xmm0
3346; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
3347; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
3348; AVX-NEXT:    retq
3349;
3350; X32-SSE2-LABEL: sext_2i8_to_2i32:
3351; X32-SSE2:       # %bb.0:
3352; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3353; X32-SSE2-NEXT:    movzwl (%eax), %eax
3354; X32-SSE2-NEXT:    movd %eax, %xmm0
3355; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3356; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3357; X32-SSE2-NEXT:    psrad $24, %xmm0
3358; X32-SSE2-NEXT:    paddd %xmm0, %xmm0
3359; X32-SSE2-NEXT:    retl
3360;
3361; X32-SSE41-LABEL: sext_2i8_to_2i32:
3362; X32-SSE41:       # %bb.0:
3363; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3364; X32-SSE41-NEXT:    movzwl (%eax), %eax
3365; X32-SSE41-NEXT:    movd %eax, %xmm0
3366; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
3367; X32-SSE41-NEXT:    paddd %xmm0, %xmm0
3368; X32-SSE41-NEXT:    retl
3369  %x = load <2 x i8>, <2 x i8>* %addr, align 1
3370  %y = sext <2 x i8> %x to <2 x i32>
3371  %z = add <2 x i32>%y, %y
3372  ret <2 x i32>%z
3373}
3374
3375define <4 x i32> @sext_4i17_to_4i32(<4 x i17>* %ptr) {
3376; SSE2-LABEL: sext_4i17_to_4i32:
3377; SSE2:       # %bb.0:
3378; SSE2-NEXT:    movq (%rdi), %rax
3379; SSE2-NEXT:    movq %rax, %rcx
3380; SSE2-NEXT:    shlq $30, %rcx
3381; SSE2-NEXT:    sarq $47, %rcx
3382; SSE2-NEXT:    movd %ecx, %xmm1
3383; SSE2-NEXT:    movq %rax, %rcx
3384; SSE2-NEXT:    shlq $47, %rcx
3385; SSE2-NEXT:    sarq $47, %rcx
3386; SSE2-NEXT:    movd %ecx, %xmm0
3387; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3388; SSE2-NEXT:    movl 8(%rdi), %ecx
3389; SSE2-NEXT:    shll $13, %ecx
3390; SSE2-NEXT:    movq %rax, %rdx
3391; SSE2-NEXT:    shrq $51, %rdx
3392; SSE2-NEXT:    orl %ecx, %edx
3393; SSE2-NEXT:    shlq $47, %rdx
3394; SSE2-NEXT:    sarq $47, %rdx
3395; SSE2-NEXT:    movd %edx, %xmm1
3396; SSE2-NEXT:    shlq $13, %rax
3397; SSE2-NEXT:    sarq $47, %rax
3398; SSE2-NEXT:    movd %eax, %xmm2
3399; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3400; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3401; SSE2-NEXT:    retq
3402;
3403; SSSE3-LABEL: sext_4i17_to_4i32:
3404; SSSE3:       # %bb.0:
3405; SSSE3-NEXT:    movq (%rdi), %rax
3406; SSSE3-NEXT:    movq %rax, %rcx
3407; SSSE3-NEXT:    shlq $30, %rcx
3408; SSSE3-NEXT:    sarq $47, %rcx
3409; SSSE3-NEXT:    movd %ecx, %xmm1
3410; SSSE3-NEXT:    movq %rax, %rcx
3411; SSSE3-NEXT:    shlq $47, %rcx
3412; SSSE3-NEXT:    sarq $47, %rcx
3413; SSSE3-NEXT:    movd %ecx, %xmm0
3414; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3415; SSSE3-NEXT:    movl 8(%rdi), %ecx
3416; SSSE3-NEXT:    shll $13, %ecx
3417; SSSE3-NEXT:    movq %rax, %rdx
3418; SSSE3-NEXT:    shrq $51, %rdx
3419; SSSE3-NEXT:    orl %ecx, %edx
3420; SSSE3-NEXT:    shlq $47, %rdx
3421; SSSE3-NEXT:    sarq $47, %rdx
3422; SSSE3-NEXT:    movd %edx, %xmm1
3423; SSSE3-NEXT:    shlq $13, %rax
3424; SSSE3-NEXT:    sarq $47, %rax
3425; SSSE3-NEXT:    movd %eax, %xmm2
3426; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3427; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
3428; SSSE3-NEXT:    retq
3429;
3430; SSE41-LABEL: sext_4i17_to_4i32:
3431; SSE41:       # %bb.0:
3432; SSE41-NEXT:    movq (%rdi), %rax
3433; SSE41-NEXT:    movq %rax, %rcx
3434; SSE41-NEXT:    shlq $30, %rcx
3435; SSE41-NEXT:    sarq $47, %rcx
3436; SSE41-NEXT:    movq %rax, %rdx
3437; SSE41-NEXT:    shlq $47, %rdx
3438; SSE41-NEXT:    sarq $47, %rdx
3439; SSE41-NEXT:    movd %edx, %xmm0
3440; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
3441; SSE41-NEXT:    movq %rax, %rcx
3442; SSE41-NEXT:    shlq $13, %rcx
3443; SSE41-NEXT:    sarq $47, %rcx
3444; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
3445; SSE41-NEXT:    movl 8(%rdi), %ecx
3446; SSE41-NEXT:    shll $13, %ecx
3447; SSE41-NEXT:    shrq $51, %rax
3448; SSE41-NEXT:    orl %ecx, %eax
3449; SSE41-NEXT:    shlq $47, %rax
3450; SSE41-NEXT:    sarq $47, %rax
3451; SSE41-NEXT:    pinsrd $3, %eax, %xmm0
3452; SSE41-NEXT:    retq
3453;
3454; AVX-LABEL: sext_4i17_to_4i32:
3455; AVX:       # %bb.0:
3456; AVX-NEXT:    movq (%rdi), %rax
3457; AVX-NEXT:    movq %rax, %rcx
3458; AVX-NEXT:    shlq $30, %rcx
3459; AVX-NEXT:    sarq $47, %rcx
3460; AVX-NEXT:    movq %rax, %rdx
3461; AVX-NEXT:    shlq $47, %rdx
3462; AVX-NEXT:    sarq $47, %rdx
3463; AVX-NEXT:    vmovd %edx, %xmm0
3464; AVX-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
3465; AVX-NEXT:    movq %rax, %rcx
3466; AVX-NEXT:    shlq $13, %rcx
3467; AVX-NEXT:    sarq $47, %rcx
3468; AVX-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
3469; AVX-NEXT:    movl 8(%rdi), %ecx
3470; AVX-NEXT:    shll $13, %ecx
3471; AVX-NEXT:    shrq $51, %rax
3472; AVX-NEXT:    orl %ecx, %eax
3473; AVX-NEXT:    shlq $47, %rax
3474; AVX-NEXT:    sarq $47, %rax
3475; AVX-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
3476; AVX-NEXT:    retq
3477;
3478; X32-SSE2-LABEL: sext_4i17_to_4i32:
3479; X32-SSE2:       # %bb.0:
3480; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3481; X32-SSE2-NEXT:    movl (%eax), %ecx
3482; X32-SSE2-NEXT:    movl 4(%eax), %edx
3483; X32-SSE2-NEXT:    movl 8(%eax), %eax
3484; X32-SSE2-NEXT:    shldl $13, %edx, %eax
3485; X32-SSE2-NEXT:    shll $15, %eax
3486; X32-SSE2-NEXT:    movd %eax, %xmm0
3487; X32-SSE2-NEXT:    movl %edx, %eax
3488; X32-SSE2-NEXT:    shll $13, %eax
3489; X32-SSE2-NEXT:    movd %eax, %xmm1
3490; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3491; X32-SSE2-NEXT:    shldl $15, %ecx, %edx
3492; X32-SSE2-NEXT:    shll $15, %ecx
3493; X32-SSE2-NEXT:    movd %ecx, %xmm0
3494; X32-SSE2-NEXT:    shll $15, %edx
3495; X32-SSE2-NEXT:    movd %edx, %xmm2
3496; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
3497; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
3498; X32-SSE2-NEXT:    psrad $15, %xmm0
3499; X32-SSE2-NEXT:    retl
3500;
3501; X32-SSE41-LABEL: sext_4i17_to_4i32:
3502; X32-SSE41:       # %bb.0:
3503; X32-SSE41-NEXT:    pushl %esi
3504; X32-SSE41-NEXT:    .cfi_def_cfa_offset 8
3505; X32-SSE41-NEXT:    .cfi_offset %esi, -8
3506; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3507; X32-SSE41-NEXT:    movl (%eax), %ecx
3508; X32-SSE41-NEXT:    movl 4(%eax), %edx
3509; X32-SSE41-NEXT:    movl %edx, %esi
3510; X32-SSE41-NEXT:    movl 8(%eax), %eax
3511; X32-SSE41-NEXT:    shldl $13, %edx, %eax
3512; X32-SSE41-NEXT:    shldl $15, %ecx, %edx
3513; X32-SSE41-NEXT:    shll $15, %edx
3514; X32-SSE41-NEXT:    shll $15, %ecx
3515; X32-SSE41-NEXT:    movd %ecx, %xmm0
3516; X32-SSE41-NEXT:    pinsrd $1, %edx, %xmm0
3517; X32-SSE41-NEXT:    shll $13, %esi
3518; X32-SSE41-NEXT:    pinsrd $2, %esi, %xmm0
3519; X32-SSE41-NEXT:    shll $15, %eax
3520; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
3521; X32-SSE41-NEXT:    psrad $15, %xmm0
3522; X32-SSE41-NEXT:    popl %esi
3523; X32-SSE41-NEXT:    .cfi_def_cfa_offset 4
3524; X32-SSE41-NEXT:    retl
3525  %a = load <4 x i17>, <4 x i17>* %ptr
3526  %b = sext <4 x i17> %a to <4 x i32>
3527  ret <4 x i32> %b
3528}
3529
3530define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
3531; SSE2-LABEL: sext_8i6_to_8i64:
3532; SSE2:       # %bb.0: # %entry
3533; SSE2-NEXT:    movd %edi, %xmm0
3534; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3535; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
3536; SSE2-NEXT:    paddw {{.*}}(%rip), %xmm3
3537; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
3538; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
3539; SSE2-NEXT:    psllq $58, %xmm0
3540; SSE2-NEXT:    movdqa %xmm0, %xmm1
3541; SSE2-NEXT:    psrad $31, %xmm1
3542; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3543; SSE2-NEXT:    psrad $26, %xmm0
3544; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
3545; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3546; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
3547; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
3548; SSE2-NEXT:    psllq $58, %xmm1
3549; SSE2-NEXT:    movdqa %xmm1, %xmm2
3550; SSE2-NEXT:    psrad $31, %xmm2
3551; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3552; SSE2-NEXT:    psrad $26, %xmm1
3553; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3554; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3555; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
3556; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
3557; SSE2-NEXT:    psllq $58, %xmm2
3558; SSE2-NEXT:    movdqa %xmm2, %xmm4
3559; SSE2-NEXT:    psrad $31, %xmm4
3560; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3561; SSE2-NEXT:    psrad $26, %xmm2
3562; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3563; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
3564; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
3565; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
3566; SSE2-NEXT:    psllq $58, %xmm3
3567; SSE2-NEXT:    movdqa %xmm3, %xmm4
3568; SSE2-NEXT:    psrad $31, %xmm4
3569; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3570; SSE2-NEXT:    psrad $26, %xmm3
3571; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
3572; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
3573; SSE2-NEXT:    retq
3574;
3575; SSSE3-LABEL: sext_8i6_to_8i64:
3576; SSSE3:       # %bb.0: # %entry
3577; SSSE3-NEXT:    movd %edi, %xmm0
3578; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3579; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
3580; SSSE3-NEXT:    paddw {{.*}}(%rip), %xmm3
3581; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
3582; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
3583; SSSE3-NEXT:    psllq $58, %xmm0
3584; SSSE3-NEXT:    movdqa %xmm0, %xmm1
3585; SSSE3-NEXT:    psrad $31, %xmm1
3586; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3587; SSSE3-NEXT:    psrad $26, %xmm0
3588; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
3589; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3590; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
3591; SSSE3-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
3592; SSSE3-NEXT:    psllq $58, %xmm1
3593; SSSE3-NEXT:    movdqa %xmm1, %xmm2
3594; SSSE3-NEXT:    psrad $31, %xmm2
3595; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3596; SSSE3-NEXT:    psrad $26, %xmm1
3597; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3598; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3599; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
3600; SSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
3601; SSSE3-NEXT:    psllq $58, %xmm2
3602; SSSE3-NEXT:    movdqa %xmm2, %xmm4
3603; SSSE3-NEXT:    psrad $31, %xmm4
3604; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3605; SSSE3-NEXT:    psrad $26, %xmm2
3606; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3607; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
3608; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
3609; SSSE3-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
3610; SSSE3-NEXT:    psllq $58, %xmm3
3611; SSSE3-NEXT:    movdqa %xmm3, %xmm4
3612; SSSE3-NEXT:    psrad $31, %xmm4
3613; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3614; SSSE3-NEXT:    psrad $26, %xmm3
3615; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
3616; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
3617; SSSE3-NEXT:    retq
3618;
3619; SSE41-LABEL: sext_8i6_to_8i64:
3620; SSE41:       # %bb.0: # %entry
3621; SSE41-NEXT:    movd %edi, %xmm0
3622; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3623; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
3624; SSE41-NEXT:    paddw {{.*}}(%rip), %xmm3
3625; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
3626; SSE41-NEXT:    psllq $58, %xmm0
3627; SSE41-NEXT:    movdqa %xmm0, %xmm1
3628; SSE41-NEXT:    psrad $31, %xmm1
3629; SSE41-NEXT:    psrad $26, %xmm0
3630; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
3631; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
3632; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
3633; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
3634; SSE41-NEXT:    psllq $58, %xmm1
3635; SSE41-NEXT:    movdqa %xmm1, %xmm2
3636; SSE41-NEXT:    psrad $31, %xmm2
3637; SSE41-NEXT:    psrad $26, %xmm1
3638; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3639; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
3640; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
3641; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
3642; SSE41-NEXT:    psllq $58, %xmm2
3643; SSE41-NEXT:    movdqa %xmm2, %xmm4
3644; SSE41-NEXT:    psrad $31, %xmm4
3645; SSE41-NEXT:    psrad $26, %xmm2
3646; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3647; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
3648; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
3649; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
3650; SSE41-NEXT:    psllq $58, %xmm3
3651; SSE41-NEXT:    movdqa %xmm3, %xmm4
3652; SSE41-NEXT:    psrad $31, %xmm4
3653; SSE41-NEXT:    psrad $26, %xmm3
3654; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3655; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
3656; SSE41-NEXT:    retq
3657;
3658; AVX1-LABEL: sext_8i6_to_8i64:
3659; AVX1:       # %bb.0: # %entry
3660; AVX1-NEXT:    vmovd %edi, %xmm0
3661; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3662; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
3663; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
3664; AVX1-NEXT:    vpsllw $10, %xmm0, %xmm0
3665; AVX1-NEXT:    vpsraw $10, %xmm0, %xmm1
3666; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm0
3667; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
3668; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
3669; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
3670; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
3671; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
3672; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
3673; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
3674; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
3675; AVX1-NEXT:    retq
3676;
3677; AVX2-LABEL: sext_8i6_to_8i64:
3678; AVX2:       # %bb.0: # %entry
3679; AVX2-NEXT:    vmovd %edi, %xmm0
3680; AVX2-NEXT:    vpbroadcastw %xmm0, %xmm0
3681; AVX2-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
3682; AVX2-NEXT:    vpsllw $10, %xmm0, %xmm0
3683; AVX2-NEXT:    vpsraw $10, %xmm0, %xmm1
3684; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm0
3685; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
3686; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm1
3687; AVX2-NEXT:    retq
3688;
3689; AVX512-LABEL: sext_8i6_to_8i64:
3690; AVX512:       # %bb.0: # %entry
3691; AVX512-NEXT:    vmovd %edi, %xmm0
3692; AVX512-NEXT:    vpbroadcastw %xmm0, %xmm0
3693; AVX512-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
3694; AVX512-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
3695; AVX512-NEXT:    vpsllq $58, %zmm0, %zmm0
3696; AVX512-NEXT:    vpsraq $58, %zmm0, %zmm0
3697; AVX512-NEXT:    retq
3698;
3699; X32-SSE2-LABEL: sext_8i6_to_8i64:
3700; X32-SSE2:       # %bb.0: # %entry
3701; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
3702; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3703; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
3704; X32-SSE2-NEXT:    paddw {{\.LCPI.*}}, %xmm3
3705; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
3706; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
3707; X32-SSE2-NEXT:    psllq $58, %xmm0
3708; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
3709; X32-SSE2-NEXT:    psrad $31, %xmm1
3710; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3711; X32-SSE2-NEXT:    psrad $26, %xmm0
3712; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
3713; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3714; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
3715; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
3716; X32-SSE2-NEXT:    psllq $58, %xmm1
3717; X32-SSE2-NEXT:    movdqa %xmm1, %xmm2
3718; X32-SSE2-NEXT:    psrad $31, %xmm2
3719; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3720; X32-SSE2-NEXT:    psrad $26, %xmm1
3721; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
3722; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3723; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
3724; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
3725; X32-SSE2-NEXT:    psllq $58, %xmm2
3726; X32-SSE2-NEXT:    movdqa %xmm2, %xmm4
3727; X32-SSE2-NEXT:    psrad $31, %xmm4
3728; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3729; X32-SSE2-NEXT:    psrad $26, %xmm2
3730; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
3731; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
3732; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
3733; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
3734; X32-SSE2-NEXT:    psllq $58, %xmm3
3735; X32-SSE2-NEXT:    movdqa %xmm3, %xmm4
3736; X32-SSE2-NEXT:    psrad $31, %xmm4
3737; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
3738; X32-SSE2-NEXT:    psrad $26, %xmm3
3739; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
3740; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
3741; X32-SSE2-NEXT:    retl
3742;
3743; X32-SSE41-LABEL: sext_8i6_to_8i64:
3744; X32-SSE41:       # %bb.0: # %entry
3745; X32-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
3746; X32-SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
3747; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
3748; X32-SSE41-NEXT:    paddw {{\.LCPI.*}}, %xmm3
3749; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
3750; X32-SSE41-NEXT:    psllq $58, %xmm0
3751; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
3752; X32-SSE41-NEXT:    psrad $31, %xmm1
3753; X32-SSE41-NEXT:    psrad $26, %xmm0
3754; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
3755; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
3756; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
3757; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
3758; X32-SSE41-NEXT:    psllq $58, %xmm1
3759; X32-SSE41-NEXT:    movdqa %xmm1, %xmm2
3760; X32-SSE41-NEXT:    psrad $31, %xmm2
3761; X32-SSE41-NEXT:    psrad $26, %xmm1
3762; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
3763; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
3764; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
3765; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
3766; X32-SSE41-NEXT:    psllq $58, %xmm2
3767; X32-SSE41-NEXT:    movdqa %xmm2, %xmm4
3768; X32-SSE41-NEXT:    psrad $31, %xmm4
3769; X32-SSE41-NEXT:    psrad $26, %xmm2
3770; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
3771; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
3772; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
3773; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
3774; X32-SSE41-NEXT:    psllq $58, %xmm3
3775; X32-SSE41-NEXT:    movdqa %xmm3, %xmm4
3776; X32-SSE41-NEXT:    psrad $31, %xmm4
3777; X32-SSE41-NEXT:    psrad $26, %xmm3
3778; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
3779; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
3780; X32-SSE41-NEXT:    retl
3781entry:
3782  %a = trunc i32 %x to i6
3783  %b = insertelement <8 x i6> undef, i6 %a, i32 0
3784  %c = shufflevector <8 x i6> %b, <8 x i6> undef, <8 x i32> zeroinitializer
3785  %d = add <8 x i6> %c, <i6 0, i6 1, i6 2, i6 3, i6 4, i6 5, i6 6, i6 7>
3786  %e = sext <8 x i6> %d to <8 x i64>
3787  ret <8 x i64> %e
3788}
3789
3790define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
3791; SSE2-LABEL: zext_negate_sext:
3792; SSE2:       # %bb.0:
3793; SSE2-NEXT:    pxor %xmm1, %xmm1
3794; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3795; SSE2-NEXT:    psubw %xmm0, %xmm1
3796; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3797; SSE2-NEXT:    psrad $16, %xmm0
3798; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3799; SSE2-NEXT:    psrad $16, %xmm1
3800; SSE2-NEXT:    retq
3801;
3802; SSSE3-LABEL: zext_negate_sext:
3803; SSSE3:       # %bb.0:
3804; SSSE3-NEXT:    pxor %xmm1, %xmm1
3805; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3806; SSSE3-NEXT:    psubw %xmm0, %xmm1
3807; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3808; SSSE3-NEXT:    psrad $16, %xmm0
3809; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3810; SSSE3-NEXT:    psrad $16, %xmm1
3811; SSSE3-NEXT:    retq
3812;
3813; SSE41-LABEL: zext_negate_sext:
3814; SSE41:       # %bb.0:
3815; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3816; SSE41-NEXT:    pxor %xmm1, %xmm1
3817; SSE41-NEXT:    psubw %xmm0, %xmm1
3818; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
3819; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
3820; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
3821; SSE41-NEXT:    retq
3822;
3823; AVX1-LABEL: zext_negate_sext:
3824; AVX1:       # %bb.0:
3825; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
3826; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
3827; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
3828; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
3829; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
3830; AVX1-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
3831; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
3832; AVX1-NEXT:    retq
3833;
3834; AVX2-LABEL: zext_negate_sext:
3835; AVX2:       # %bb.0:
3836; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
3837; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
3838; AVX2-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
3839; AVX2-NEXT:    retq
3840;
3841; AVX512-LABEL: zext_negate_sext:
3842; AVX512:       # %bb.0:
3843; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
3844; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
3845; AVX512-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
3846; AVX512-NEXT:    retq
3847;
3848; X32-SSE2-LABEL: zext_negate_sext:
3849; X32-SSE2:       # %bb.0:
3850; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
3851; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3852; X32-SSE2-NEXT:    psubw %xmm0, %xmm1
3853; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3854; X32-SSE2-NEXT:    psrad $16, %xmm0
3855; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3856; X32-SSE2-NEXT:    psrad $16, %xmm1
3857; X32-SSE2-NEXT:    retl
3858;
3859; X32-SSE41-LABEL: zext_negate_sext:
3860; X32-SSE41:       # %bb.0:
3861; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3862; X32-SSE41-NEXT:    pxor %xmm1, %xmm1
3863; X32-SSE41-NEXT:    psubw %xmm0, %xmm1
3864; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
3865; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
3866; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
3867; X32-SSE41-NEXT:    retl
3868  %z = zext <8 x i8> %x to <8 x i16>
3869  %neg = sub nsw <8 x i16> zeroinitializer, %z
3870  %r = sext <8 x i16> %neg to <8 x i32>
3871  ret <8 x i32> %r
3872}
3873
3874define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
3875; SSE2-LABEL: zext_decremenet_sext:
3876; SSE2:       # %bb.0:
3877; SSE2-NEXT:    pxor %xmm1, %xmm1
3878; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3879; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
3880; SSE2-NEXT:    paddw %xmm0, %xmm1
3881; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3882; SSE2-NEXT:    psrad $16, %xmm0
3883; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3884; SSE2-NEXT:    psrad $16, %xmm1
3885; SSE2-NEXT:    retq
3886;
3887; SSSE3-LABEL: zext_decremenet_sext:
3888; SSSE3:       # %bb.0:
3889; SSSE3-NEXT:    pxor %xmm1, %xmm1
3890; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3891; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
3892; SSSE3-NEXT:    paddw %xmm0, %xmm1
3893; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3894; SSSE3-NEXT:    psrad $16, %xmm0
3895; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3896; SSSE3-NEXT:    psrad $16, %xmm1
3897; SSSE3-NEXT:    retq
3898;
3899; SSE41-LABEL: zext_decremenet_sext:
3900; SSE41:       # %bb.0:
3901; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3902; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
3903; SSE41-NEXT:    paddw %xmm0, %xmm1
3904; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
3905; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
3906; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
3907; SSE41-NEXT:    retq
3908;
3909; AVX1-LABEL: zext_decremenet_sext:
3910; AVX1:       # %bb.0:
3911; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
3912; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
3913; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
3914; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
3915; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
3916; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
3917; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
3918; AVX1-NEXT:    retq
3919;
3920; AVX2-LABEL: zext_decremenet_sext:
3921; AVX2:       # %bb.0:
3922; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
3923; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
3924; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
3925; AVX2-NEXT:    retq
3926;
3927; AVX512-LABEL: zext_decremenet_sext:
3928; AVX512:       # %bb.0:
3929; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
3930; AVX512-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
3931; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
3932; AVX512-NEXT:    retq
3933;
3934; X32-SSE2-LABEL: zext_decremenet_sext:
3935; X32-SSE2:       # %bb.0:
3936; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
3937; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
3938; X32-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
3939; X32-SSE2-NEXT:    paddw %xmm0, %xmm1
3940; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3941; X32-SSE2-NEXT:    psrad $16, %xmm0
3942; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
3943; X32-SSE2-NEXT:    psrad $16, %xmm1
3944; X32-SSE2-NEXT:    retl
3945;
3946; X32-SSE41-LABEL: zext_decremenet_sext:
3947; X32-SSE41:       # %bb.0:
3948; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3949; X32-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
3950; X32-SSE41-NEXT:    paddw %xmm0, %xmm1
3951; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
3952; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
3953; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
3954; X32-SSE41-NEXT:    retl
3955  %z = zext <8 x i8> %x to <8 x i16>
3956  %dec = add <8 x i16> %z, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
3957  %r = sext <8 x i16> %dec to <8 x i32>
3958  ret <8 x i32> %r
3959}
3960