1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
9;
10; Just two 32-bit runs to make sure we do reasonable things there.
11; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE2
12; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41
13
14define <8 x i16> @sext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
15; SSE2-LABEL: sext_16i8_to_8i16:
16; SSE2:       # %bb.0: # %entry
17; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
18; SSE2-NEXT:    psraw $8, %xmm0
19; SSE2-NEXT:    retq
20;
21; SSSE3-LABEL: sext_16i8_to_8i16:
22; SSSE3:       # %bb.0: # %entry
23; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
24; SSSE3-NEXT:    psraw $8, %xmm0
25; SSSE3-NEXT:    retq
26;
27; SSE41-LABEL: sext_16i8_to_8i16:
28; SSE41:       # %bb.0: # %entry
29; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
30; SSE41-NEXT:    retq
31;
32; AVX-LABEL: sext_16i8_to_8i16:
33; AVX:       # %bb.0: # %entry
34; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
35; AVX-NEXT:    retq
36;
37; X32-SSE2-LABEL: sext_16i8_to_8i16:
38; X32-SSE2:       # %bb.0: # %entry
39; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
40; X32-SSE2-NEXT:    psraw $8, %xmm0
41; X32-SSE2-NEXT:    retl
42;
43; X32-SSE41-LABEL: sext_16i8_to_8i16:
44; X32-SSE41:       # %bb.0: # %entry
45; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
46; X32-SSE41-NEXT:    retl
47entry:
48  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
49  %C = sext <8 x i8> %B to <8 x i16>
50  ret <8 x i16> %C
51}
52
53define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ssp {
54; SSE2-LABEL: sext_16i8_to_16i16:
55; SSE2:       # %bb.0: # %entry
56; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
57; SSE2-NEXT:    psraw $8, %xmm2
58; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
59; SSE2-NEXT:    psraw $8, %xmm1
60; SSE2-NEXT:    movdqa %xmm2, %xmm0
61; SSE2-NEXT:    retq
62;
63; SSSE3-LABEL: sext_16i8_to_16i16:
64; SSSE3:       # %bb.0: # %entry
65; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
66; SSSE3-NEXT:    psraw $8, %xmm2
67; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
68; SSSE3-NEXT:    psraw $8, %xmm1
69; SSSE3-NEXT:    movdqa %xmm2, %xmm0
70; SSSE3-NEXT:    retq
71;
72; SSE41-LABEL: sext_16i8_to_16i16:
73; SSE41:       # %bb.0: # %entry
74; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
75; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
76; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
77; SSE41-NEXT:    movdqa %xmm2, %xmm0
78; SSE41-NEXT:    retq
79;
80; AVX1-LABEL: sext_16i8_to_16i16:
81; AVX1:       # %bb.0: # %entry
82; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
83; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
84; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
85; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
86; AVX1-NEXT:    retq
87;
88; AVX2-LABEL: sext_16i8_to_16i16:
89; AVX2:       # %bb.0: # %entry
90; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
91; AVX2-NEXT:    retq
92;
93; AVX512-LABEL: sext_16i8_to_16i16:
94; AVX512:       # %bb.0: # %entry
95; AVX512-NEXT:    vpmovsxbw %xmm0, %ymm0
96; AVX512-NEXT:    retq
97;
98; X32-SSE2-LABEL: sext_16i8_to_16i16:
99; X32-SSE2:       # %bb.0: # %entry
100; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
101; X32-SSE2-NEXT:    psraw $8, %xmm2
102; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
103; X32-SSE2-NEXT:    psraw $8, %xmm1
104; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
105; X32-SSE2-NEXT:    retl
106;
107; X32-SSE41-LABEL: sext_16i8_to_16i16:
108; X32-SSE41:       # %bb.0: # %entry
109; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
110; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
111; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
112; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
113; X32-SSE41-NEXT:    retl
114entry:
115  %B = sext <16 x i8> %A to <16 x i16>
116  ret <16 x i16> %B
117}
118
119define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ssp {
120; SSE2-LABEL: sext_32i8_to_32i16:
121; SSE2:       # %bb.0: # %entry
122; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
123; SSE2-NEXT:    psraw $8, %xmm4
124; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
125; SSE2-NEXT:    psraw $8, %xmm5
126; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
127; SSE2-NEXT:    psraw $8, %xmm2
128; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
129; SSE2-NEXT:    psraw $8, %xmm3
130; SSE2-NEXT:    movdqa %xmm4, %xmm0
131; SSE2-NEXT:    movdqa %xmm5, %xmm1
132; SSE2-NEXT:    retq
133;
134; SSSE3-LABEL: sext_32i8_to_32i16:
135; SSSE3:       # %bb.0: # %entry
136; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
137; SSSE3-NEXT:    psraw $8, %xmm4
138; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
139; SSSE3-NEXT:    psraw $8, %xmm5
140; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
141; SSSE3-NEXT:    psraw $8, %xmm2
142; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
143; SSSE3-NEXT:    psraw $8, %xmm3
144; SSSE3-NEXT:    movdqa %xmm4, %xmm0
145; SSSE3-NEXT:    movdqa %xmm5, %xmm1
146; SSSE3-NEXT:    retq
147;
148; SSE41-LABEL: sext_32i8_to_32i16:
149; SSE41:       # %bb.0: # %entry
150; SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
151; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
152; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
153; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
154; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
155; SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
156; SSE41-NEXT:    movdqa %xmm5, %xmm0
157; SSE41-NEXT:    movdqa %xmm4, %xmm1
158; SSE41-NEXT:    retq
159;
160; AVX1-LABEL: sext_32i8_to_32i16:
161; AVX1:       # %bb.0: # %entry
162; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
163; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
164; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
165; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
166; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
167; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
168; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
169; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
170; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
171; AVX1-NEXT:    vmovaps %ymm2, %ymm0
172; AVX1-NEXT:    retq
173;
174; AVX2-LABEL: sext_32i8_to_32i16:
175; AVX2:       # %bb.0: # %entry
176; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm2
177; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
178; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm1
179; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
180; AVX2-NEXT:    retq
181;
182; AVX512F-LABEL: sext_32i8_to_32i16:
183; AVX512F:       # %bb.0: # %entry
184; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm2
185; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
186; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
187; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
188; AVX512F-NEXT:    retq
189;
190; AVX512BW-LABEL: sext_32i8_to_32i16:
191; AVX512BW:       # %bb.0: # %entry
192; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
193; AVX512BW-NEXT:    retq
194;
195; X32-SSE2-LABEL: sext_32i8_to_32i16:
196; X32-SSE2:       # %bb.0: # %entry
197; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
198; X32-SSE2-NEXT:    psraw $8, %xmm4
199; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
200; X32-SSE2-NEXT:    psraw $8, %xmm5
201; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
202; X32-SSE2-NEXT:    psraw $8, %xmm2
203; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
204; X32-SSE2-NEXT:    psraw $8, %xmm3
205; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
206; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
207; X32-SSE2-NEXT:    retl
208;
209; X32-SSE41-LABEL: sext_32i8_to_32i16:
210; X32-SSE41:       # %bb.0: # %entry
211; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
212; X32-SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
213; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
214; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
215; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
216; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
217; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
218; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
219; X32-SSE41-NEXT:    retl
220entry:
221  %B = sext <32 x i8> %A to <32 x i16>
222  ret <32 x i16> %B
223}
224
225define <4 x i32> @sext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
226; SSE2-LABEL: sext_16i8_to_4i32:
227; SSE2:       # %bb.0: # %entry
228; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
229; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
230; SSE2-NEXT:    psrad $24, %xmm0
231; SSE2-NEXT:    retq
232;
233; SSSE3-LABEL: sext_16i8_to_4i32:
234; SSSE3:       # %bb.0: # %entry
235; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
236; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
237; SSSE3-NEXT:    psrad $24, %xmm0
238; SSSE3-NEXT:    retq
239;
240; SSE41-LABEL: sext_16i8_to_4i32:
241; SSE41:       # %bb.0: # %entry
242; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
243; SSE41-NEXT:    retq
244;
245; AVX-LABEL: sext_16i8_to_4i32:
246; AVX:       # %bb.0: # %entry
247; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
248; AVX-NEXT:    retq
249;
250; X32-SSE2-LABEL: sext_16i8_to_4i32:
251; X32-SSE2:       # %bb.0: # %entry
252; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
253; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
254; X32-SSE2-NEXT:    psrad $24, %xmm0
255; X32-SSE2-NEXT:    retl
256;
257; X32-SSE41-LABEL: sext_16i8_to_4i32:
258; X32-SSE41:       # %bb.0: # %entry
259; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
260; X32-SSE41-NEXT:    retl
261entry:
262  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
263  %C = sext <4 x i8> %B to <4 x i32>
264  ret <4 x i32> %C
265}
266
267define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
268; SSE2-LABEL: sext_16i8_to_8i32:
269; SSE2:       # %bb.0: # %entry
270; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
271; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
272; SSE2-NEXT:    psrad $24, %xmm2
273; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
274; SSE2-NEXT:    psrad $24, %xmm1
275; SSE2-NEXT:    movdqa %xmm2, %xmm0
276; SSE2-NEXT:    retq
277;
278; SSSE3-LABEL: sext_16i8_to_8i32:
279; SSSE3:       # %bb.0: # %entry
280; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
281; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
282; SSSE3-NEXT:    psrad $24, %xmm2
283; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
284; SSSE3-NEXT:    psrad $24, %xmm1
285; SSSE3-NEXT:    movdqa %xmm2, %xmm0
286; SSSE3-NEXT:    retq
287;
288; SSE41-LABEL: sext_16i8_to_8i32:
289; SSE41:       # %bb.0: # %entry
290; SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
291; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
292; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
293; SSE41-NEXT:    movdqa %xmm2, %xmm0
294; SSE41-NEXT:    retq
295;
296; AVX1-LABEL: sext_16i8_to_8i32:
297; AVX1:       # %bb.0: # %entry
298; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
299; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
300; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
301; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
302; AVX1-NEXT:    retq
303;
304; AVX2-LABEL: sext_16i8_to_8i32:
305; AVX2:       # %bb.0: # %entry
306; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
307; AVX2-NEXT:    retq
308;
309; AVX512-LABEL: sext_16i8_to_8i32:
310; AVX512:       # %bb.0: # %entry
311; AVX512-NEXT:    vpmovsxbd %xmm0, %ymm0
312; AVX512-NEXT:    retq
313;
314; X32-SSE2-LABEL: sext_16i8_to_8i32:
315; X32-SSE2:       # %bb.0: # %entry
316; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
317; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
318; X32-SSE2-NEXT:    psrad $24, %xmm2
319; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
320; X32-SSE2-NEXT:    psrad $24, %xmm1
321; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
322; X32-SSE2-NEXT:    retl
323;
324; X32-SSE41-LABEL: sext_16i8_to_8i32:
325; X32-SSE41:       # %bb.0: # %entry
326; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
327; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
328; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
329; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
330; X32-SSE41-NEXT:    retl
331entry:
332  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
333  %C = sext <8 x i8> %B to <8 x i32>
334  ret <8 x i32> %C
335}
336
337define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ssp {
338; SSE2-LABEL: sext_16i8_to_16i32:
339; SSE2:       # %bb.0: # %entry
340; SSE2-NEXT:    movdqa %xmm0, %xmm1
341; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
342; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
343; SSE2-NEXT:    psrad $24, %xmm4
344; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
345; SSE2-NEXT:    psrad $24, %xmm1
346; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
347; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
348; SSE2-NEXT:    psrad $24, %xmm2
349; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
350; SSE2-NEXT:    psrad $24, %xmm3
351; SSE2-NEXT:    movdqa %xmm4, %xmm0
352; SSE2-NEXT:    retq
353;
354; SSSE3-LABEL: sext_16i8_to_16i32:
355; SSSE3:       # %bb.0: # %entry
356; SSSE3-NEXT:    movdqa %xmm0, %xmm1
357; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
358; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
359; SSSE3-NEXT:    psrad $24, %xmm4
360; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
361; SSSE3-NEXT:    psrad $24, %xmm1
362; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
363; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
364; SSSE3-NEXT:    psrad $24, %xmm2
365; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
366; SSSE3-NEXT:    psrad $24, %xmm3
367; SSSE3-NEXT:    movdqa %xmm4, %xmm0
368; SSSE3-NEXT:    retq
369;
370; SSE41-LABEL: sext_16i8_to_16i32:
371; SSE41:       # %bb.0: # %entry
372; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
373; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
374; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
375; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
376; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
377; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
378; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
379; SSE41-NEXT:    movdqa %xmm4, %xmm0
380; SSE41-NEXT:    retq
381;
382; AVX1-LABEL: sext_16i8_to_16i32:
383; AVX1:       # %bb.0: # %entry
384; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
385; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
386; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
387; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
388; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
389; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
390; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
391; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
392; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
393; AVX1-NEXT:    vmovaps %ymm2, %ymm0
394; AVX1-NEXT:    retq
395;
396; AVX2-LABEL: sext_16i8_to_16i32:
397; AVX2:       # %bb.0: # %entry
398; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm2
399; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
400; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm1
401; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
402; AVX2-NEXT:    retq
403;
404; AVX512-LABEL: sext_16i8_to_16i32:
405; AVX512:       # %bb.0: # %entry
406; AVX512-NEXT:    vpmovsxbd %xmm0, %zmm0
407; AVX512-NEXT:    retq
408;
409; X32-SSE2-LABEL: sext_16i8_to_16i32:
410; X32-SSE2:       # %bb.0: # %entry
411; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
412; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
413; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
414; X32-SSE2-NEXT:    psrad $24, %xmm4
415; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
416; X32-SSE2-NEXT:    psrad $24, %xmm1
417; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
418; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
419; X32-SSE2-NEXT:    psrad $24, %xmm2
420; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
421; X32-SSE2-NEXT:    psrad $24, %xmm3
422; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
423; X32-SSE2-NEXT:    retl
424;
425; X32-SSE41-LABEL: sext_16i8_to_16i32:
426; X32-SSE41:       # %bb.0: # %entry
427; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
428; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
429; X32-SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
430; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
431; X32-SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
432; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
433; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
434; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
435; X32-SSE41-NEXT:    retl
436entry:
437  %B = sext <16 x i8> %A to <16 x i32>
438  ret <16 x i32> %B
439}
440
441define <2 x i64> @sext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
442; SSE2-LABEL: sext_16i8_to_2i64:
443; SSE2:       # %bb.0: # %entry
444; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
445; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
446; SSE2-NEXT:    pxor %xmm1, %xmm1
447; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
448; SSE2-NEXT:    psrad $24, %xmm0
449; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
450; SSE2-NEXT:    retq
451;
452; SSSE3-LABEL: sext_16i8_to_2i64:
453; SSSE3:       # %bb.0: # %entry
454; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
455; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
456; SSSE3-NEXT:    pxor %xmm1, %xmm1
457; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
458; SSSE3-NEXT:    psrad $24, %xmm0
459; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
460; SSSE3-NEXT:    retq
461;
462; SSE41-LABEL: sext_16i8_to_2i64:
463; SSE41:       # %bb.0: # %entry
464; SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
465; SSE41-NEXT:    retq
466;
467; AVX-LABEL: sext_16i8_to_2i64:
468; AVX:       # %bb.0: # %entry
469; AVX-NEXT:    vpmovsxbq %xmm0, %xmm0
470; AVX-NEXT:    retq
471;
472; X32-SSE2-LABEL: sext_16i8_to_2i64:
473; X32-SSE2:       # %bb.0: # %entry
474; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
475; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
476; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
477; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
478; X32-SSE2-NEXT:    psrad $24, %xmm0
479; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
480; X32-SSE2-NEXT:    retl
481;
482; X32-SSE41-LABEL: sext_16i8_to_2i64:
483; X32-SSE41:       # %bb.0: # %entry
484; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
485; X32-SSE41-NEXT:    retl
486entry:
487  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
488  %C = sext <2 x i8> %B to <2 x i64>
489  ret <2 x i64> %C
490}
491
492define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
493; SSE2-LABEL: sext_16i8_to_4i64:
494; SSE2:       # %bb.0: # %entry
495; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
496; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
497; SSE2-NEXT:    pxor %xmm3, %xmm3
498; SSE2-NEXT:    pxor %xmm1, %xmm1
499; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
500; SSE2-NEXT:    psrad $24, %xmm2
501; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
502; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
503; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
504; SSE2-NEXT:    psrad $24, %xmm1
505; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
506; SSE2-NEXT:    movdqa %xmm2, %xmm0
507; SSE2-NEXT:    retq
508;
509; SSSE3-LABEL: sext_16i8_to_4i64:
510; SSSE3:       # %bb.0: # %entry
511; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
512; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
513; SSSE3-NEXT:    pxor %xmm3, %xmm3
514; SSSE3-NEXT:    pxor %xmm1, %xmm1
515; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
516; SSSE3-NEXT:    psrad $24, %xmm2
517; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
518; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
519; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
520; SSSE3-NEXT:    psrad $24, %xmm1
521; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
522; SSSE3-NEXT:    movdqa %xmm2, %xmm0
523; SSSE3-NEXT:    retq
524;
525; SSE41-LABEL: sext_16i8_to_4i64:
526; SSE41:       # %bb.0: # %entry
527; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
528; SSE41-NEXT:    psrld $16, %xmm0
529; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
530; SSE41-NEXT:    movdqa %xmm2, %xmm0
531; SSE41-NEXT:    retq
532;
533; AVX1-LABEL: sext_16i8_to_4i64:
534; AVX1:       # %bb.0: # %entry
535; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
536; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
537; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
538; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
539; AVX1-NEXT:    retq
540;
541; AVX2-LABEL: sext_16i8_to_4i64:
542; AVX2:       # %bb.0: # %entry
543; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm0
544; AVX2-NEXT:    retq
545;
546; AVX512-LABEL: sext_16i8_to_4i64:
547; AVX512:       # %bb.0: # %entry
548; AVX512-NEXT:    vpmovsxbq %xmm0, %ymm0
549; AVX512-NEXT:    retq
550;
551; X32-SSE2-LABEL: sext_16i8_to_4i64:
552; X32-SSE2:       # %bb.0: # %entry
553; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
554; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
555; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
556; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
557; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
558; X32-SSE2-NEXT:    psrad $24, %xmm2
559; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
560; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
561; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
562; X32-SSE2-NEXT:    psrad $24, %xmm1
563; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
564; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
565; X32-SSE2-NEXT:    retl
566;
567; X32-SSE41-LABEL: sext_16i8_to_4i64:
568; X32-SSE41:       # %bb.0: # %entry
569; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
570; X32-SSE41-NEXT:    psrld $16, %xmm0
571; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
572; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
573; X32-SSE41-NEXT:    retl
574entry:
575  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
576  %C = sext <4 x i8> %B to <4 x i64>
577  ret <4 x i64> %C
578}
579
580define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
581; SSE2-LABEL: sext_16i8_to_8i64:
582; SSE2:       # %bb.0: # %entry
583; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
584; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
585; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
586; SSE2-NEXT:    pxor %xmm5, %xmm5
587; SSE2-NEXT:    pxor %xmm1, %xmm1
588; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
589; SSE2-NEXT:    psrad $24, %xmm4
590; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
591; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
592; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
593; SSE2-NEXT:    pxor %xmm1, %xmm1
594; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
595; SSE2-NEXT:    psrad $24, %xmm2
596; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
597; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
598; SSE2-NEXT:    pxor %xmm0, %xmm0
599; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
600; SSE2-NEXT:    psrad $24, %xmm1
601; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
602; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
603; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
604; SSE2-NEXT:    psrad $24, %xmm3
605; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
606; SSE2-NEXT:    movdqa %xmm4, %xmm0
607; SSE2-NEXT:    retq
608;
609; SSSE3-LABEL: sext_16i8_to_8i64:
610; SSSE3:       # %bb.0: # %entry
611; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
612; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
613; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
614; SSSE3-NEXT:    pxor %xmm5, %xmm5
615; SSSE3-NEXT:    pxor %xmm1, %xmm1
616; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
617; SSSE3-NEXT:    psrad $24, %xmm4
618; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
619; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
620; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
621; SSSE3-NEXT:    pxor %xmm1, %xmm1
622; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
623; SSSE3-NEXT:    psrad $24, %xmm2
624; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
625; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
626; SSSE3-NEXT:    pxor %xmm0, %xmm0
627; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
628; SSSE3-NEXT:    psrad $24, %xmm1
629; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
630; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
631; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
632; SSSE3-NEXT:    psrad $24, %xmm3
633; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
634; SSSE3-NEXT:    movdqa %xmm4, %xmm0
635; SSSE3-NEXT:    retq
636;
637; SSE41-LABEL: sext_16i8_to_8i64:
638; SSE41:       # %bb.0: # %entry
639; SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
640; SSE41-NEXT:    movdqa %xmm0, %xmm1
641; SSE41-NEXT:    psrld $16, %xmm1
642; SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
643; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
644; SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
645; SSE41-NEXT:    psrlq $48, %xmm0
646; SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
647; SSE41-NEXT:    movdqa %xmm4, %xmm0
648; SSE41-NEXT:    retq
649;
650; AVX1-LABEL: sext_16i8_to_8i64:
651; AVX1:       # %bb.0: # %entry
652; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
653; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
654; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
655; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
656; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
657; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
658; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
659; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
660; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
661; AVX1-NEXT:    vmovaps %ymm2, %ymm0
662; AVX1-NEXT:    retq
663;
664; AVX2-LABEL: sext_16i8_to_8i64:
665; AVX2:       # %bb.0: # %entry
666; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm2
667; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
668; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm1
669; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
670; AVX2-NEXT:    retq
671;
672; AVX512-LABEL: sext_16i8_to_8i64:
673; AVX512:       # %bb.0: # %entry
674; AVX512-NEXT:    vpmovsxbq %xmm0, %zmm0
675; AVX512-NEXT:    retq
676;
677; X32-SSE2-LABEL: sext_16i8_to_8i64:
678; X32-SSE2:       # %bb.0: # %entry
679; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
680; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
681; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
682; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
683; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
684; X32-SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
685; X32-SSE2-NEXT:    psrad $24, %xmm4
686; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
687; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
688; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
689; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
690; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
691; X32-SSE2-NEXT:    psrad $24, %xmm2
692; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
693; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
694; X32-SSE2-NEXT:    pxor %xmm0, %xmm0
695; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
696; X32-SSE2-NEXT:    psrad $24, %xmm1
697; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
698; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
699; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
700; X32-SSE2-NEXT:    psrad $24, %xmm3
701; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
702; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
703; X32-SSE2-NEXT:    retl
704;
705; X32-SSE41-LABEL: sext_16i8_to_8i64:
706; X32-SSE41:       # %bb.0: # %entry
707; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
708; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
709; X32-SSE41-NEXT:    psrld $16, %xmm1
710; X32-SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
711; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
712; X32-SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
713; X32-SSE41-NEXT:    psrlq $48, %xmm0
714; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
715; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
716; X32-SSE41-NEXT:    retl
717entry:
718  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
719  %C = sext <8 x i8> %B to <8 x i64>
720  ret <8 x i64> %C
721}
722
723define <4 x i32> @sext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
724; SSE2-LABEL: sext_8i16_to_4i32:
725; SSE2:       # %bb.0: # %entry
726; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
727; SSE2-NEXT:    psrad $16, %xmm0
728; SSE2-NEXT:    retq
729;
730; SSSE3-LABEL: sext_8i16_to_4i32:
731; SSSE3:       # %bb.0: # %entry
732; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
733; SSSE3-NEXT:    psrad $16, %xmm0
734; SSSE3-NEXT:    retq
735;
736; SSE41-LABEL: sext_8i16_to_4i32:
737; SSE41:       # %bb.0: # %entry
738; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
739; SSE41-NEXT:    retq
740;
741; AVX-LABEL: sext_8i16_to_4i32:
742; AVX:       # %bb.0: # %entry
743; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
744; AVX-NEXT:    retq
745;
746; X32-SSE2-LABEL: sext_8i16_to_4i32:
747; X32-SSE2:       # %bb.0: # %entry
748; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
749; X32-SSE2-NEXT:    psrad $16, %xmm0
750; X32-SSE2-NEXT:    retl
751;
752; X32-SSE41-LABEL: sext_8i16_to_4i32:
753; X32-SSE41:       # %bb.0: # %entry
754; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
755; X32-SSE41-NEXT:    retl
756entry:
757  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
758  %C = sext <4 x i16> %B to <4 x i32>
759  ret <4 x i32> %C
760}
761
762define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
763; SSE2-LABEL: sext_8i16_to_8i32:
764; SSE2:       # %bb.0: # %entry
765; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
766; SSE2-NEXT:    psrad $16, %xmm2
767; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
768; SSE2-NEXT:    psrad $16, %xmm1
769; SSE2-NEXT:    movdqa %xmm2, %xmm0
770; SSE2-NEXT:    retq
771;
772; SSSE3-LABEL: sext_8i16_to_8i32:
773; SSSE3:       # %bb.0: # %entry
774; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
775; SSSE3-NEXT:    psrad $16, %xmm2
776; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
777; SSSE3-NEXT:    psrad $16, %xmm1
778; SSSE3-NEXT:    movdqa %xmm2, %xmm0
779; SSSE3-NEXT:    retq
780;
781; SSE41-LABEL: sext_8i16_to_8i32:
782; SSE41:       # %bb.0: # %entry
783; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
784; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
785; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
786; SSE41-NEXT:    movdqa %xmm2, %xmm0
787; SSE41-NEXT:    retq
788;
789; AVX1-LABEL: sext_8i16_to_8i32:
790; AVX1:       # %bb.0: # %entry
791; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
792; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
793; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
794; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
795; AVX1-NEXT:    retq
796;
797; AVX2-LABEL: sext_8i16_to_8i32:
798; AVX2:       # %bb.0: # %entry
799; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
800; AVX2-NEXT:    retq
801;
802; AVX512-LABEL: sext_8i16_to_8i32:
803; AVX512:       # %bb.0: # %entry
804; AVX512-NEXT:    vpmovsxwd %xmm0, %ymm0
805; AVX512-NEXT:    retq
806;
807; X32-SSE2-LABEL: sext_8i16_to_8i32:
808; X32-SSE2:       # %bb.0: # %entry
809; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
810; X32-SSE2-NEXT:    psrad $16, %xmm2
811; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
812; X32-SSE2-NEXT:    psrad $16, %xmm1
813; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
814; X32-SSE2-NEXT:    retl
815;
816; X32-SSE41-LABEL: sext_8i16_to_8i32:
817; X32-SSE41:       # %bb.0: # %entry
818; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
819; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
820; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
821; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
822; X32-SSE41-NEXT:    retl
823entry:
824  %B = sext <8 x i16> %A to <8 x i32>
825  ret <8 x i32> %B
826}
827
828define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone ssp {
829; SSE2-LABEL: sext_16i16_to_16i32:
830; SSE2:       # %bb.0: # %entry
831; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
832; SSE2-NEXT:    psrad $16, %xmm4
833; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
834; SSE2-NEXT:    psrad $16, %xmm5
835; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
836; SSE2-NEXT:    psrad $16, %xmm2
837; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
838; SSE2-NEXT:    psrad $16, %xmm3
839; SSE2-NEXT:    movdqa %xmm4, %xmm0
840; SSE2-NEXT:    movdqa %xmm5, %xmm1
841; SSE2-NEXT:    retq
842;
843; SSSE3-LABEL: sext_16i16_to_16i32:
844; SSSE3:       # %bb.0: # %entry
845; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
846; SSSE3-NEXT:    psrad $16, %xmm4
847; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
848; SSSE3-NEXT:    psrad $16, %xmm5
849; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
850; SSSE3-NEXT:    psrad $16, %xmm2
851; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
852; SSSE3-NEXT:    psrad $16, %xmm3
853; SSSE3-NEXT:    movdqa %xmm4, %xmm0
854; SSSE3-NEXT:    movdqa %xmm5, %xmm1
855; SSSE3-NEXT:    retq
856;
857; SSE41-LABEL: sext_16i16_to_16i32:
858; SSE41:       # %bb.0: # %entry
859; SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
860; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
861; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
862; SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
863; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
864; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
865; SSE41-NEXT:    movdqa %xmm5, %xmm0
866; SSE41-NEXT:    movdqa %xmm4, %xmm1
867; SSE41-NEXT:    retq
868;
869; AVX1-LABEL: sext_16i16_to_16i32:
870; AVX1:       # %bb.0: # %entry
871; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
872; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
873; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
874; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
875; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
876; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
877; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
878; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
879; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
880; AVX1-NEXT:    vmovaps %ymm2, %ymm0
881; AVX1-NEXT:    retq
882;
883; AVX2-LABEL: sext_16i16_to_16i32:
884; AVX2:       # %bb.0: # %entry
885; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm2
886; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
887; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm1
888; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
889; AVX2-NEXT:    retq
890;
891; AVX512-LABEL: sext_16i16_to_16i32:
892; AVX512:       # %bb.0: # %entry
893; AVX512-NEXT:    vpmovsxwd %ymm0, %zmm0
894; AVX512-NEXT:    retq
895;
896; X32-SSE2-LABEL: sext_16i16_to_16i32:
897; X32-SSE2:       # %bb.0: # %entry
898; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
899; X32-SSE2-NEXT:    psrad $16, %xmm4
900; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
901; X32-SSE2-NEXT:    psrad $16, %xmm5
902; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
903; X32-SSE2-NEXT:    psrad $16, %xmm2
904; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
905; X32-SSE2-NEXT:    psrad $16, %xmm3
906; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
907; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
908; X32-SSE2-NEXT:    retl
909;
910; X32-SSE41-LABEL: sext_16i16_to_16i32:
911; X32-SSE41:       # %bb.0: # %entry
912; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
913; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
914; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
915; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
916; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
917; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
918; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
919; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
920; X32-SSE41-NEXT:    retl
921entry:
922  %B = sext <16 x i16> %A to <16 x i32>
923  ret <16 x i32> %B
924}
925
926define <2 x i64> @sext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
927; SSE2-LABEL: sext_8i16_to_2i64:
928; SSE2:       # %bb.0: # %entry
929; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
930; SSE2-NEXT:    pxor %xmm1, %xmm1
931; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
932; SSE2-NEXT:    psrad $16, %xmm0
933; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
934; SSE2-NEXT:    retq
935;
936; SSSE3-LABEL: sext_8i16_to_2i64:
937; SSSE3:       # %bb.0: # %entry
938; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
939; SSSE3-NEXT:    pxor %xmm1, %xmm1
940; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
941; SSSE3-NEXT:    psrad $16, %xmm0
942; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
943; SSSE3-NEXT:    retq
944;
945; SSE41-LABEL: sext_8i16_to_2i64:
946; SSE41:       # %bb.0: # %entry
947; SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
948; SSE41-NEXT:    retq
949;
950; AVX-LABEL: sext_8i16_to_2i64:
951; AVX:       # %bb.0: # %entry
952; AVX-NEXT:    vpmovsxwq %xmm0, %xmm0
953; AVX-NEXT:    retq
954;
955; X32-SSE2-LABEL: sext_8i16_to_2i64:
956; X32-SSE2:       # %bb.0: # %entry
957; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
958; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
959; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
960; X32-SSE2-NEXT:    psrad $16, %xmm0
961; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
962; X32-SSE2-NEXT:    retl
963;
964; X32-SSE41-LABEL: sext_8i16_to_2i64:
965; X32-SSE41:       # %bb.0: # %entry
966; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
967; X32-SSE41-NEXT:    retl
968entry:
969  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
970  %C = sext <2 x i16> %B to <2 x i64>
971  ret <2 x i64> %C
972}
973
974define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
975; SSE2-LABEL: sext_8i16_to_4i64:
976; SSE2:       # %bb.0: # %entry
977; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
978; SSE2-NEXT:    pxor %xmm3, %xmm3
979; SSE2-NEXT:    pxor %xmm1, %xmm1
980; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
981; SSE2-NEXT:    psrad $16, %xmm2
982; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
983; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
984; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
985; SSE2-NEXT:    psrad $16, %xmm1
986; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
987; SSE2-NEXT:    movdqa %xmm2, %xmm0
988; SSE2-NEXT:    retq
989;
990; SSSE3-LABEL: sext_8i16_to_4i64:
991; SSSE3:       # %bb.0: # %entry
992; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
993; SSSE3-NEXT:    pxor %xmm3, %xmm3
994; SSSE3-NEXT:    pxor %xmm1, %xmm1
995; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
996; SSSE3-NEXT:    psrad $16, %xmm2
997; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
998; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
999; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
1000; SSSE3-NEXT:    psrad $16, %xmm1
1001; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1002; SSSE3-NEXT:    movdqa %xmm2, %xmm0
1003; SSSE3-NEXT:    retq
1004;
1005; SSE41-LABEL: sext_8i16_to_4i64:
1006; SSE41:       # %bb.0: # %entry
1007; SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
1008; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1009; SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
1010; SSE41-NEXT:    movdqa %xmm2, %xmm0
1011; SSE41-NEXT:    retq
1012;
1013; AVX1-LABEL: sext_8i16_to_4i64:
1014; AVX1:       # %bb.0: # %entry
1015; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
1016; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1017; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
1018; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1019; AVX1-NEXT:    retq
1020;
1021; AVX2-LABEL: sext_8i16_to_4i64:
1022; AVX2:       # %bb.0: # %entry
1023; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
1024; AVX2-NEXT:    retq
1025;
1026; AVX512-LABEL: sext_8i16_to_4i64:
1027; AVX512:       # %bb.0: # %entry
1028; AVX512-NEXT:    vpmovsxwq %xmm0, %ymm0
1029; AVX512-NEXT:    retq
1030;
1031; X32-SSE2-LABEL: sext_8i16_to_4i64:
1032; X32-SSE2:       # %bb.0: # %entry
1033; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
1034; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1035; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1036; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
1037; X32-SSE2-NEXT:    psrad $16, %xmm2
1038; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1039; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
1040; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1041; X32-SSE2-NEXT:    psrad $16, %xmm1
1042; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1043; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
1044; X32-SSE2-NEXT:    retl
1045;
1046; X32-SSE41-LABEL: sext_8i16_to_4i64:
1047; X32-SSE41:       # %bb.0: # %entry
1048; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
1049; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1050; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
1051; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
1052; X32-SSE41-NEXT:    retl
1053entry:
1054  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1055  %C = sext <4 x i16> %B to <4 x i64>
1056  ret <4 x i64> %C
1057}
1058
1059define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp {
1060; SSE2-LABEL: sext_8i16_to_8i64:
1061; SSE2:       # %bb.0: # %entry
1062; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
1063; SSE2-NEXT:    pxor %xmm5, %xmm5
1064; SSE2-NEXT:    pxor %xmm1, %xmm1
1065; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
1066; SSE2-NEXT:    psrad $16, %xmm4
1067; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1068; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1069; SSE2-NEXT:    pxor %xmm1, %xmm1
1070; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
1071; SSE2-NEXT:    psrad $16, %xmm2
1072; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1073; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
1074; SSE2-NEXT:    pxor %xmm3, %xmm3
1075; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1076; SSE2-NEXT:    psrad $16, %xmm1
1077; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1078; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1079; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
1080; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1081; SSE2-NEXT:    psrad $16, %xmm3
1082; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
1083; SSE2-NEXT:    movdqa %xmm4, %xmm0
1084; SSE2-NEXT:    retq
1085;
1086; SSSE3-LABEL: sext_8i16_to_8i64:
1087; SSSE3:       # %bb.0: # %entry
1088; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
1089; SSSE3-NEXT:    pxor %xmm5, %xmm5
1090; SSSE3-NEXT:    pxor %xmm1, %xmm1
1091; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
1092; SSSE3-NEXT:    psrad $16, %xmm4
1093; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1094; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1095; SSSE3-NEXT:    pxor %xmm1, %xmm1
1096; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
1097; SSSE3-NEXT:    psrad $16, %xmm2
1098; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1099; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
1100; SSSE3-NEXT:    pxor %xmm3, %xmm3
1101; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
1102; SSSE3-NEXT:    psrad $16, %xmm1
1103; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1104; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1105; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
1106; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
1107; SSSE3-NEXT:    psrad $16, %xmm3
1108; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
1109; SSSE3-NEXT:    movdqa %xmm4, %xmm0
1110; SSSE3-NEXT:    retq
1111;
1112; SSE41-LABEL: sext_8i16_to_8i64:
1113; SSE41:       # %bb.0: # %entry
1114; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1115; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1116; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1117; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1118; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1119; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1120; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1121; SSE41-NEXT:    movdqa %xmm4, %xmm0
1122; SSE41-NEXT:    retq
1123;
1124; AVX1-LABEL: sext_8i16_to_8i64:
1125; AVX1:       # %bb.0: # %entry
1126; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
1127; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
1128; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
1129; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1130; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1131; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
1132; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
1133; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
1134; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1135; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1136; AVX1-NEXT:    retq
1137;
1138; AVX2-LABEL: sext_8i16_to_8i64:
1139; AVX2:       # %bb.0: # %entry
1140; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm2
1141; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1142; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm1
1143; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1144; AVX2-NEXT:    retq
1145;
1146; AVX512-LABEL: sext_8i16_to_8i64:
1147; AVX512:       # %bb.0: # %entry
1148; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
1149; AVX512-NEXT:    retq
1150;
1151; X32-SSE2-LABEL: sext_8i16_to_8i64:
1152; X32-SSE2:       # %bb.0: # %entry
1153; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
1154; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1155; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1156; X32-SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
1157; X32-SSE2-NEXT:    psrad $16, %xmm4
1158; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1159; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1160; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1161; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
1162; X32-SSE2-NEXT:    psrad $16, %xmm2
1163; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1164; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
1165; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1166; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1167; X32-SSE2-NEXT:    psrad $16, %xmm1
1168; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1169; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1170; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
1171; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1172; X32-SSE2-NEXT:    psrad $16, %xmm3
1173; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
1174; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
1175; X32-SSE2-NEXT:    retl
1176;
1177; X32-SSE41-LABEL: sext_8i16_to_8i64:
1178; X32-SSE41:       # %bb.0: # %entry
1179; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1180; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1181; X32-SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1182; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1183; X32-SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1184; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1185; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1186; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
1187; X32-SSE41-NEXT:    retl
1188entry:
1189  %B = sext <8 x i16> %A to <8 x i64>
1190  ret <8 x i64> %B
1191}
1192
1193define <2 x i64> @sext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1194; SSE2-LABEL: sext_4i32_to_2i64:
1195; SSE2:       # %bb.0: # %entry
1196; SSE2-NEXT:    pxor %xmm1, %xmm1
1197; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1198; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1199; SSE2-NEXT:    retq
1200;
1201; SSSE3-LABEL: sext_4i32_to_2i64:
1202; SSSE3:       # %bb.0: # %entry
1203; SSSE3-NEXT:    pxor %xmm1, %xmm1
1204; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1205; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1206; SSSE3-NEXT:    retq
1207;
1208; SSE41-LABEL: sext_4i32_to_2i64:
1209; SSE41:       # %bb.0: # %entry
1210; SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1211; SSE41-NEXT:    retq
1212;
1213; AVX-LABEL: sext_4i32_to_2i64:
1214; AVX:       # %bb.0: # %entry
1215; AVX-NEXT:    vpmovsxdq %xmm0, %xmm0
1216; AVX-NEXT:    retq
1217;
1218; X32-SSE2-LABEL: sext_4i32_to_2i64:
1219; X32-SSE2:       # %bb.0: # %entry
1220; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1221; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1222; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1223; X32-SSE2-NEXT:    retl
1224;
1225; X32-SSE41-LABEL: sext_4i32_to_2i64:
1226; X32-SSE41:       # %bb.0: # %entry
1227; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1228; X32-SSE41-NEXT:    retl
1229entry:
1230  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
1231  %C = sext <2 x i32> %B to <2 x i64>
1232  ret <2 x i64> %C
1233}
1234
1235define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1236; SSE2-LABEL: sext_4i32_to_4i64:
1237; SSE2:       # %bb.0: # %entry
1238; SSE2-NEXT:    pxor %xmm2, %xmm2
1239; SSE2-NEXT:    pxor %xmm3, %xmm3
1240; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1241; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1242; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1243; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1244; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1245; SSE2-NEXT:    retq
1246;
1247; SSSE3-LABEL: sext_4i32_to_4i64:
1248; SSSE3:       # %bb.0: # %entry
1249; SSSE3-NEXT:    pxor %xmm2, %xmm2
1250; SSSE3-NEXT:    pxor %xmm3, %xmm3
1251; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1252; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1253; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1254; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1255; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1256; SSSE3-NEXT:    retq
1257;
1258; SSE41-LABEL: sext_4i32_to_4i64:
1259; SSE41:       # %bb.0: # %entry
1260; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1261; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1262; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1263; SSE41-NEXT:    movdqa %xmm2, %xmm0
1264; SSE41-NEXT:    retq
1265;
1266; AVX1-LABEL: sext_4i32_to_4i64:
1267; AVX1:       # %bb.0: # %entry
1268; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1269; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1270; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1271; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1272; AVX1-NEXT:    retq
1273;
1274; AVX2-LABEL: sext_4i32_to_4i64:
1275; AVX2:       # %bb.0: # %entry
1276; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
1277; AVX2-NEXT:    retq
1278;
1279; AVX512-LABEL: sext_4i32_to_4i64:
1280; AVX512:       # %bb.0: # %entry
1281; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
1282; AVX512-NEXT:    retq
1283;
1284; X32-SSE2-LABEL: sext_4i32_to_4i64:
1285; X32-SSE2:       # %bb.0: # %entry
1286; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1287; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1288; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1289; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1290; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1291; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1292; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1293; X32-SSE2-NEXT:    retl
1294;
1295; X32-SSE41-LABEL: sext_4i32_to_4i64:
1296; X32-SSE41:       # %bb.0: # %entry
1297; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1298; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1299; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1300; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
1301; X32-SSE41-NEXT:    retl
1302entry:
1303  %B = sext <4 x i32> %A to <4 x i64>
1304  ret <4 x i64> %B
1305}
1306
1307define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp {
1308; SSE2-LABEL: sext_8i32_to_8i64:
1309; SSE2:       # %bb.0: # %entry
1310; SSE2-NEXT:    movdqa %xmm1, %xmm2
1311; SSE2-NEXT:    pxor %xmm4, %xmm4
1312; SSE2-NEXT:    pxor %xmm3, %xmm3
1313; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1314; SSE2-NEXT:    pxor %xmm5, %xmm5
1315; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1316; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1317; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1318; SSE2-NEXT:    pxor %xmm3, %xmm3
1319; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1320; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1321; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1322; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1323; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1324; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1325; SSE2-NEXT:    retq
1326;
1327; SSSE3-LABEL: sext_8i32_to_8i64:
1328; SSSE3:       # %bb.0: # %entry
1329; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1330; SSSE3-NEXT:    pxor %xmm4, %xmm4
1331; SSSE3-NEXT:    pxor %xmm3, %xmm3
1332; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1333; SSSE3-NEXT:    pxor %xmm5, %xmm5
1334; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
1335; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1336; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1337; SSSE3-NEXT:    pxor %xmm3, %xmm3
1338; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
1339; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1340; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1341; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1342; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
1343; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1344; SSSE3-NEXT:    retq
1345;
1346; SSE41-LABEL: sext_8i32_to_8i64:
1347; SSE41:       # %bb.0: # %entry
1348; SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1349; SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1350; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1351; SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1352; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1353; SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1354; SSE41-NEXT:    movdqa %xmm5, %xmm0
1355; SSE41-NEXT:    movdqa %xmm4, %xmm1
1356; SSE41-NEXT:    retq
1357;
1358; AVX1-LABEL: sext_8i32_to_8i64:
1359; AVX1:       # %bb.0: # %entry
1360; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1361; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1362; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
1363; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1364; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1365; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1366; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1367; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1368; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1369; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1370; AVX1-NEXT:    retq
1371;
1372; AVX2-LABEL: sext_8i32_to_8i64:
1373; AVX2:       # %bb.0: # %entry
1374; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm2
1375; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1376; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm1
1377; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1378; AVX2-NEXT:    retq
1379;
1380; AVX512-LABEL: sext_8i32_to_8i64:
1381; AVX512:       # %bb.0: # %entry
1382; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
1383; AVX512-NEXT:    retq
1384;
1385; X32-SSE2-LABEL: sext_8i32_to_8i64:
1386; X32-SSE2:       # %bb.0: # %entry
1387; X32-SSE2-NEXT:    movdqa %xmm1, %xmm2
1388; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
1389; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1390; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1391; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1392; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1393; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1394; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1395; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1396; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1397; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1398; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1399; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1400; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1401; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1402; X32-SSE2-NEXT:    retl
1403;
1404; X32-SSE41-LABEL: sext_8i32_to_8i64:
1405; X32-SSE41:       # %bb.0: # %entry
1406; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1407; X32-SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1408; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1409; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1410; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1411; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1412; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
1413; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
1414; X32-SSE41-NEXT:    retl
1415entry:
1416  %B = sext <8 x i32> %A to <8 x i64>
1417  ret <8 x i64> %B
1418}
1419
1420define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) {
1421; SSE-LABEL: load_sext_2i1_to_2i64:
1422; SSE:       # %bb.0: # %entry
1423; SSE-NEXT:    movzbl (%rdi), %eax
1424; SSE-NEXT:    movq %rax, %rcx
1425; SSE-NEXT:    shlq $62, %rcx
1426; SSE-NEXT:    sarq $63, %rcx
1427; SSE-NEXT:    movq %rcx, %xmm1
1428; SSE-NEXT:    shlq $63, %rax
1429; SSE-NEXT:    sarq $63, %rax
1430; SSE-NEXT:    movq %rax, %xmm0
1431; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1432; SSE-NEXT:    retq
1433;
1434; AVX1-LABEL: load_sext_2i1_to_2i64:
1435; AVX1:       # %bb.0: # %entry
1436; AVX1-NEXT:    movzbl (%rdi), %eax
1437; AVX1-NEXT:    movq %rax, %rcx
1438; AVX1-NEXT:    shlq $62, %rcx
1439; AVX1-NEXT:    sarq $63, %rcx
1440; AVX1-NEXT:    vmovq %rcx, %xmm0
1441; AVX1-NEXT:    shlq $63, %rax
1442; AVX1-NEXT:    sarq $63, %rax
1443; AVX1-NEXT:    vmovq %rax, %xmm1
1444; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1445; AVX1-NEXT:    retq
1446;
1447; AVX2-LABEL: load_sext_2i1_to_2i64:
1448; AVX2:       # %bb.0: # %entry
1449; AVX2-NEXT:    movzbl (%rdi), %eax
1450; AVX2-NEXT:    movq %rax, %rcx
1451; AVX2-NEXT:    shlq $62, %rcx
1452; AVX2-NEXT:    sarq $63, %rcx
1453; AVX2-NEXT:    vmovq %rcx, %xmm0
1454; AVX2-NEXT:    shlq $63, %rax
1455; AVX2-NEXT:    sarq $63, %rax
1456; AVX2-NEXT:    vmovq %rax, %xmm1
1457; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1458; AVX2-NEXT:    retq
1459;
1460; AVX512F-LABEL: load_sext_2i1_to_2i64:
1461; AVX512F:       # %bb.0: # %entry
1462; AVX512F-NEXT:    movzbl (%rdi), %eax
1463; AVX512F-NEXT:    kmovw %eax, %k1
1464; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1465; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1466; AVX512F-NEXT:    vzeroupper
1467; AVX512F-NEXT:    retq
1468;
1469; AVX512BW-LABEL: load_sext_2i1_to_2i64:
1470; AVX512BW:       # %bb.0: # %entry
1471; AVX512BW-NEXT:    movzbl (%rdi), %eax
1472; AVX512BW-NEXT:    kmovd %eax, %k1
1473; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1474; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1475; AVX512BW-NEXT:    vzeroupper
1476; AVX512BW-NEXT:    retq
1477;
1478; X32-SSE2-LABEL: load_sext_2i1_to_2i64:
1479; X32-SSE2:       # %bb.0: # %entry
1480; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1481; X32-SSE2-NEXT:    movzbl (%eax), %eax
1482; X32-SSE2-NEXT:    movl %eax, %ecx
1483; X32-SSE2-NEXT:    shll $30, %ecx
1484; X32-SSE2-NEXT:    sarl $31, %ecx
1485; X32-SSE2-NEXT:    movd %ecx, %xmm0
1486; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]
1487; X32-SSE2-NEXT:    shll $31, %eax
1488; X32-SSE2-NEXT:    sarl $31, %eax
1489; X32-SSE2-NEXT:    movd %eax, %xmm0
1490; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
1491; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1492; X32-SSE2-NEXT:    retl
1493;
1494; X32-SSE41-LABEL: load_sext_2i1_to_2i64:
1495; X32-SSE41:       # %bb.0: # %entry
1496; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1497; X32-SSE41-NEXT:    movzbl (%eax), %eax
1498; X32-SSE41-NEXT:    movl %eax, %ecx
1499; X32-SSE41-NEXT:    shll $31, %ecx
1500; X32-SSE41-NEXT:    sarl $31, %ecx
1501; X32-SSE41-NEXT:    movd %ecx, %xmm0
1502; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1503; X32-SSE41-NEXT:    shll $30, %eax
1504; X32-SSE41-NEXT:    sarl $31, %eax
1505; X32-SSE41-NEXT:    pinsrd $2, %eax, %xmm0
1506; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1507; X32-SSE41-NEXT:    retl
1508entry:
1509 %X = load <2 x i1>, <2 x i1>* %ptr
1510 %Y = sext <2 x i1> %X to <2 x i64>
1511 ret <2 x i64> %Y
1512}
1513
1514define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) {
1515; SSE2-LABEL: load_sext_2i8_to_2i64:
1516; SSE2:       # %bb.0: # %entry
1517; SSE2-NEXT:    movzwl (%rdi), %eax
1518; SSE2-NEXT:    movd %eax, %xmm0
1519; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1520; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1521; SSE2-NEXT:    pxor %xmm1, %xmm1
1522; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1523; SSE2-NEXT:    psrad $24, %xmm0
1524; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1525; SSE2-NEXT:    retq
1526;
1527; SSSE3-LABEL: load_sext_2i8_to_2i64:
1528; SSSE3:       # %bb.0: # %entry
1529; SSSE3-NEXT:    movzwl (%rdi), %eax
1530; SSSE3-NEXT:    movd %eax, %xmm0
1531; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1532; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1533; SSSE3-NEXT:    pxor %xmm1, %xmm1
1534; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1535; SSSE3-NEXT:    psrad $24, %xmm0
1536; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1537; SSSE3-NEXT:    retq
1538;
1539; SSE41-LABEL: load_sext_2i8_to_2i64:
1540; SSE41:       # %bb.0: # %entry
1541; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
1542; SSE41-NEXT:    retq
1543;
1544; AVX-LABEL: load_sext_2i8_to_2i64:
1545; AVX:       # %bb.0: # %entry
1546; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
1547; AVX-NEXT:    retq
1548;
1549; X32-SSE2-LABEL: load_sext_2i8_to_2i64:
1550; X32-SSE2:       # %bb.0: # %entry
1551; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1552; X32-SSE2-NEXT:    movzwl (%eax), %eax
1553; X32-SSE2-NEXT:    movd %eax, %xmm0
1554; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1555; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1556; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1557; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1558; X32-SSE2-NEXT:    psrad $24, %xmm0
1559; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1560; X32-SSE2-NEXT:    retl
1561;
1562; X32-SSE41-LABEL: load_sext_2i8_to_2i64:
1563; X32-SSE41:       # %bb.0: # %entry
1564; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1565; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
1566; X32-SSE41-NEXT:    retl
1567entry:
1568 %X = load <2 x i8>, <2 x i8>* %ptr
1569 %Y = sext <2 x i8> %X to <2 x i64>
1570 ret <2 x i64> %Y
1571}
1572
1573define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) {
1574; SSE2-LABEL: load_sext_4i1_to_4i32:
1575; SSE2:       # %bb.0: # %entry
1576; SSE2-NEXT:    movzbl (%rdi), %eax
1577; SSE2-NEXT:    movq %rax, %rcx
1578; SSE2-NEXT:    shlq $60, %rcx
1579; SSE2-NEXT:    sarq $63, %rcx
1580; SSE2-NEXT:    movd %ecx, %xmm0
1581; SSE2-NEXT:    movq %rax, %rcx
1582; SSE2-NEXT:    shlq $61, %rcx
1583; SSE2-NEXT:    sarq $63, %rcx
1584; SSE2-NEXT:    movd %ecx, %xmm1
1585; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1586; SSE2-NEXT:    movq %rax, %rcx
1587; SSE2-NEXT:    shlq $62, %rcx
1588; SSE2-NEXT:    sarq $63, %rcx
1589; SSE2-NEXT:    movd %ecx, %xmm2
1590; SSE2-NEXT:    shlq $63, %rax
1591; SSE2-NEXT:    sarq $63, %rax
1592; SSE2-NEXT:    movd %eax, %xmm0
1593; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1594; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1595; SSE2-NEXT:    retq
1596;
1597; SSSE3-LABEL: load_sext_4i1_to_4i32:
1598; SSSE3:       # %bb.0: # %entry
1599; SSSE3-NEXT:    movzbl (%rdi), %eax
1600; SSSE3-NEXT:    movq %rax, %rcx
1601; SSSE3-NEXT:    shlq $60, %rcx
1602; SSSE3-NEXT:    sarq $63, %rcx
1603; SSSE3-NEXT:    movd %ecx, %xmm0
1604; SSSE3-NEXT:    movq %rax, %rcx
1605; SSSE3-NEXT:    shlq $61, %rcx
1606; SSSE3-NEXT:    sarq $63, %rcx
1607; SSSE3-NEXT:    movd %ecx, %xmm1
1608; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1609; SSSE3-NEXT:    movq %rax, %rcx
1610; SSSE3-NEXT:    shlq $62, %rcx
1611; SSSE3-NEXT:    sarq $63, %rcx
1612; SSSE3-NEXT:    movd %ecx, %xmm2
1613; SSSE3-NEXT:    shlq $63, %rax
1614; SSSE3-NEXT:    sarq $63, %rax
1615; SSSE3-NEXT:    movd %eax, %xmm0
1616; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1617; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1618; SSSE3-NEXT:    retq
1619;
1620; SSE41-LABEL: load_sext_4i1_to_4i32:
1621; SSE41:       # %bb.0: # %entry
1622; SSE41-NEXT:    movzbl (%rdi), %eax
1623; SSE41-NEXT:    movq %rax, %rcx
1624; SSE41-NEXT:    shlq $62, %rcx
1625; SSE41-NEXT:    sarq $63, %rcx
1626; SSE41-NEXT:    movq %rax, %rdx
1627; SSE41-NEXT:    shlq $63, %rdx
1628; SSE41-NEXT:    sarq $63, %rdx
1629; SSE41-NEXT:    movd %edx, %xmm0
1630; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1631; SSE41-NEXT:    movq %rax, %rcx
1632; SSE41-NEXT:    shlq $61, %rcx
1633; SSE41-NEXT:    sarq $63, %rcx
1634; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1635; SSE41-NEXT:    shlq $60, %rax
1636; SSE41-NEXT:    sarq $63, %rax
1637; SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1638; SSE41-NEXT:    retq
1639;
1640; AVX1-LABEL: load_sext_4i1_to_4i32:
1641; AVX1:       # %bb.0: # %entry
1642; AVX1-NEXT:    movzbl (%rdi), %eax
1643; AVX1-NEXT:    movq %rax, %rcx
1644; AVX1-NEXT:    shlq $62, %rcx
1645; AVX1-NEXT:    sarq $63, %rcx
1646; AVX1-NEXT:    movq %rax, %rdx
1647; AVX1-NEXT:    shlq $63, %rdx
1648; AVX1-NEXT:    sarq $63, %rdx
1649; AVX1-NEXT:    vmovd %edx, %xmm0
1650; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1651; AVX1-NEXT:    movq %rax, %rcx
1652; AVX1-NEXT:    shlq $61, %rcx
1653; AVX1-NEXT:    sarq $63, %rcx
1654; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1655; AVX1-NEXT:    shlq $60, %rax
1656; AVX1-NEXT:    sarq $63, %rax
1657; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1658; AVX1-NEXT:    retq
1659;
1660; AVX2-LABEL: load_sext_4i1_to_4i32:
1661; AVX2:       # %bb.0: # %entry
1662; AVX2-NEXT:    movzbl (%rdi), %eax
1663; AVX2-NEXT:    movq %rax, %rcx
1664; AVX2-NEXT:    shlq $62, %rcx
1665; AVX2-NEXT:    sarq $63, %rcx
1666; AVX2-NEXT:    movq %rax, %rdx
1667; AVX2-NEXT:    shlq $63, %rdx
1668; AVX2-NEXT:    sarq $63, %rdx
1669; AVX2-NEXT:    vmovd %edx, %xmm0
1670; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1671; AVX2-NEXT:    movq %rax, %rcx
1672; AVX2-NEXT:    shlq $61, %rcx
1673; AVX2-NEXT:    sarq $63, %rcx
1674; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1675; AVX2-NEXT:    shlq $60, %rax
1676; AVX2-NEXT:    sarq $63, %rax
1677; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1678; AVX2-NEXT:    retq
1679;
1680; AVX512F-LABEL: load_sext_4i1_to_4i32:
1681; AVX512F:       # %bb.0: # %entry
1682; AVX512F-NEXT:    movzbl (%rdi), %eax
1683; AVX512F-NEXT:    kmovw %eax, %k1
1684; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1685; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1686; AVX512F-NEXT:    vzeroupper
1687; AVX512F-NEXT:    retq
1688;
1689; AVX512BW-LABEL: load_sext_4i1_to_4i32:
1690; AVX512BW:       # %bb.0: # %entry
1691; AVX512BW-NEXT:    movzbl (%rdi), %eax
1692; AVX512BW-NEXT:    kmovd %eax, %k1
1693; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1694; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1695; AVX512BW-NEXT:    vzeroupper
1696; AVX512BW-NEXT:    retq
1697;
1698; X32-SSE2-LABEL: load_sext_4i1_to_4i32:
1699; X32-SSE2:       # %bb.0: # %entry
1700; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1701; X32-SSE2-NEXT:    movl (%eax), %eax
1702; X32-SSE2-NEXT:    movl %eax, %ecx
1703; X32-SSE2-NEXT:    shll $28, %ecx
1704; X32-SSE2-NEXT:    sarl $31, %ecx
1705; X32-SSE2-NEXT:    movd %ecx, %xmm0
1706; X32-SSE2-NEXT:    movl %eax, %ecx
1707; X32-SSE2-NEXT:    shll $29, %ecx
1708; X32-SSE2-NEXT:    sarl $31, %ecx
1709; X32-SSE2-NEXT:    movd %ecx, %xmm1
1710; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1711; X32-SSE2-NEXT:    movl %eax, %ecx
1712; X32-SSE2-NEXT:    shll $30, %ecx
1713; X32-SSE2-NEXT:    sarl $31, %ecx
1714; X32-SSE2-NEXT:    movd %ecx, %xmm2
1715; X32-SSE2-NEXT:    shll $31, %eax
1716; X32-SSE2-NEXT:    sarl $31, %eax
1717; X32-SSE2-NEXT:    movd %eax, %xmm0
1718; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1719; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1720; X32-SSE2-NEXT:    retl
1721;
1722; X32-SSE41-LABEL: load_sext_4i1_to_4i32:
1723; X32-SSE41:       # %bb.0: # %entry
1724; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1725; X32-SSE41-NEXT:    movl (%eax), %eax
1726; X32-SSE41-NEXT:    movl %eax, %ecx
1727; X32-SSE41-NEXT:    shll $30, %ecx
1728; X32-SSE41-NEXT:    sarl $31, %ecx
1729; X32-SSE41-NEXT:    movl %eax, %edx
1730; X32-SSE41-NEXT:    shll $31, %edx
1731; X32-SSE41-NEXT:    sarl $31, %edx
1732; X32-SSE41-NEXT:    movd %edx, %xmm0
1733; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1734; X32-SSE41-NEXT:    movl %eax, %ecx
1735; X32-SSE41-NEXT:    shll $29, %ecx
1736; X32-SSE41-NEXT:    sarl $31, %ecx
1737; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1738; X32-SSE41-NEXT:    shll $28, %eax
1739; X32-SSE41-NEXT:    sarl $31, %eax
1740; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1741; X32-SSE41-NEXT:    retl
1742entry:
1743 %X = load <4 x i1>, <4 x i1>* %ptr
1744 %Y = sext <4 x i1> %X to <4 x i32>
1745 ret <4 x i32> %Y
1746}
1747
1748define <4 x i32> @load_sext_4i8_to_4i32(<4 x i8> *%ptr) {
1749; SSE2-LABEL: load_sext_4i8_to_4i32:
1750; SSE2:       # %bb.0: # %entry
1751; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1752; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1753; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1754; SSE2-NEXT:    psrad $24, %xmm0
1755; SSE2-NEXT:    retq
1756;
1757; SSSE3-LABEL: load_sext_4i8_to_4i32:
1758; SSSE3:       # %bb.0: # %entry
1759; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1760; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1761; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1762; SSSE3-NEXT:    psrad $24, %xmm0
1763; SSSE3-NEXT:    retq
1764;
1765; SSE41-LABEL: load_sext_4i8_to_4i32:
1766; SSE41:       # %bb.0: # %entry
1767; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
1768; SSE41-NEXT:    retq
1769;
1770; AVX-LABEL: load_sext_4i8_to_4i32:
1771; AVX:       # %bb.0: # %entry
1772; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
1773; AVX-NEXT:    retq
1774;
1775; X32-SSE2-LABEL: load_sext_4i8_to_4i32:
1776; X32-SSE2:       # %bb.0: # %entry
1777; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1778; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1779; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1780; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1781; X32-SSE2-NEXT:    psrad $24, %xmm0
1782; X32-SSE2-NEXT:    retl
1783;
1784; X32-SSE41-LABEL: load_sext_4i8_to_4i32:
1785; X32-SSE41:       # %bb.0: # %entry
1786; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1787; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
1788; X32-SSE41-NEXT:    retl
1789entry:
1790 %X = load <4 x i8>, <4 x i8>* %ptr
1791 %Y = sext <4 x i8> %X to <4 x i32>
1792 ret <4 x i32> %Y
1793}
1794
1795define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
1796; SSE2-LABEL: load_sext_4i1_to_4i64:
1797; SSE2:       # %bb.0: # %entry
1798; SSE2-NEXT:    movl (%rdi), %eax
1799; SSE2-NEXT:    movl %eax, %ecx
1800; SSE2-NEXT:    shrl $3, %ecx
1801; SSE2-NEXT:    movd %ecx, %xmm0
1802; SSE2-NEXT:    movl %eax, %ecx
1803; SSE2-NEXT:    shrl $2, %ecx
1804; SSE2-NEXT:    movd %ecx, %xmm1
1805; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1806; SSE2-NEXT:    movd %eax, %xmm2
1807; SSE2-NEXT:    shrl %eax
1808; SSE2-NEXT:    movd %eax, %xmm0
1809; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1810; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1811; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
1812; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1813; SSE2-NEXT:    psllq $63, %xmm0
1814; SSE2-NEXT:    psrad $31, %xmm0
1815; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1816; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1817; SSE2-NEXT:    psllq $63, %xmm1
1818; SSE2-NEXT:    psrad $31, %xmm1
1819; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1820; SSE2-NEXT:    retq
1821;
1822; SSSE3-LABEL: load_sext_4i1_to_4i64:
1823; SSSE3:       # %bb.0: # %entry
1824; SSSE3-NEXT:    movl (%rdi), %eax
1825; SSSE3-NEXT:    movl %eax, %ecx
1826; SSSE3-NEXT:    shrl $3, %ecx
1827; SSSE3-NEXT:    movd %ecx, %xmm0
1828; SSSE3-NEXT:    movl %eax, %ecx
1829; SSSE3-NEXT:    shrl $2, %ecx
1830; SSSE3-NEXT:    movd %ecx, %xmm1
1831; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1832; SSSE3-NEXT:    movd %eax, %xmm2
1833; SSSE3-NEXT:    shrl %eax
1834; SSSE3-NEXT:    movd %eax, %xmm0
1835; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1836; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1837; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
1838; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1839; SSSE3-NEXT:    psllq $63, %xmm0
1840; SSSE3-NEXT:    psrad $31, %xmm0
1841; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1842; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1843; SSSE3-NEXT:    psllq $63, %xmm1
1844; SSSE3-NEXT:    psrad $31, %xmm1
1845; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1846; SSSE3-NEXT:    retq
1847;
1848; SSE41-LABEL: load_sext_4i1_to_4i64:
1849; SSE41:       # %bb.0: # %entry
1850; SSE41-NEXT:    movl (%rdi), %eax
1851; SSE41-NEXT:    movl %eax, %ecx
1852; SSE41-NEXT:    shrl %ecx
1853; SSE41-NEXT:    movd %eax, %xmm1
1854; SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1855; SSE41-NEXT:    movl %eax, %ecx
1856; SSE41-NEXT:    shrl $2, %ecx
1857; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1858; SSE41-NEXT:    shrl $3, %eax
1859; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1860; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
1861; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1862; SSE41-NEXT:    psllq $63, %xmm0
1863; SSE41-NEXT:    psrad $31, %xmm0
1864; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1865; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1866; SSE41-NEXT:    psllq $63, %xmm1
1867; SSE41-NEXT:    psrad $31, %xmm1
1868; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1869; SSE41-NEXT:    retq
1870;
1871; AVX1-LABEL: load_sext_4i1_to_4i64:
1872; AVX1:       # %bb.0: # %entry
1873; AVX1-NEXT:    movzbl (%rdi), %eax
1874; AVX1-NEXT:    movq %rax, %rcx
1875; AVX1-NEXT:    shlq $62, %rcx
1876; AVX1-NEXT:    sarq $63, %rcx
1877; AVX1-NEXT:    movq %rax, %rdx
1878; AVX1-NEXT:    shlq $63, %rdx
1879; AVX1-NEXT:    sarq $63, %rdx
1880; AVX1-NEXT:    vmovd %edx, %xmm0
1881; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1882; AVX1-NEXT:    movq %rax, %rcx
1883; AVX1-NEXT:    shlq $61, %rcx
1884; AVX1-NEXT:    sarq $63, %rcx
1885; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1886; AVX1-NEXT:    shlq $60, %rax
1887; AVX1-NEXT:    sarq $63, %rax
1888; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1889; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1890; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1891; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1892; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1893; AVX1-NEXT:    retq
1894;
1895; AVX2-LABEL: load_sext_4i1_to_4i64:
1896; AVX2:       # %bb.0: # %entry
1897; AVX2-NEXT:    movzbl (%rdi), %eax
1898; AVX2-NEXT:    movq %rax, %rcx
1899; AVX2-NEXT:    shlq $60, %rcx
1900; AVX2-NEXT:    sarq $63, %rcx
1901; AVX2-NEXT:    vmovq %rcx, %xmm0
1902; AVX2-NEXT:    movq %rax, %rcx
1903; AVX2-NEXT:    shlq $61, %rcx
1904; AVX2-NEXT:    sarq $63, %rcx
1905; AVX2-NEXT:    vmovq %rcx, %xmm1
1906; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1907; AVX2-NEXT:    movq %rax, %rcx
1908; AVX2-NEXT:    shlq $62, %rcx
1909; AVX2-NEXT:    sarq $63, %rcx
1910; AVX2-NEXT:    vmovq %rcx, %xmm1
1911; AVX2-NEXT:    shlq $63, %rax
1912; AVX2-NEXT:    sarq $63, %rax
1913; AVX2-NEXT:    vmovq %rax, %xmm2
1914; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1915; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
1916; AVX2-NEXT:    retq
1917;
1918; AVX512F-LABEL: load_sext_4i1_to_4i64:
1919; AVX512F:       # %bb.0: # %entry
1920; AVX512F-NEXT:    movzbl (%rdi), %eax
1921; AVX512F-NEXT:    kmovw %eax, %k1
1922; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1923; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1924; AVX512F-NEXT:    retq
1925;
1926; AVX512BW-LABEL: load_sext_4i1_to_4i64:
1927; AVX512BW:       # %bb.0: # %entry
1928; AVX512BW-NEXT:    movzbl (%rdi), %eax
1929; AVX512BW-NEXT:    kmovd %eax, %k1
1930; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1931; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1932; AVX512BW-NEXT:    retq
1933;
1934; X32-SSE2-LABEL: load_sext_4i1_to_4i64:
1935; X32-SSE2:       # %bb.0: # %entry
1936; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1937; X32-SSE2-NEXT:    movzbl (%eax), %eax
1938; X32-SSE2-NEXT:    movl %eax, %ecx
1939; X32-SSE2-NEXT:    shrl $3, %ecx
1940; X32-SSE2-NEXT:    movd %ecx, %xmm0
1941; X32-SSE2-NEXT:    movl %eax, %ecx
1942; X32-SSE2-NEXT:    shrl $2, %ecx
1943; X32-SSE2-NEXT:    movd %ecx, %xmm1
1944; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1945; X32-SSE2-NEXT:    movd %eax, %xmm2
1946; X32-SSE2-NEXT:    shrl %eax
1947; X32-SSE2-NEXT:    movd %eax, %xmm0
1948; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1949; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1950; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
1951; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1952; X32-SSE2-NEXT:    psllq $63, %xmm0
1953; X32-SSE2-NEXT:    psrad $31, %xmm0
1954; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1955; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1956; X32-SSE2-NEXT:    psllq $63, %xmm1
1957; X32-SSE2-NEXT:    psrad $31, %xmm1
1958; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1959; X32-SSE2-NEXT:    retl
1960;
1961; X32-SSE41-LABEL: load_sext_4i1_to_4i64:
1962; X32-SSE41:       # %bb.0: # %entry
1963; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1964; X32-SSE41-NEXT:    movzbl (%eax), %eax
1965; X32-SSE41-NEXT:    movl %eax, %ecx
1966; X32-SSE41-NEXT:    shrl %ecx
1967; X32-SSE41-NEXT:    movd %eax, %xmm1
1968; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1969; X32-SSE41-NEXT:    movl %eax, %ecx
1970; X32-SSE41-NEXT:    shrl $2, %ecx
1971; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1972; X32-SSE41-NEXT:    shrl $3, %eax
1973; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1974; X32-SSE41-NEXT:    pand {{\.LCPI.*}}, %xmm1
1975; X32-SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1976; X32-SSE41-NEXT:    psllq $63, %xmm0
1977; X32-SSE41-NEXT:    psrad $31, %xmm0
1978; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1979; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1980; X32-SSE41-NEXT:    psllq $63, %xmm1
1981; X32-SSE41-NEXT:    psrad $31, %xmm1
1982; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1983; X32-SSE41-NEXT:    retl
1984entry:
1985 %X = load <4 x i1>, <4 x i1>* %ptr
1986 %Y = sext <4 x i1> %X to <4 x i64>
1987 ret <4 x i64> %Y
1988}
1989
1990define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
1991; SSE2-LABEL: load_sext_4i8_to_4i64:
1992; SSE2:       # %bb.0: # %entry
1993; SSE2-NEXT:    movsbq 1(%rdi), %rax
1994; SSE2-NEXT:    movq %rax, %xmm1
1995; SSE2-NEXT:    movsbq (%rdi), %rax
1996; SSE2-NEXT:    movq %rax, %xmm0
1997; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1998; SSE2-NEXT:    movsbq 3(%rdi), %rax
1999; SSE2-NEXT:    movq %rax, %xmm2
2000; SSE2-NEXT:    movsbq 2(%rdi), %rax
2001; SSE2-NEXT:    movq %rax, %xmm1
2002; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2003; SSE2-NEXT:    retq
2004;
2005; SSSE3-LABEL: load_sext_4i8_to_4i64:
2006; SSSE3:       # %bb.0: # %entry
2007; SSSE3-NEXT:    movsbq 1(%rdi), %rax
2008; SSSE3-NEXT:    movq %rax, %xmm1
2009; SSSE3-NEXT:    movsbq (%rdi), %rax
2010; SSSE3-NEXT:    movq %rax, %xmm0
2011; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2012; SSSE3-NEXT:    movsbq 3(%rdi), %rax
2013; SSSE3-NEXT:    movq %rax, %xmm2
2014; SSSE3-NEXT:    movsbq 2(%rdi), %rax
2015; SSSE3-NEXT:    movq %rax, %xmm1
2016; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2017; SSSE3-NEXT:    retq
2018;
2019; SSE41-LABEL: load_sext_4i8_to_4i64:
2020; SSE41:       # %bb.0: # %entry
2021; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
2022; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
2023; SSE41-NEXT:    retq
2024;
2025; AVX1-LABEL: load_sext_4i8_to_4i64:
2026; AVX1:       # %bb.0: # %entry
2027; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
2028; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
2029; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2030; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
2031; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2032; AVX1-NEXT:    retq
2033;
2034; AVX2-LABEL: load_sext_4i8_to_4i64:
2035; AVX2:       # %bb.0: # %entry
2036; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2037; AVX2-NEXT:    retq
2038;
2039; AVX512-LABEL: load_sext_4i8_to_4i64:
2040; AVX512:       # %bb.0: # %entry
2041; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
2042; AVX512-NEXT:    retq
2043;
2044; X32-SSE2-LABEL: load_sext_4i8_to_4i64:
2045; X32-SSE2:       # %bb.0: # %entry
2046; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2047; X32-SSE2-NEXT:    movsbl 1(%eax), %ecx
2048; X32-SSE2-NEXT:    movd %ecx, %xmm1
2049; X32-SSE2-NEXT:    sarl $31, %ecx
2050; X32-SSE2-NEXT:    movd %ecx, %xmm0
2051; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2052; X32-SSE2-NEXT:    movsbl (%eax), %ecx
2053; X32-SSE2-NEXT:    movd %ecx, %xmm0
2054; X32-SSE2-NEXT:    sarl $31, %ecx
2055; X32-SSE2-NEXT:    movd %ecx, %xmm2
2056; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2057; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2058; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
2059; X32-SSE2-NEXT:    movd %ecx, %xmm2
2060; X32-SSE2-NEXT:    sarl $31, %ecx
2061; X32-SSE2-NEXT:    movd %ecx, %xmm1
2062; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2063; X32-SSE2-NEXT:    movsbl 2(%eax), %eax
2064; X32-SSE2-NEXT:    movd %eax, %xmm1
2065; X32-SSE2-NEXT:    sarl $31, %eax
2066; X32-SSE2-NEXT:    movd %eax, %xmm3
2067; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
2068; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2069; X32-SSE2-NEXT:    retl
2070;
2071; X32-SSE41-LABEL: load_sext_4i8_to_4i64:
2072; X32-SSE41:       # %bb.0: # %entry
2073; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2074; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
2075; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
2076; X32-SSE41-NEXT:    retl
2077entry:
2078 %X = load <4 x i8>, <4 x i8>* %ptr
2079 %Y = sext <4 x i8> %X to <4 x i64>
2080 ret <4 x i64> %Y
2081}
2082
2083define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
2084; SSE2-LABEL: load_sext_4i8_to_4i64_extract:
2085; SSE2:       # %bb.0:
2086; SSE2-NEXT:    movsbq 3(%rdi), %rax
2087; SSE2-NEXT:    movq %rax, %xmm1
2088; SSE2-NEXT:    movsbq 2(%rdi), %rax
2089; SSE2-NEXT:    movq %rax, %xmm0
2090; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2091; SSE2-NEXT:    retq
2092;
2093; SSSE3-LABEL: load_sext_4i8_to_4i64_extract:
2094; SSSE3:       # %bb.0:
2095; SSSE3-NEXT:    movsbq 3(%rdi), %rax
2096; SSSE3-NEXT:    movq %rax, %xmm1
2097; SSSE3-NEXT:    movsbq 2(%rdi), %rax
2098; SSSE3-NEXT:    movq %rax, %xmm0
2099; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2100; SSSE3-NEXT:    retq
2101;
2102; SSE41-LABEL: load_sext_4i8_to_4i64_extract:
2103; SSE41:       # %bb.0:
2104; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm0
2105; SSE41-NEXT:    retq
2106;
2107; AVX1-LABEL: load_sext_4i8_to_4i64_extract:
2108; AVX1:       # %bb.0:
2109; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
2110; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2111; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
2112; AVX1-NEXT:    retq
2113;
2114; AVX2-LABEL: load_sext_4i8_to_4i64_extract:
2115; AVX2:       # %bb.0:
2116; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2117; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
2118; AVX2-NEXT:    vzeroupper
2119; AVX2-NEXT:    retq
2120;
2121; AVX512-LABEL: load_sext_4i8_to_4i64_extract:
2122; AVX512:       # %bb.0:
2123; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
2124; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
2125; AVX512-NEXT:    vzeroupper
2126; AVX512-NEXT:    retq
2127;
2128; X32-SSE2-LABEL: load_sext_4i8_to_4i64_extract:
2129; X32-SSE2:       # %bb.0:
2130; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2131; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
2132; X32-SSE2-NEXT:    movd %ecx, %xmm1
2133; X32-SSE2-NEXT:    sarl $31, %ecx
2134; X32-SSE2-NEXT:    movd %ecx, %xmm0
2135; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2136; X32-SSE2-NEXT:    movsbl 2(%eax), %eax
2137; X32-SSE2-NEXT:    movd %eax, %xmm0
2138; X32-SSE2-NEXT:    sarl $31, %eax
2139; X32-SSE2-NEXT:    movd %eax, %xmm2
2140; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2141; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2142; X32-SSE2-NEXT:    retl
2143;
2144; X32-SSE41-LABEL: load_sext_4i8_to_4i64_extract:
2145; X32-SSE41:       # %bb.0:
2146; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2147; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm0
2148; X32-SSE41-NEXT:    retl
2149 %ld = load <4 x i8>, <4 x i8>* %ptr
2150 %sext = sext <4 x i8> %ld to <4 x i64>
2151 %extract = shufflevector <4 x i64> %sext, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
2152 ret <2 x i64> %extract
2153}
2154
2155define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
2156; SSE2-LABEL: load_sext_8i1_to_8i16:
2157; SSE2:       # %bb.0: # %entry
2158; SSE2-NEXT:    movsbq (%rdi), %rax
2159; SSE2-NEXT:    movq %rax, %rcx
2160; SSE2-NEXT:    shrq $7, %rcx
2161; SSE2-NEXT:    movd %ecx, %xmm0
2162; SSE2-NEXT:    movq %rax, %rcx
2163; SSE2-NEXT:    shlq $57, %rcx
2164; SSE2-NEXT:    sarq $63, %rcx
2165; SSE2-NEXT:    movd %ecx, %xmm2
2166; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2167; SSE2-NEXT:    movq %rax, %rcx
2168; SSE2-NEXT:    shlq $58, %rcx
2169; SSE2-NEXT:    sarq $63, %rcx
2170; SSE2-NEXT:    movd %ecx, %xmm0
2171; SSE2-NEXT:    movq %rax, %rcx
2172; SSE2-NEXT:    shlq $59, %rcx
2173; SSE2-NEXT:    sarq $63, %rcx
2174; SSE2-NEXT:    movd %ecx, %xmm1
2175; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2176; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2177; SSE2-NEXT:    movq %rax, %rcx
2178; SSE2-NEXT:    shlq $60, %rcx
2179; SSE2-NEXT:    sarq $63, %rcx
2180; SSE2-NEXT:    movd %ecx, %xmm0
2181; SSE2-NEXT:    movq %rax, %rcx
2182; SSE2-NEXT:    shlq $61, %rcx
2183; SSE2-NEXT:    sarq $63, %rcx
2184; SSE2-NEXT:    movd %ecx, %xmm2
2185; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2186; SSE2-NEXT:    movq %rax, %rcx
2187; SSE2-NEXT:    shlq $62, %rcx
2188; SSE2-NEXT:    sarq $63, %rcx
2189; SSE2-NEXT:    movd %ecx, %xmm3
2190; SSE2-NEXT:    shlq $63, %rax
2191; SSE2-NEXT:    sarq $63, %rax
2192; SSE2-NEXT:    movd %eax, %xmm0
2193; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2194; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2195; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2196; SSE2-NEXT:    retq
2197;
2198; SSSE3-LABEL: load_sext_8i1_to_8i16:
2199; SSSE3:       # %bb.0: # %entry
2200; SSSE3-NEXT:    movsbq (%rdi), %rax
2201; SSSE3-NEXT:    movq %rax, %rcx
2202; SSSE3-NEXT:    shrq $7, %rcx
2203; SSSE3-NEXT:    movd %ecx, %xmm0
2204; SSSE3-NEXT:    movq %rax, %rcx
2205; SSSE3-NEXT:    shlq $57, %rcx
2206; SSSE3-NEXT:    sarq $63, %rcx
2207; SSSE3-NEXT:    movd %ecx, %xmm2
2208; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2209; SSSE3-NEXT:    movq %rax, %rcx
2210; SSSE3-NEXT:    shlq $58, %rcx
2211; SSSE3-NEXT:    sarq $63, %rcx
2212; SSSE3-NEXT:    movd %ecx, %xmm0
2213; SSSE3-NEXT:    movq %rax, %rcx
2214; SSSE3-NEXT:    shlq $59, %rcx
2215; SSSE3-NEXT:    sarq $63, %rcx
2216; SSSE3-NEXT:    movd %ecx, %xmm1
2217; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2218; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2219; SSSE3-NEXT:    movq %rax, %rcx
2220; SSSE3-NEXT:    shlq $60, %rcx
2221; SSSE3-NEXT:    sarq $63, %rcx
2222; SSSE3-NEXT:    movd %ecx, %xmm0
2223; SSSE3-NEXT:    movq %rax, %rcx
2224; SSSE3-NEXT:    shlq $61, %rcx
2225; SSSE3-NEXT:    sarq $63, %rcx
2226; SSSE3-NEXT:    movd %ecx, %xmm2
2227; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2228; SSSE3-NEXT:    movq %rax, %rcx
2229; SSSE3-NEXT:    shlq $62, %rcx
2230; SSSE3-NEXT:    sarq $63, %rcx
2231; SSSE3-NEXT:    movd %ecx, %xmm3
2232; SSSE3-NEXT:    shlq $63, %rax
2233; SSSE3-NEXT:    sarq $63, %rax
2234; SSSE3-NEXT:    movd %eax, %xmm0
2235; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2236; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2237; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2238; SSSE3-NEXT:    retq
2239;
2240; SSE41-LABEL: load_sext_8i1_to_8i16:
2241; SSE41:       # %bb.0: # %entry
2242; SSE41-NEXT:    movsbq (%rdi), %rax
2243; SSE41-NEXT:    movq %rax, %rcx
2244; SSE41-NEXT:    shlq $62, %rcx
2245; SSE41-NEXT:    sarq $63, %rcx
2246; SSE41-NEXT:    movq %rax, %rdx
2247; SSE41-NEXT:    shlq $63, %rdx
2248; SSE41-NEXT:    sarq $63, %rdx
2249; SSE41-NEXT:    movd %edx, %xmm0
2250; SSE41-NEXT:    pinsrw $1, %ecx, %xmm0
2251; SSE41-NEXT:    movq %rax, %rcx
2252; SSE41-NEXT:    shlq $61, %rcx
2253; SSE41-NEXT:    sarq $63, %rcx
2254; SSE41-NEXT:    pinsrw $2, %ecx, %xmm0
2255; SSE41-NEXT:    movq %rax, %rcx
2256; SSE41-NEXT:    shlq $60, %rcx
2257; SSE41-NEXT:    sarq $63, %rcx
2258; SSE41-NEXT:    pinsrw $3, %ecx, %xmm0
2259; SSE41-NEXT:    movq %rax, %rcx
2260; SSE41-NEXT:    shlq $59, %rcx
2261; SSE41-NEXT:    sarq $63, %rcx
2262; SSE41-NEXT:    pinsrw $4, %ecx, %xmm0
2263; SSE41-NEXT:    movq %rax, %rcx
2264; SSE41-NEXT:    shlq $58, %rcx
2265; SSE41-NEXT:    sarq $63, %rcx
2266; SSE41-NEXT:    pinsrw $5, %ecx, %xmm0
2267; SSE41-NEXT:    movq %rax, %rcx
2268; SSE41-NEXT:    shlq $57, %rcx
2269; SSE41-NEXT:    sarq $63, %rcx
2270; SSE41-NEXT:    pinsrw $6, %ecx, %xmm0
2271; SSE41-NEXT:    shrq $7, %rax
2272; SSE41-NEXT:    pinsrw $7, %eax, %xmm0
2273; SSE41-NEXT:    retq
2274;
2275; AVX1-LABEL: load_sext_8i1_to_8i16:
2276; AVX1:       # %bb.0: # %entry
2277; AVX1-NEXT:    movsbq (%rdi), %rax
2278; AVX1-NEXT:    movq %rax, %rcx
2279; AVX1-NEXT:    shlq $62, %rcx
2280; AVX1-NEXT:    sarq $63, %rcx
2281; AVX1-NEXT:    movq %rax, %rdx
2282; AVX1-NEXT:    shlq $63, %rdx
2283; AVX1-NEXT:    sarq $63, %rdx
2284; AVX1-NEXT:    vmovd %edx, %xmm0
2285; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0
2286; AVX1-NEXT:    movq %rax, %rcx
2287; AVX1-NEXT:    shlq $61, %rcx
2288; AVX1-NEXT:    sarq $63, %rcx
2289; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0
2290; AVX1-NEXT:    movq %rax, %rcx
2291; AVX1-NEXT:    shlq $60, %rcx
2292; AVX1-NEXT:    sarq $63, %rcx
2293; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
2294; AVX1-NEXT:    movq %rax, %rcx
2295; AVX1-NEXT:    shlq $59, %rcx
2296; AVX1-NEXT:    sarq $63, %rcx
2297; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
2298; AVX1-NEXT:    movq %rax, %rcx
2299; AVX1-NEXT:    shlq $58, %rcx
2300; AVX1-NEXT:    sarq $63, %rcx
2301; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
2302; AVX1-NEXT:    movq %rax, %rcx
2303; AVX1-NEXT:    shlq $57, %rcx
2304; AVX1-NEXT:    sarq $63, %rcx
2305; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
2306; AVX1-NEXT:    shrq $7, %rax
2307; AVX1-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
2308; AVX1-NEXT:    retq
2309;
2310; AVX2-LABEL: load_sext_8i1_to_8i16:
2311; AVX2:       # %bb.0: # %entry
2312; AVX2-NEXT:    movsbq (%rdi), %rax
2313; AVX2-NEXT:    movq %rax, %rcx
2314; AVX2-NEXT:    shlq $62, %rcx
2315; AVX2-NEXT:    sarq $63, %rcx
2316; AVX2-NEXT:    movq %rax, %rdx
2317; AVX2-NEXT:    shlq $63, %rdx
2318; AVX2-NEXT:    sarq $63, %rdx
2319; AVX2-NEXT:    vmovd %edx, %xmm0
2320; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0
2321; AVX2-NEXT:    movq %rax, %rcx
2322; AVX2-NEXT:    shlq $61, %rcx
2323; AVX2-NEXT:    sarq $63, %rcx
2324; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0
2325; AVX2-NEXT:    movq %rax, %rcx
2326; AVX2-NEXT:    shlq $60, %rcx
2327; AVX2-NEXT:    sarq $63, %rcx
2328; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
2329; AVX2-NEXT:    movq %rax, %rcx
2330; AVX2-NEXT:    shlq $59, %rcx
2331; AVX2-NEXT:    sarq $63, %rcx
2332; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
2333; AVX2-NEXT:    movq %rax, %rcx
2334; AVX2-NEXT:    shlq $58, %rcx
2335; AVX2-NEXT:    sarq $63, %rcx
2336; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
2337; AVX2-NEXT:    movq %rax, %rcx
2338; AVX2-NEXT:    shlq $57, %rcx
2339; AVX2-NEXT:    sarq $63, %rcx
2340; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
2341; AVX2-NEXT:    shrq $7, %rax
2342; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
2343; AVX2-NEXT:    retq
2344;
2345; AVX512F-LABEL: load_sext_8i1_to_8i16:
2346; AVX512F:       # %bb.0: # %entry
2347; AVX512F-NEXT:    movzbl (%rdi), %eax
2348; AVX512F-NEXT:    kmovw %eax, %k1
2349; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2350; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
2351; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2352; AVX512F-NEXT:    vzeroupper
2353; AVX512F-NEXT:    retq
2354;
2355; AVX512BW-LABEL: load_sext_8i1_to_8i16:
2356; AVX512BW:       # %bb.0: # %entry
2357; AVX512BW-NEXT:    movzbl (%rdi), %eax
2358; AVX512BW-NEXT:    kmovd %eax, %k0
2359; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
2360; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
2361; AVX512BW-NEXT:    vzeroupper
2362; AVX512BW-NEXT:    retq
2363;
2364; X32-SSE2-LABEL: load_sext_8i1_to_8i16:
2365; X32-SSE2:       # %bb.0: # %entry
2366; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2367; X32-SSE2-NEXT:    movsbl (%eax), %eax
2368; X32-SSE2-NEXT:    movl %eax, %ecx
2369; X32-SSE2-NEXT:    shrl $7, %ecx
2370; X32-SSE2-NEXT:    movd %ecx, %xmm0
2371; X32-SSE2-NEXT:    movl %eax, %ecx
2372; X32-SSE2-NEXT:    shll $25, %ecx
2373; X32-SSE2-NEXT:    sarl $31, %ecx
2374; X32-SSE2-NEXT:    movd %ecx, %xmm1
2375; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2376; X32-SSE2-NEXT:    movl %eax, %ecx
2377; X32-SSE2-NEXT:    shll $26, %ecx
2378; X32-SSE2-NEXT:    sarl $31, %ecx
2379; X32-SSE2-NEXT:    movd %ecx, %xmm0
2380; X32-SSE2-NEXT:    movl %eax, %ecx
2381; X32-SSE2-NEXT:    shll $27, %ecx
2382; X32-SSE2-NEXT:    sarl $31, %ecx
2383; X32-SSE2-NEXT:    movd %ecx, %xmm2
2384; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2385; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2386; X32-SSE2-NEXT:    movl %eax, %ecx
2387; X32-SSE2-NEXT:    shll $28, %ecx
2388; X32-SSE2-NEXT:    sarl $31, %ecx
2389; X32-SSE2-NEXT:    movd %ecx, %xmm0
2390; X32-SSE2-NEXT:    movl %eax, %ecx
2391; X32-SSE2-NEXT:    shll $29, %ecx
2392; X32-SSE2-NEXT:    sarl $31, %ecx
2393; X32-SSE2-NEXT:    movd %ecx, %xmm1
2394; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2395; X32-SSE2-NEXT:    movl %eax, %ecx
2396; X32-SSE2-NEXT:    shll $30, %ecx
2397; X32-SSE2-NEXT:    sarl $31, %ecx
2398; X32-SSE2-NEXT:    movd %ecx, %xmm3
2399; X32-SSE2-NEXT:    shll $31, %eax
2400; X32-SSE2-NEXT:    sarl $31, %eax
2401; X32-SSE2-NEXT:    movd %eax, %xmm0
2402; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2403; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2404; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2405; X32-SSE2-NEXT:    retl
2406;
2407; X32-SSE41-LABEL: load_sext_8i1_to_8i16:
2408; X32-SSE41:       # %bb.0: # %entry
2409; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2410; X32-SSE41-NEXT:    movsbl (%eax), %eax
2411; X32-SSE41-NEXT:    movl %eax, %ecx
2412; X32-SSE41-NEXT:    shll $30, %ecx
2413; X32-SSE41-NEXT:    sarl $31, %ecx
2414; X32-SSE41-NEXT:    movl %eax, %edx
2415; X32-SSE41-NEXT:    shll $31, %edx
2416; X32-SSE41-NEXT:    sarl $31, %edx
2417; X32-SSE41-NEXT:    movd %edx, %xmm0
2418; X32-SSE41-NEXT:    pinsrw $1, %ecx, %xmm0
2419; X32-SSE41-NEXT:    movl %eax, %ecx
2420; X32-SSE41-NEXT:    shll $29, %ecx
2421; X32-SSE41-NEXT:    sarl $31, %ecx
2422; X32-SSE41-NEXT:    pinsrw $2, %ecx, %xmm0
2423; X32-SSE41-NEXT:    movl %eax, %ecx
2424; X32-SSE41-NEXT:    shll $28, %ecx
2425; X32-SSE41-NEXT:    sarl $31, %ecx
2426; X32-SSE41-NEXT:    pinsrw $3, %ecx, %xmm0
2427; X32-SSE41-NEXT:    movl %eax, %ecx
2428; X32-SSE41-NEXT:    shll $27, %ecx
2429; X32-SSE41-NEXT:    sarl $31, %ecx
2430; X32-SSE41-NEXT:    pinsrw $4, %ecx, %xmm0
2431; X32-SSE41-NEXT:    movl %eax, %ecx
2432; X32-SSE41-NEXT:    shll $26, %ecx
2433; X32-SSE41-NEXT:    sarl $31, %ecx
2434; X32-SSE41-NEXT:    pinsrw $5, %ecx, %xmm0
2435; X32-SSE41-NEXT:    movl %eax, %ecx
2436; X32-SSE41-NEXT:    shll $25, %ecx
2437; X32-SSE41-NEXT:    sarl $31, %ecx
2438; X32-SSE41-NEXT:    pinsrw $6, %ecx, %xmm0
2439; X32-SSE41-NEXT:    shrl $7, %eax
2440; X32-SSE41-NEXT:    pinsrw $7, %eax, %xmm0
2441; X32-SSE41-NEXT:    retl
2442entry:
2443 %X = load <8 x i1>, <8 x i1>* %ptr
2444 %Y = sext <8 x i1> %X to <8 x i16>
2445 ret <8 x i16> %Y
2446}
2447
2448define <8 x i16> @load_sext_8i8_to_8i16(<8 x i8> *%ptr) {
2449; SSE2-LABEL: load_sext_8i8_to_8i16:
2450; SSE2:       # %bb.0: # %entry
2451; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2452; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2453; SSE2-NEXT:    psraw $8, %xmm0
2454; SSE2-NEXT:    retq
2455;
2456; SSSE3-LABEL: load_sext_8i8_to_8i16:
2457; SSSE3:       # %bb.0: # %entry
2458; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2459; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2460; SSSE3-NEXT:    psraw $8, %xmm0
2461; SSSE3-NEXT:    retq
2462;
2463; SSE41-LABEL: load_sext_8i8_to_8i16:
2464; SSE41:       # %bb.0: # %entry
2465; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
2466; SSE41-NEXT:    retq
2467;
2468; AVX-LABEL: load_sext_8i8_to_8i16:
2469; AVX:       # %bb.0: # %entry
2470; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
2471; AVX-NEXT:    retq
2472;
2473; X32-SSE2-LABEL: load_sext_8i8_to_8i16:
2474; X32-SSE2:       # %bb.0: # %entry
2475; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2476; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2477; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2478; X32-SSE2-NEXT:    psraw $8, %xmm0
2479; X32-SSE2-NEXT:    retl
2480;
2481; X32-SSE41-LABEL: load_sext_8i8_to_8i16:
2482; X32-SSE41:       # %bb.0: # %entry
2483; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2484; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
2485; X32-SSE41-NEXT:    retl
2486entry:
2487 %X = load <8 x i8>, <8 x i8>* %ptr
2488 %Y = sext <8 x i8> %X to <8 x i16>
2489 ret <8 x i16> %Y
2490}
2491
2492define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
2493; SSE2-LABEL: load_sext_8i8_to_8i64:
2494; SSE2:       # %bb.0: # %entry
2495; SSE2-NEXT:    movsbq 1(%rdi), %rax
2496; SSE2-NEXT:    movq %rax, %xmm1
2497; SSE2-NEXT:    movsbq (%rdi), %rax
2498; SSE2-NEXT:    movq %rax, %xmm0
2499; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2500; SSE2-NEXT:    movsbq 3(%rdi), %rax
2501; SSE2-NEXT:    movq %rax, %xmm2
2502; SSE2-NEXT:    movsbq 2(%rdi), %rax
2503; SSE2-NEXT:    movq %rax, %xmm1
2504; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2505; SSE2-NEXT:    movsbq 5(%rdi), %rax
2506; SSE2-NEXT:    movq %rax, %xmm3
2507; SSE2-NEXT:    movsbq 4(%rdi), %rax
2508; SSE2-NEXT:    movq %rax, %xmm2
2509; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
2510; SSE2-NEXT:    movsbq 7(%rdi), %rax
2511; SSE2-NEXT:    movq %rax, %xmm4
2512; SSE2-NEXT:    movsbq 6(%rdi), %rax
2513; SSE2-NEXT:    movq %rax, %xmm3
2514; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
2515; SSE2-NEXT:    retq
2516;
2517; SSSE3-LABEL: load_sext_8i8_to_8i64:
2518; SSSE3:       # %bb.0: # %entry
2519; SSSE3-NEXT:    movsbq 1(%rdi), %rax
2520; SSSE3-NEXT:    movq %rax, %xmm1
2521; SSSE3-NEXT:    movsbq (%rdi), %rax
2522; SSSE3-NEXT:    movq %rax, %xmm0
2523; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2524; SSSE3-NEXT:    movsbq 3(%rdi), %rax
2525; SSSE3-NEXT:    movq %rax, %xmm2
2526; SSSE3-NEXT:    movsbq 2(%rdi), %rax
2527; SSSE3-NEXT:    movq %rax, %xmm1
2528; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2529; SSSE3-NEXT:    movsbq 5(%rdi), %rax
2530; SSSE3-NEXT:    movq %rax, %xmm3
2531; SSSE3-NEXT:    movsbq 4(%rdi), %rax
2532; SSSE3-NEXT:    movq %rax, %xmm2
2533; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
2534; SSSE3-NEXT:    movsbq 7(%rdi), %rax
2535; SSSE3-NEXT:    movq %rax, %xmm4
2536; SSSE3-NEXT:    movsbq 6(%rdi), %rax
2537; SSSE3-NEXT:    movq %rax, %xmm3
2538; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
2539; SSSE3-NEXT:    retq
2540;
2541; SSE41-LABEL: load_sext_8i8_to_8i64:
2542; SSE41:       # %bb.0: # %entry
2543; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
2544; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
2545; SSE41-NEXT:    pmovsxbq 4(%rdi), %xmm2
2546; SSE41-NEXT:    pmovsxbq 6(%rdi), %xmm3
2547; SSE41-NEXT:    retq
2548;
2549; AVX1-LABEL: load_sext_8i8_to_8i64:
2550; AVX1:       # %bb.0: # %entry
2551; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
2552; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
2553; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2554; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
2555; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2556; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm1
2557; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm2
2558; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
2559; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
2560; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
2561; AVX1-NEXT:    retq
2562;
2563; AVX2-LABEL: load_sext_8i8_to_8i64:
2564; AVX2:       # %bb.0: # %entry
2565; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2566; AVX2-NEXT:    vpmovsxbq 4(%rdi), %ymm1
2567; AVX2-NEXT:    retq
2568;
2569; AVX512-LABEL: load_sext_8i8_to_8i64:
2570; AVX512:       # %bb.0: # %entry
2571; AVX512-NEXT:    vpmovsxbq (%rdi), %zmm0
2572; AVX512-NEXT:    retq
2573;
2574; X32-SSE2-LABEL: load_sext_8i8_to_8i64:
2575; X32-SSE2:       # %bb.0: # %entry
2576; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2577; X32-SSE2-NEXT:    movsbl 1(%eax), %ecx
2578; X32-SSE2-NEXT:    movd %ecx, %xmm1
2579; X32-SSE2-NEXT:    sarl $31, %ecx
2580; X32-SSE2-NEXT:    movd %ecx, %xmm0
2581; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2582; X32-SSE2-NEXT:    movsbl (%eax), %ecx
2583; X32-SSE2-NEXT:    movd %ecx, %xmm0
2584; X32-SSE2-NEXT:    sarl $31, %ecx
2585; X32-SSE2-NEXT:    movd %ecx, %xmm2
2586; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2587; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2588; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
2589; X32-SSE2-NEXT:    movd %ecx, %xmm2
2590; X32-SSE2-NEXT:    sarl $31, %ecx
2591; X32-SSE2-NEXT:    movd %ecx, %xmm1
2592; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2593; X32-SSE2-NEXT:    movsbl 2(%eax), %ecx
2594; X32-SSE2-NEXT:    movd %ecx, %xmm1
2595; X32-SSE2-NEXT:    sarl $31, %ecx
2596; X32-SSE2-NEXT:    movd %ecx, %xmm3
2597; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
2598; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2599; X32-SSE2-NEXT:    movsbl 5(%eax), %ecx
2600; X32-SSE2-NEXT:    movd %ecx, %xmm3
2601; X32-SSE2-NEXT:    sarl $31, %ecx
2602; X32-SSE2-NEXT:    movd %ecx, %xmm2
2603; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2604; X32-SSE2-NEXT:    movsbl 4(%eax), %ecx
2605; X32-SSE2-NEXT:    movd %ecx, %xmm2
2606; X32-SSE2-NEXT:    sarl $31, %ecx
2607; X32-SSE2-NEXT:    movd %ecx, %xmm4
2608; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2609; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
2610; X32-SSE2-NEXT:    movsbl 7(%eax), %ecx
2611; X32-SSE2-NEXT:    movd %ecx, %xmm4
2612; X32-SSE2-NEXT:    sarl $31, %ecx
2613; X32-SSE2-NEXT:    movd %ecx, %xmm3
2614; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2615; X32-SSE2-NEXT:    movsbl 6(%eax), %eax
2616; X32-SSE2-NEXT:    movd %eax, %xmm3
2617; X32-SSE2-NEXT:    sarl $31, %eax
2618; X32-SSE2-NEXT:    movd %eax, %xmm5
2619; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
2620; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
2621; X32-SSE2-NEXT:    retl
2622;
2623; X32-SSE41-LABEL: load_sext_8i8_to_8i64:
2624; X32-SSE41:       # %bb.0: # %entry
2625; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2626; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
2627; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
2628; X32-SSE41-NEXT:    pmovsxbq 4(%eax), %xmm2
2629; X32-SSE41-NEXT:    pmovsxbq 6(%eax), %xmm3
2630; X32-SSE41-NEXT:    retl
2631entry:
2632 %X = load <8 x i8>, <8 x i8>* %ptr
2633 %Y = sext <8 x i8> %X to <8 x i64>
2634 ret <8 x i64> %Y
2635}
2636
2637define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
2638; SSE2-LABEL: load_sext_8i1_to_8i32:
2639; SSE2:       # %bb.0: # %entry
2640; SSE2-NEXT:    movzbl (%rdi), %eax
2641; SSE2-NEXT:    movl %eax, %ecx
2642; SSE2-NEXT:    shrl $7, %ecx
2643; SSE2-NEXT:    movd %ecx, %xmm0
2644; SSE2-NEXT:    movl %eax, %ecx
2645; SSE2-NEXT:    shrl $6, %ecx
2646; SSE2-NEXT:    andl $1, %ecx
2647; SSE2-NEXT:    movd %ecx, %xmm2
2648; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2649; SSE2-NEXT:    movl %eax, %ecx
2650; SSE2-NEXT:    shrl $5, %ecx
2651; SSE2-NEXT:    andl $1, %ecx
2652; SSE2-NEXT:    movd %ecx, %xmm0
2653; SSE2-NEXT:    movl %eax, %ecx
2654; SSE2-NEXT:    shrl $4, %ecx
2655; SSE2-NEXT:    andl $1, %ecx
2656; SSE2-NEXT:    movd %ecx, %xmm1
2657; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2658; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2659; SSE2-NEXT:    movl %eax, %ecx
2660; SSE2-NEXT:    shrl $3, %ecx
2661; SSE2-NEXT:    andl $1, %ecx
2662; SSE2-NEXT:    movd %ecx, %xmm0
2663; SSE2-NEXT:    movl %eax, %ecx
2664; SSE2-NEXT:    shrl $2, %ecx
2665; SSE2-NEXT:    andl $1, %ecx
2666; SSE2-NEXT:    movd %ecx, %xmm2
2667; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2668; SSE2-NEXT:    movl %eax, %ecx
2669; SSE2-NEXT:    andl $1, %ecx
2670; SSE2-NEXT:    movd %ecx, %xmm0
2671; SSE2-NEXT:    shrl %eax
2672; SSE2-NEXT:    andl $1, %eax
2673; SSE2-NEXT:    movd %eax, %xmm3
2674; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2675; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2676; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2677; SSE2-NEXT:    pslld $31, %xmm0
2678; SSE2-NEXT:    psrad $31, %xmm0
2679; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2680; SSE2-NEXT:    pslld $31, %xmm1
2681; SSE2-NEXT:    psrad $31, %xmm1
2682; SSE2-NEXT:    retq
2683;
2684; SSSE3-LABEL: load_sext_8i1_to_8i32:
2685; SSSE3:       # %bb.0: # %entry
2686; SSSE3-NEXT:    movzbl (%rdi), %eax
2687; SSSE3-NEXT:    movl %eax, %ecx
2688; SSSE3-NEXT:    shrl $7, %ecx
2689; SSSE3-NEXT:    movd %ecx, %xmm0
2690; SSSE3-NEXT:    movl %eax, %ecx
2691; SSSE3-NEXT:    shrl $6, %ecx
2692; SSSE3-NEXT:    andl $1, %ecx
2693; SSSE3-NEXT:    movd %ecx, %xmm2
2694; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2695; SSSE3-NEXT:    movl %eax, %ecx
2696; SSSE3-NEXT:    shrl $5, %ecx
2697; SSSE3-NEXT:    andl $1, %ecx
2698; SSSE3-NEXT:    movd %ecx, %xmm0
2699; SSSE3-NEXT:    movl %eax, %ecx
2700; SSSE3-NEXT:    shrl $4, %ecx
2701; SSSE3-NEXT:    andl $1, %ecx
2702; SSSE3-NEXT:    movd %ecx, %xmm1
2703; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2704; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2705; SSSE3-NEXT:    movl %eax, %ecx
2706; SSSE3-NEXT:    shrl $3, %ecx
2707; SSSE3-NEXT:    andl $1, %ecx
2708; SSSE3-NEXT:    movd %ecx, %xmm0
2709; SSSE3-NEXT:    movl %eax, %ecx
2710; SSSE3-NEXT:    shrl $2, %ecx
2711; SSSE3-NEXT:    andl $1, %ecx
2712; SSSE3-NEXT:    movd %ecx, %xmm2
2713; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2714; SSSE3-NEXT:    movl %eax, %ecx
2715; SSSE3-NEXT:    andl $1, %ecx
2716; SSSE3-NEXT:    movd %ecx, %xmm0
2717; SSSE3-NEXT:    shrl %eax
2718; SSSE3-NEXT:    andl $1, %eax
2719; SSSE3-NEXT:    movd %eax, %xmm3
2720; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2721; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2722; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2723; SSSE3-NEXT:    pslld $31, %xmm0
2724; SSSE3-NEXT:    psrad $31, %xmm0
2725; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2726; SSSE3-NEXT:    pslld $31, %xmm1
2727; SSSE3-NEXT:    psrad $31, %xmm1
2728; SSSE3-NEXT:    retq
2729;
2730; SSE41-LABEL: load_sext_8i1_to_8i32:
2731; SSE41:       # %bb.0: # %entry
2732; SSE41-NEXT:    movzbl (%rdi), %eax
2733; SSE41-NEXT:    movl %eax, %ecx
2734; SSE41-NEXT:    shrl %ecx
2735; SSE41-NEXT:    andl $1, %ecx
2736; SSE41-NEXT:    movl %eax, %edx
2737; SSE41-NEXT:    andl $1, %edx
2738; SSE41-NEXT:    movd %edx, %xmm1
2739; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
2740; SSE41-NEXT:    movl %eax, %ecx
2741; SSE41-NEXT:    shrl $2, %ecx
2742; SSE41-NEXT:    andl $1, %ecx
2743; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
2744; SSE41-NEXT:    movl %eax, %ecx
2745; SSE41-NEXT:    shrl $3, %ecx
2746; SSE41-NEXT:    andl $1, %ecx
2747; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
2748; SSE41-NEXT:    movl %eax, %ecx
2749; SSE41-NEXT:    shrl $4, %ecx
2750; SSE41-NEXT:    andl $1, %ecx
2751; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
2752; SSE41-NEXT:    movl %eax, %ecx
2753; SSE41-NEXT:    shrl $5, %ecx
2754; SSE41-NEXT:    andl $1, %ecx
2755; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
2756; SSE41-NEXT:    movl %eax, %ecx
2757; SSE41-NEXT:    shrl $6, %ecx
2758; SSE41-NEXT:    andl $1, %ecx
2759; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
2760; SSE41-NEXT:    shrl $7, %eax
2761; SSE41-NEXT:    pinsrw $7, %eax, %xmm1
2762; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2763; SSE41-NEXT:    pslld $31, %xmm0
2764; SSE41-NEXT:    psrad $31, %xmm0
2765; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2766; SSE41-NEXT:    pslld $31, %xmm1
2767; SSE41-NEXT:    psrad $31, %xmm1
2768; SSE41-NEXT:    retq
2769;
2770; AVX1-LABEL: load_sext_8i1_to_8i32:
2771; AVX1:       # %bb.0: # %entry
2772; AVX1-NEXT:    movsbq (%rdi), %rax
2773; AVX1-NEXT:    movq %rax, %rcx
2774; AVX1-NEXT:    shlq $58, %rcx
2775; AVX1-NEXT:    sarq $63, %rcx
2776; AVX1-NEXT:    movq %rax, %rdx
2777; AVX1-NEXT:    shlq $59, %rdx
2778; AVX1-NEXT:    sarq $63, %rdx
2779; AVX1-NEXT:    vmovd %edx, %xmm0
2780; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
2781; AVX1-NEXT:    movq %rax, %rcx
2782; AVX1-NEXT:    shlq $57, %rcx
2783; AVX1-NEXT:    sarq $63, %rcx
2784; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
2785; AVX1-NEXT:    movq %rax, %rcx
2786; AVX1-NEXT:    shrq $7, %rcx
2787; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
2788; AVX1-NEXT:    movq %rax, %rcx
2789; AVX1-NEXT:    shlq $62, %rcx
2790; AVX1-NEXT:    sarq $63, %rcx
2791; AVX1-NEXT:    movq %rax, %rdx
2792; AVX1-NEXT:    shlq $63, %rdx
2793; AVX1-NEXT:    sarq $63, %rdx
2794; AVX1-NEXT:    vmovd %edx, %xmm1
2795; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
2796; AVX1-NEXT:    movq %rax, %rcx
2797; AVX1-NEXT:    shlq $61, %rcx
2798; AVX1-NEXT:    sarq $63, %rcx
2799; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
2800; AVX1-NEXT:    shlq $60, %rax
2801; AVX1-NEXT:    sarq $63, %rax
2802; AVX1-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
2803; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2804; AVX1-NEXT:    retq
2805;
2806; AVX2-LABEL: load_sext_8i1_to_8i32:
2807; AVX2:       # %bb.0: # %entry
2808; AVX2-NEXT:    movsbq (%rdi), %rax
2809; AVX2-NEXT:    movq %rax, %rcx
2810; AVX2-NEXT:    shlq $58, %rcx
2811; AVX2-NEXT:    sarq $63, %rcx
2812; AVX2-NEXT:    movq %rax, %rdx
2813; AVX2-NEXT:    shlq $59, %rdx
2814; AVX2-NEXT:    sarq $63, %rdx
2815; AVX2-NEXT:    vmovd %edx, %xmm0
2816; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
2817; AVX2-NEXT:    movq %rax, %rcx
2818; AVX2-NEXT:    shlq $57, %rcx
2819; AVX2-NEXT:    sarq $63, %rcx
2820; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
2821; AVX2-NEXT:    movq %rax, %rcx
2822; AVX2-NEXT:    shrq $7, %rcx
2823; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
2824; AVX2-NEXT:    movq %rax, %rcx
2825; AVX2-NEXT:    shlq $62, %rcx
2826; AVX2-NEXT:    sarq $63, %rcx
2827; AVX2-NEXT:    movq %rax, %rdx
2828; AVX2-NEXT:    shlq $63, %rdx
2829; AVX2-NEXT:    sarq $63, %rdx
2830; AVX2-NEXT:    vmovd %edx, %xmm1
2831; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
2832; AVX2-NEXT:    movq %rax, %rcx
2833; AVX2-NEXT:    shlq $61, %rcx
2834; AVX2-NEXT:    sarq $63, %rcx
2835; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
2836; AVX2-NEXT:    shlq $60, %rax
2837; AVX2-NEXT:    sarq $63, %rax
2838; AVX2-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
2839; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
2840; AVX2-NEXT:    retq
2841;
2842; AVX512F-LABEL: load_sext_8i1_to_8i32:
2843; AVX512F:       # %bb.0: # %entry
2844; AVX512F-NEXT:    movzbl (%rdi), %eax
2845; AVX512F-NEXT:    kmovw %eax, %k1
2846; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2847; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2848; AVX512F-NEXT:    retq
2849;
2850; AVX512BW-LABEL: load_sext_8i1_to_8i32:
2851; AVX512BW:       # %bb.0: # %entry
2852; AVX512BW-NEXT:    movzbl (%rdi), %eax
2853; AVX512BW-NEXT:    kmovd %eax, %k1
2854; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2855; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2856; AVX512BW-NEXT:    retq
2857;
2858; X32-SSE2-LABEL: load_sext_8i1_to_8i32:
2859; X32-SSE2:       # %bb.0: # %entry
2860; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2861; X32-SSE2-NEXT:    movzbl (%eax), %eax
2862; X32-SSE2-NEXT:    movl %eax, %ecx
2863; X32-SSE2-NEXT:    shrl $7, %ecx
2864; X32-SSE2-NEXT:    movd %ecx, %xmm0
2865; X32-SSE2-NEXT:    movl %eax, %ecx
2866; X32-SSE2-NEXT:    shrl $6, %ecx
2867; X32-SSE2-NEXT:    andl $1, %ecx
2868; X32-SSE2-NEXT:    movd %ecx, %xmm2
2869; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2870; X32-SSE2-NEXT:    movl %eax, %ecx
2871; X32-SSE2-NEXT:    shrl $5, %ecx
2872; X32-SSE2-NEXT:    andl $1, %ecx
2873; X32-SSE2-NEXT:    movd %ecx, %xmm0
2874; X32-SSE2-NEXT:    movl %eax, %ecx
2875; X32-SSE2-NEXT:    shrl $4, %ecx
2876; X32-SSE2-NEXT:    andl $1, %ecx
2877; X32-SSE2-NEXT:    movd %ecx, %xmm1
2878; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2879; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2880; X32-SSE2-NEXT:    movl %eax, %ecx
2881; X32-SSE2-NEXT:    shrl $3, %ecx
2882; X32-SSE2-NEXT:    andl $1, %ecx
2883; X32-SSE2-NEXT:    movd %ecx, %xmm0
2884; X32-SSE2-NEXT:    movl %eax, %ecx
2885; X32-SSE2-NEXT:    shrl $2, %ecx
2886; X32-SSE2-NEXT:    andl $1, %ecx
2887; X32-SSE2-NEXT:    movd %ecx, %xmm2
2888; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2889; X32-SSE2-NEXT:    movl %eax, %ecx
2890; X32-SSE2-NEXT:    andl $1, %ecx
2891; X32-SSE2-NEXT:    movd %ecx, %xmm0
2892; X32-SSE2-NEXT:    shrl %eax
2893; X32-SSE2-NEXT:    andl $1, %eax
2894; X32-SSE2-NEXT:    movd %eax, %xmm3
2895; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2896; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2897; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2898; X32-SSE2-NEXT:    pslld $31, %xmm0
2899; X32-SSE2-NEXT:    psrad $31, %xmm0
2900; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2901; X32-SSE2-NEXT:    pslld $31, %xmm1
2902; X32-SSE2-NEXT:    psrad $31, %xmm1
2903; X32-SSE2-NEXT:    retl
2904;
2905; X32-SSE41-LABEL: load_sext_8i1_to_8i32:
2906; X32-SSE41:       # %bb.0: # %entry
2907; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2908; X32-SSE41-NEXT:    movzbl (%eax), %eax
2909; X32-SSE41-NEXT:    movl %eax, %ecx
2910; X32-SSE41-NEXT:    shrl %ecx
2911; X32-SSE41-NEXT:    andl $1, %ecx
2912; X32-SSE41-NEXT:    movl %eax, %edx
2913; X32-SSE41-NEXT:    andl $1, %edx
2914; X32-SSE41-NEXT:    movd %edx, %xmm1
2915; X32-SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
2916; X32-SSE41-NEXT:    movl %eax, %ecx
2917; X32-SSE41-NEXT:    shrl $2, %ecx
2918; X32-SSE41-NEXT:    andl $1, %ecx
2919; X32-SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
2920; X32-SSE41-NEXT:    movl %eax, %ecx
2921; X32-SSE41-NEXT:    shrl $3, %ecx
2922; X32-SSE41-NEXT:    andl $1, %ecx
2923; X32-SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
2924; X32-SSE41-NEXT:    movl %eax, %ecx
2925; X32-SSE41-NEXT:    shrl $4, %ecx
2926; X32-SSE41-NEXT:    andl $1, %ecx
2927; X32-SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
2928; X32-SSE41-NEXT:    movl %eax, %ecx
2929; X32-SSE41-NEXT:    shrl $5, %ecx
2930; X32-SSE41-NEXT:    andl $1, %ecx
2931; X32-SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
2932; X32-SSE41-NEXT:    movl %eax, %ecx
2933; X32-SSE41-NEXT:    shrl $6, %ecx
2934; X32-SSE41-NEXT:    andl $1, %ecx
2935; X32-SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
2936; X32-SSE41-NEXT:    shrl $7, %eax
2937; X32-SSE41-NEXT:    pinsrw $7, %eax, %xmm1
2938; X32-SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2939; X32-SSE41-NEXT:    pslld $31, %xmm0
2940; X32-SSE41-NEXT:    psrad $31, %xmm0
2941; X32-SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2942; X32-SSE41-NEXT:    pslld $31, %xmm1
2943; X32-SSE41-NEXT:    psrad $31, %xmm1
2944; X32-SSE41-NEXT:    retl
2945entry:
2946 %X = load <8 x i1>, <8 x i1>* %ptr
2947 %Y = sext <8 x i1> %X to <8 x i32>
2948 ret <8 x i32> %Y
2949}
2950
2951define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
2952; SSE2-LABEL: load_sext_8i8_to_8i32:
2953; SSE2:       # %bb.0: # %entry
2954; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2955; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2956; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2957; SSE2-NEXT:    psrad $24, %xmm0
2958; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
2959; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2960; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2961; SSE2-NEXT:    psrad $24, %xmm1
2962; SSE2-NEXT:    retq
2963;
2964; SSSE3-LABEL: load_sext_8i8_to_8i32:
2965; SSSE3:       # %bb.0: # %entry
2966; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2967; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2968; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2969; SSSE3-NEXT:    psrad $24, %xmm0
2970; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
2971; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2972; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
2973; SSSE3-NEXT:    psrad $24, %xmm1
2974; SSSE3-NEXT:    retq
2975;
2976; SSE41-LABEL: load_sext_8i8_to_8i32:
2977; SSE41:       # %bb.0: # %entry
2978; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
2979; SSE41-NEXT:    pmovsxbd 4(%rdi), %xmm1
2980; SSE41-NEXT:    retq
2981;
2982; AVX1-LABEL: load_sext_8i8_to_8i32:
2983; AVX1:       # %bb.0: # %entry
2984; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
2985; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
2986; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2987; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
2988; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2989; AVX1-NEXT:    retq
2990;
2991; AVX2-LABEL: load_sext_8i8_to_8i32:
2992; AVX2:       # %bb.0: # %entry
2993; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm0
2994; AVX2-NEXT:    retq
2995;
2996; AVX512-LABEL: load_sext_8i8_to_8i32:
2997; AVX512:       # %bb.0: # %entry
2998; AVX512-NEXT:    vpmovsxbd (%rdi), %ymm0
2999; AVX512-NEXT:    retq
3000;
3001; X32-SSE2-LABEL: load_sext_8i8_to_8i32:
3002; X32-SSE2:       # %bb.0: # %entry
3003; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3004; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
3005; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3006; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
3007; X32-SSE2-NEXT:    psrad $24, %xmm0
3008; X32-SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
3009; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3010; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
3011; X32-SSE2-NEXT:    psrad $24, %xmm1
3012; X32-SSE2-NEXT:    retl
3013;
3014; X32-SSE41-LABEL: load_sext_8i8_to_8i32:
3015; X32-SSE41:       # %bb.0: # %entry
3016; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3017; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
3018; X32-SSE41-NEXT:    pmovsxbd 4(%eax), %xmm1
3019; X32-SSE41-NEXT:    retl
3020entry:
3021 %X = load <8 x i8>, <8 x i8>* %ptr
3022 %Y = sext <8 x i8> %X to <8 x i32>
3023 ret <8 x i32> %Y
3024}
3025
3026define <16 x i8> @load_sext_16i1_to_16i8(<16 x i1> *%ptr) nounwind readnone {
3027; SSE2-LABEL: load_sext_16i1_to_16i8:
3028; SSE2:       # %bb.0: # %entry
3029; SSE2-NEXT:    pushq %rbp
3030; SSE2-NEXT:    pushq %r15
3031; SSE2-NEXT:    pushq %r14
3032; SSE2-NEXT:    pushq %r13
3033; SSE2-NEXT:    pushq %r12
3034; SSE2-NEXT:    pushq %rbx
3035; SSE2-NEXT:    movswq (%rdi), %rax
3036; SSE2-NEXT:    movq %rax, %r8
3037; SSE2-NEXT:    movq %rax, %r9
3038; SSE2-NEXT:    movq %rax, %r10
3039; SSE2-NEXT:    movq %rax, %r11
3040; SSE2-NEXT:    movq %rax, %r14
3041; SSE2-NEXT:    movq %rax, %r15
3042; SSE2-NEXT:    movq %rax, %r12
3043; SSE2-NEXT:    movq %rax, %r13
3044; SSE2-NEXT:    movq %rax, %rbx
3045; SSE2-NEXT:    movq %rax, %rcx
3046; SSE2-NEXT:    movq %rax, %rdx
3047; SSE2-NEXT:    movq %rax, %rsi
3048; SSE2-NEXT:    movq %rax, %rdi
3049; SSE2-NEXT:    movq %rax, %rbp
3050; SSE2-NEXT:    shrq $15, %rbp
3051; SSE2-NEXT:    movd %ebp, %xmm0
3052; SSE2-NEXT:    movq %rax, %rbp
3053; SSE2-NEXT:    movsbq %al, %rax
3054; SSE2-NEXT:    shlq $49, %r8
3055; SSE2-NEXT:    sarq $63, %r8
3056; SSE2-NEXT:    movd %r8d, %xmm1
3057; SSE2-NEXT:    shlq $50, %r9
3058; SSE2-NEXT:    sarq $63, %r9
3059; SSE2-NEXT:    movd %r9d, %xmm2
3060; SSE2-NEXT:    shlq $51, %r10
3061; SSE2-NEXT:    sarq $63, %r10
3062; SSE2-NEXT:    movd %r10d, %xmm3
3063; SSE2-NEXT:    shlq $52, %r11
3064; SSE2-NEXT:    sarq $63, %r11
3065; SSE2-NEXT:    movd %r11d, %xmm4
3066; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3067; SSE2-NEXT:    shlq $53, %r14
3068; SSE2-NEXT:    sarq $63, %r14
3069; SSE2-NEXT:    movd %r14d, %xmm0
3070; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
3071; SSE2-NEXT:    shlq $54, %r15
3072; SSE2-NEXT:    sarq $63, %r15
3073; SSE2-NEXT:    movd %r15d, %xmm2
3074; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3075; SSE2-NEXT:    shlq $55, %r12
3076; SSE2-NEXT:    sarq $63, %r12
3077; SSE2-NEXT:    movd %r12d, %xmm1
3078; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3079; SSE2-NEXT:    shlq $60, %r13
3080; SSE2-NEXT:    sarq $63, %r13
3081; SSE2-NEXT:    movd %r13d, %xmm4
3082; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3083; SSE2-NEXT:    shlq $61, %rbx
3084; SSE2-NEXT:    sarq $63, %rbx
3085; SSE2-NEXT:    movd %ebx, %xmm2
3086; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3087; SSE2-NEXT:    shlq $62, %rcx
3088; SSE2-NEXT:    sarq $63, %rcx
3089; SSE2-NEXT:    movd %ecx, %xmm5
3090; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
3091; SSE2-NEXT:    shlq $63, %rdx
3092; SSE2-NEXT:    sarq $63, %rdx
3093; SSE2-NEXT:    movd %edx, %xmm0
3094; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
3095; SSE2-NEXT:    shlq $58, %rsi
3096; SSE2-NEXT:    sarq $63, %rsi
3097; SSE2-NEXT:    movd %esi, %xmm3
3098; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
3099; SSE2-NEXT:    shlq $59, %rdi
3100; SSE2-NEXT:    sarq $63, %rdi
3101; SSE2-NEXT:    movd %edi, %xmm4
3102; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3103; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
3104; SSE2-NEXT:    shlq $57, %rbp
3105; SSE2-NEXT:    sarq $63, %rbp
3106; SSE2-NEXT:    movd %ebp, %xmm2
3107; SSE2-NEXT:    shrq $7, %rax
3108; SSE2-NEXT:    movd %eax, %xmm3
3109; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
3110; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
3111; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
3112; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
3113; SSE2-NEXT:    popq %rbx
3114; SSE2-NEXT:    popq %r12
3115; SSE2-NEXT:    popq %r13
3116; SSE2-NEXT:    popq %r14
3117; SSE2-NEXT:    popq %r15
3118; SSE2-NEXT:    popq %rbp
3119; SSE2-NEXT:    retq
3120;
3121; SSSE3-LABEL: load_sext_16i1_to_16i8:
3122; SSSE3:       # %bb.0: # %entry
3123; SSSE3-NEXT:    pushq %rbp
3124; SSSE3-NEXT:    pushq %r15
3125; SSSE3-NEXT:    pushq %r14
3126; SSSE3-NEXT:    pushq %r13
3127; SSSE3-NEXT:    pushq %r12
3128; SSSE3-NEXT:    pushq %rbx
3129; SSSE3-NEXT:    movswq (%rdi), %rax
3130; SSSE3-NEXT:    movq %rax, %r8
3131; SSSE3-NEXT:    movq %rax, %r9
3132; SSSE3-NEXT:    movq %rax, %r10
3133; SSSE3-NEXT:    movq %rax, %r11
3134; SSSE3-NEXT:    movq %rax, %r14
3135; SSSE3-NEXT:    movq %rax, %r15
3136; SSSE3-NEXT:    movq %rax, %r12
3137; SSSE3-NEXT:    movq %rax, %r13
3138; SSSE3-NEXT:    movq %rax, %rbx
3139; SSSE3-NEXT:    movq %rax, %rcx
3140; SSSE3-NEXT:    movq %rax, %rdx
3141; SSSE3-NEXT:    movq %rax, %rsi
3142; SSSE3-NEXT:    movq %rax, %rdi
3143; SSSE3-NEXT:    movq %rax, %rbp
3144; SSSE3-NEXT:    shrq $15, %rbp
3145; SSSE3-NEXT:    movd %ebp, %xmm0
3146; SSSE3-NEXT:    movq %rax, %rbp
3147; SSSE3-NEXT:    movsbq %al, %rax
3148; SSSE3-NEXT:    shlq $49, %r8
3149; SSSE3-NEXT:    sarq $63, %r8
3150; SSSE3-NEXT:    movd %r8d, %xmm1
3151; SSSE3-NEXT:    shlq $50, %r9
3152; SSSE3-NEXT:    sarq $63, %r9
3153; SSSE3-NEXT:    movd %r9d, %xmm2
3154; SSSE3-NEXT:    shlq $51, %r10
3155; SSSE3-NEXT:    sarq $63, %r10
3156; SSSE3-NEXT:    movd %r10d, %xmm3
3157; SSSE3-NEXT:    shlq $52, %r11
3158; SSSE3-NEXT:    sarq $63, %r11
3159; SSSE3-NEXT:    movd %r11d, %xmm4
3160; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3161; SSSE3-NEXT:    shlq $53, %r14
3162; SSSE3-NEXT:    sarq $63, %r14
3163; SSSE3-NEXT:    movd %r14d, %xmm0
3164; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
3165; SSSE3-NEXT:    shlq $54, %r15
3166; SSSE3-NEXT:    sarq $63, %r15
3167; SSSE3-NEXT:    movd %r15d, %xmm2
3168; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3169; SSSE3-NEXT:    shlq $55, %r12
3170; SSSE3-NEXT:    sarq $63, %r12
3171; SSSE3-NEXT:    movd %r12d, %xmm1
3172; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3173; SSSE3-NEXT:    shlq $60, %r13
3174; SSSE3-NEXT:    sarq $63, %r13
3175; SSSE3-NEXT:    movd %r13d, %xmm4
3176; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3177; SSSE3-NEXT:    shlq $61, %rbx
3178; SSSE3-NEXT:    sarq $63, %rbx
3179; SSSE3-NEXT:    movd %ebx, %xmm2
3180; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3181; SSSE3-NEXT:    shlq $62, %rcx
3182; SSSE3-NEXT:    sarq $63, %rcx
3183; SSSE3-NEXT:    movd %ecx, %xmm5
3184; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
3185; SSSE3-NEXT:    shlq $63, %rdx
3186; SSSE3-NEXT:    sarq $63, %rdx
3187; SSSE3-NEXT:    movd %edx, %xmm0
3188; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
3189; SSSE3-NEXT:    shlq $58, %rsi
3190; SSSE3-NEXT:    sarq $63, %rsi
3191; SSSE3-NEXT:    movd %esi, %xmm3
3192; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
3193; SSSE3-NEXT:    shlq $59, %rdi
3194; SSSE3-NEXT:    sarq $63, %rdi
3195; SSSE3-NEXT:    movd %edi, %xmm4
3196; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3197; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
3198; SSSE3-NEXT:    shlq $57, %rbp
3199; SSSE3-NEXT:    sarq $63, %rbp
3200; SSSE3-NEXT:    movd %ebp, %xmm2
3201; SSSE3-NEXT:    shrq $7, %rax
3202; SSSE3-NEXT:    movd %eax, %xmm3
3203; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
3204; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
3205; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
3206; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
3207; SSSE3-NEXT:    popq %rbx
3208; SSSE3-NEXT:    popq %r12
3209; SSSE3-NEXT:    popq %r13
3210; SSSE3-NEXT:    popq %r14
3211; SSSE3-NEXT:    popq %r15
3212; SSSE3-NEXT:    popq %rbp
3213; SSSE3-NEXT:    retq
3214;
3215; SSE41-LABEL: load_sext_16i1_to_16i8:
3216; SSE41:       # %bb.0: # %entry
3217; SSE41-NEXT:    movswq (%rdi), %rax
3218; SSE41-NEXT:    movq %rax, %rcx
3219; SSE41-NEXT:    shlq $62, %rcx
3220; SSE41-NEXT:    sarq $63, %rcx
3221; SSE41-NEXT:    movq %rax, %rdx
3222; SSE41-NEXT:    shlq $63, %rdx
3223; SSE41-NEXT:    sarq $63, %rdx
3224; SSE41-NEXT:    movd %edx, %xmm0
3225; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
3226; SSE41-NEXT:    movq %rax, %rcx
3227; SSE41-NEXT:    shlq $61, %rcx
3228; SSE41-NEXT:    sarq $63, %rcx
3229; SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
3230; SSE41-NEXT:    movq %rax, %rcx
3231; SSE41-NEXT:    shlq $60, %rcx
3232; SSE41-NEXT:    sarq $63, %rcx
3233; SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
3234; SSE41-NEXT:    movq %rax, %rcx
3235; SSE41-NEXT:    shlq $59, %rcx
3236; SSE41-NEXT:    sarq $63, %rcx
3237; SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
3238; SSE41-NEXT:    movq %rax, %rcx
3239; SSE41-NEXT:    shlq $58, %rcx
3240; SSE41-NEXT:    sarq $63, %rcx
3241; SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
3242; SSE41-NEXT:    movq %rax, %rcx
3243; SSE41-NEXT:    shlq $57, %rcx
3244; SSE41-NEXT:    sarq $63, %rcx
3245; SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
3246; SSE41-NEXT:    movsbq %al, %rcx
3247; SSE41-NEXT:    shrq $7, %rcx
3248; SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
3249; SSE41-NEXT:    movq %rax, %rcx
3250; SSE41-NEXT:    shlq $55, %rcx
3251; SSE41-NEXT:    sarq $63, %rcx
3252; SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
3253; SSE41-NEXT:    movq %rax, %rcx
3254; SSE41-NEXT:    shlq $54, %rcx
3255; SSE41-NEXT:    sarq $63, %rcx
3256; SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
3257; SSE41-NEXT:    movq %rax, %rcx
3258; SSE41-NEXT:    shlq $53, %rcx
3259; SSE41-NEXT:    sarq $63, %rcx
3260; SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
3261; SSE41-NEXT:    movq %rax, %rcx
3262; SSE41-NEXT:    shlq $52, %rcx
3263; SSE41-NEXT:    sarq $63, %rcx
3264; SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
3265; SSE41-NEXT:    movq %rax, %rcx
3266; SSE41-NEXT:    shlq $51, %rcx
3267; SSE41-NEXT:    sarq $63, %rcx
3268; SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
3269; SSE41-NEXT:    movq %rax, %rcx
3270; SSE41-NEXT:    shlq $50, %rcx
3271; SSE41-NEXT:    sarq $63, %rcx
3272; SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
3273; SSE41-NEXT:    movq %rax, %rcx
3274; SSE41-NEXT:    shlq $49, %rcx
3275; SSE41-NEXT:    sarq $63, %rcx
3276; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
3277; SSE41-NEXT:    shrq $15, %rax
3278; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
3279; SSE41-NEXT:    retq
3280;
3281; AVX1-LABEL: load_sext_16i1_to_16i8:
3282; AVX1:       # %bb.0: # %entry
3283; AVX1-NEXT:    movswq (%rdi), %rax
3284; AVX1-NEXT:    movq %rax, %rcx
3285; AVX1-NEXT:    shlq $62, %rcx
3286; AVX1-NEXT:    sarq $63, %rcx
3287; AVX1-NEXT:    movq %rax, %rdx
3288; AVX1-NEXT:    shlq $63, %rdx
3289; AVX1-NEXT:    sarq $63, %rdx
3290; AVX1-NEXT:    vmovd %edx, %xmm0
3291; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
3292; AVX1-NEXT:    movq %rax, %rcx
3293; AVX1-NEXT:    shlq $61, %rcx
3294; AVX1-NEXT:    sarq $63, %rcx
3295; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0
3296; AVX1-NEXT:    movq %rax, %rcx
3297; AVX1-NEXT:    shlq $60, %rcx
3298; AVX1-NEXT:    sarq $63, %rcx
3299; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
3300; AVX1-NEXT:    movq %rax, %rcx
3301; AVX1-NEXT:    shlq $59, %rcx
3302; AVX1-NEXT:    sarq $63, %rcx
3303; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
3304; AVX1-NEXT:    movq %rax, %rcx
3305; AVX1-NEXT:    shlq $58, %rcx
3306; AVX1-NEXT:    sarq $63, %rcx
3307; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0
3308; AVX1-NEXT:    movq %rax, %rcx
3309; AVX1-NEXT:    shlq $57, %rcx
3310; AVX1-NEXT:    sarq $63, %rcx
3311; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0
3312; AVX1-NEXT:    movsbq %al, %rcx
3313; AVX1-NEXT:    shrq $7, %rcx
3314; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm0, %xmm0
3315; AVX1-NEXT:    movq %rax, %rcx
3316; AVX1-NEXT:    shlq $55, %rcx
3317; AVX1-NEXT:    sarq $63, %rcx
3318; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
3319; AVX1-NEXT:    movq %rax, %rcx
3320; AVX1-NEXT:    shlq $54, %rcx
3321; AVX1-NEXT:    sarq $63, %rcx
3322; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
3323; AVX1-NEXT:    movq %rax, %rcx
3324; AVX1-NEXT:    shlq $53, %rcx
3325; AVX1-NEXT:    sarq $63, %rcx
3326; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
3327; AVX1-NEXT:    movq %rax, %rcx
3328; AVX1-NEXT:    shlq $52, %rcx
3329; AVX1-NEXT:    sarq $63, %rcx
3330; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
3331; AVX1-NEXT:    movq %rax, %rcx
3332; AVX1-NEXT:    shlq $51, %rcx
3333; AVX1-NEXT:    sarq $63, %rcx
3334; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
3335; AVX1-NEXT:    movq %rax, %rcx
3336; AVX1-NEXT:    shlq $50, %rcx
3337; AVX1-NEXT:    sarq $63, %rcx
3338; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
3339; AVX1-NEXT:    movq %rax, %rcx
3340; AVX1-NEXT:    shlq $49, %rcx
3341; AVX1-NEXT:    sarq $63, %rcx
3342; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
3343; AVX1-NEXT:    shrq $15, %rax
3344; AVX1-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
3345; AVX1-NEXT:    retq
3346;
3347; AVX2-LABEL: load_sext_16i1_to_16i8:
3348; AVX2:       # %bb.0: # %entry
3349; AVX2-NEXT:    movswq (%rdi), %rax
3350; AVX2-NEXT:    movq %rax, %rcx
3351; AVX2-NEXT:    shlq $62, %rcx
3352; AVX2-NEXT:    sarq $63, %rcx
3353; AVX2-NEXT:    movq %rax, %rdx
3354; AVX2-NEXT:    shlq $63, %rdx
3355; AVX2-NEXT:    sarq $63, %rdx
3356; AVX2-NEXT:    vmovd %edx, %xmm0
3357; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
3358; AVX2-NEXT:    movq %rax, %rcx
3359; AVX2-NEXT:    shlq $61, %rcx
3360; AVX2-NEXT:    sarq $63, %rcx
3361; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0
3362; AVX2-NEXT:    movq %rax, %rcx
3363; AVX2-NEXT:    shlq $60, %rcx
3364; AVX2-NEXT:    sarq $63, %rcx
3365; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
3366; AVX2-NEXT:    movq %rax, %rcx
3367; AVX2-NEXT:    shlq $59, %rcx
3368; AVX2-NEXT:    sarq $63, %rcx
3369; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
3370; AVX2-NEXT:    movq %rax, %rcx
3371; AVX2-NEXT:    shlq $58, %rcx
3372; AVX2-NEXT:    sarq $63, %rcx
3373; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0
3374; AVX2-NEXT:    movq %rax, %rcx
3375; AVX2-NEXT:    shlq $57, %rcx
3376; AVX2-NEXT:    sarq $63, %rcx
3377; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0
3378; AVX2-NEXT:    movsbq %al, %rcx
3379; AVX2-NEXT:    shrq $7, %rcx
3380; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm0, %xmm0
3381; AVX2-NEXT:    movq %rax, %rcx
3382; AVX2-NEXT:    shlq $55, %rcx
3383; AVX2-NEXT:    sarq $63, %rcx
3384; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
3385; AVX2-NEXT:    movq %rax, %rcx
3386; AVX2-NEXT:    shlq $54, %rcx
3387; AVX2-NEXT:    sarq $63, %rcx
3388; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
3389; AVX2-NEXT:    movq %rax, %rcx
3390; AVX2-NEXT:    shlq $53, %rcx
3391; AVX2-NEXT:    sarq $63, %rcx
3392; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
3393; AVX2-NEXT:    movq %rax, %rcx
3394; AVX2-NEXT:    shlq $52, %rcx
3395; AVX2-NEXT:    sarq $63, %rcx
3396; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
3397; AVX2-NEXT:    movq %rax, %rcx
3398; AVX2-NEXT:    shlq $51, %rcx
3399; AVX2-NEXT:    sarq $63, %rcx
3400; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
3401; AVX2-NEXT:    movq %rax, %rcx
3402; AVX2-NEXT:    shlq $50, %rcx
3403; AVX2-NEXT:    sarq $63, %rcx
3404; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
3405; AVX2-NEXT:    movq %rax, %rcx
3406; AVX2-NEXT:    shlq $49, %rcx
3407; AVX2-NEXT:    sarq $63, %rcx
3408; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
3409; AVX2-NEXT:    shrq $15, %rax
3410; AVX2-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
3411; AVX2-NEXT:    retq
3412;
3413; AVX512F-LABEL: load_sext_16i1_to_16i8:
3414; AVX512F:       # %bb.0: # %entry
3415; AVX512F-NEXT:    kmovw (%rdi), %k1
3416; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
3417; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
3418; AVX512F-NEXT:    vzeroupper
3419; AVX512F-NEXT:    retq
3420;
3421; AVX512BW-LABEL: load_sext_16i1_to_16i8:
3422; AVX512BW:       # %bb.0: # %entry
3423; AVX512BW-NEXT:    kmovw (%rdi), %k0
3424; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
3425; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
3426; AVX512BW-NEXT:    vzeroupper
3427; AVX512BW-NEXT:    retq
3428;
3429; X32-SSE2-LABEL: load_sext_16i1_to_16i8:
3430; X32-SSE2:       # %bb.0: # %entry
3431; X32-SSE2-NEXT:    pushl %ebp
3432; X32-SSE2-NEXT:    pushl %ebx
3433; X32-SSE2-NEXT:    pushl %edi
3434; X32-SSE2-NEXT:    pushl %esi
3435; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3436; X32-SSE2-NEXT:    movswl (%eax), %eax
3437; X32-SSE2-NEXT:    movl %eax, %edx
3438; X32-SSE2-NEXT:    movl %eax, %ebp
3439; X32-SSE2-NEXT:    movl %eax, %ecx
3440; X32-SSE2-NEXT:    movl %eax, %esi
3441; X32-SSE2-NEXT:    movl %eax, %edi
3442; X32-SSE2-NEXT:    movl %eax, %ebx
3443; X32-SSE2-NEXT:    shrl $15, %ebx
3444; X32-SSE2-NEXT:    movd %ebx, %xmm2
3445; X32-SSE2-NEXT:    movl %eax, %ebx
3446; X32-SSE2-NEXT:    shll $17, %edx
3447; X32-SSE2-NEXT:    sarl $31, %edx
3448; X32-SSE2-NEXT:    movd %edx, %xmm0
3449; X32-SSE2-NEXT:    movl %eax, %edx
3450; X32-SSE2-NEXT:    shll $18, %ebp
3451; X32-SSE2-NEXT:    sarl $31, %ebp
3452; X32-SSE2-NEXT:    movd %ebp, %xmm1
3453; X32-SSE2-NEXT:    movl %eax, %ebp
3454; X32-SSE2-NEXT:    shll $19, %ecx
3455; X32-SSE2-NEXT:    sarl $31, %ecx
3456; X32-SSE2-NEXT:    movd %ecx, %xmm3
3457; X32-SSE2-NEXT:    movl %eax, %ecx
3458; X32-SSE2-NEXT:    shll $20, %esi
3459; X32-SSE2-NEXT:    sarl $31, %esi
3460; X32-SSE2-NEXT:    movd %esi, %xmm4
3461; X32-SSE2-NEXT:    movl %eax, %esi
3462; X32-SSE2-NEXT:    shll $21, %edi
3463; X32-SSE2-NEXT:    sarl $31, %edi
3464; X32-SSE2-NEXT:    movd %edi, %xmm6
3465; X32-SSE2-NEXT:    movl %eax, %edi
3466; X32-SSE2-NEXT:    shll $22, %ebx
3467; X32-SSE2-NEXT:    sarl $31, %ebx
3468; X32-SSE2-NEXT:    movd %ebx, %xmm7
3469; X32-SSE2-NEXT:    movl %eax, %ebx
3470; X32-SSE2-NEXT:    shll $23, %edx
3471; X32-SSE2-NEXT:    sarl $31, %edx
3472; X32-SSE2-NEXT:    movd %edx, %xmm5
3473; X32-SSE2-NEXT:    movl %eax, %edx
3474; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
3475; X32-SSE2-NEXT:    shll $28, %ebp
3476; X32-SSE2-NEXT:    sarl $31, %ebp
3477; X32-SSE2-NEXT:    movd %ebp, %xmm2
3478; X32-SSE2-NEXT:    movl %eax, %ebp
3479; X32-SSE2-NEXT:    movsbl %al, %eax
3480; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
3481; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
3482; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
3483; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
3484; X32-SSE2-NEXT:    shll $29, %ecx
3485; X32-SSE2-NEXT:    sarl $31, %ecx
3486; X32-SSE2-NEXT:    movd %ecx, %xmm1
3487; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3488; X32-SSE2-NEXT:    shll $30, %esi
3489; X32-SSE2-NEXT:    sarl $31, %esi
3490; X32-SSE2-NEXT:    movd %esi, %xmm4
3491; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
3492; X32-SSE2-NEXT:    shll $31, %edi
3493; X32-SSE2-NEXT:    sarl $31, %edi
3494; X32-SSE2-NEXT:    movd %edi, %xmm0
3495; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3496; X32-SSE2-NEXT:    shll $26, %ebx
3497; X32-SSE2-NEXT:    sarl $31, %ebx
3498; X32-SSE2-NEXT:    movd %ebx, %xmm2
3499; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3500; X32-SSE2-NEXT:    shll $27, %edx
3501; X32-SSE2-NEXT:    sarl $31, %edx
3502; X32-SSE2-NEXT:    movd %edx, %xmm3
3503; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3504; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
3505; X32-SSE2-NEXT:    shll $25, %ebp
3506; X32-SSE2-NEXT:    sarl $31, %ebp
3507; X32-SSE2-NEXT:    movd %ebp, %xmm1
3508; X32-SSE2-NEXT:    shrl $7, %eax
3509; X32-SSE2-NEXT:    movd %eax, %xmm2
3510; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3511; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3512; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3513; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
3514; X32-SSE2-NEXT:    popl %esi
3515; X32-SSE2-NEXT:    popl %edi
3516; X32-SSE2-NEXT:    popl %ebx
3517; X32-SSE2-NEXT:    popl %ebp
3518; X32-SSE2-NEXT:    retl
3519;
3520; X32-SSE41-LABEL: load_sext_16i1_to_16i8:
3521; X32-SSE41:       # %bb.0: # %entry
3522; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3523; X32-SSE41-NEXT:    movswl (%eax), %eax
3524; X32-SSE41-NEXT:    movl %eax, %ecx
3525; X32-SSE41-NEXT:    shll $30, %ecx
3526; X32-SSE41-NEXT:    sarl $31, %ecx
3527; X32-SSE41-NEXT:    movl %eax, %edx
3528; X32-SSE41-NEXT:    shll $31, %edx
3529; X32-SSE41-NEXT:    sarl $31, %edx
3530; X32-SSE41-NEXT:    movd %edx, %xmm0
3531; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
3532; X32-SSE41-NEXT:    movl %eax, %ecx
3533; X32-SSE41-NEXT:    shll $29, %ecx
3534; X32-SSE41-NEXT:    sarl $31, %ecx
3535; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
3536; X32-SSE41-NEXT:    movl %eax, %ecx
3537; X32-SSE41-NEXT:    shll $28, %ecx
3538; X32-SSE41-NEXT:    sarl $31, %ecx
3539; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
3540; X32-SSE41-NEXT:    movl %eax, %ecx
3541; X32-SSE41-NEXT:    shll $27, %ecx
3542; X32-SSE41-NEXT:    sarl $31, %ecx
3543; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
3544; X32-SSE41-NEXT:    movl %eax, %ecx
3545; X32-SSE41-NEXT:    shll $26, %ecx
3546; X32-SSE41-NEXT:    sarl $31, %ecx
3547; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
3548; X32-SSE41-NEXT:    movl %eax, %ecx
3549; X32-SSE41-NEXT:    shll $25, %ecx
3550; X32-SSE41-NEXT:    sarl $31, %ecx
3551; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
3552; X32-SSE41-NEXT:    movsbl %al, %ecx
3553; X32-SSE41-NEXT:    shrl $7, %ecx
3554; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
3555; X32-SSE41-NEXT:    movl %eax, %ecx
3556; X32-SSE41-NEXT:    shll $23, %ecx
3557; X32-SSE41-NEXT:    sarl $31, %ecx
3558; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
3559; X32-SSE41-NEXT:    movl %eax, %ecx
3560; X32-SSE41-NEXT:    shll $22, %ecx
3561; X32-SSE41-NEXT:    sarl $31, %ecx
3562; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
3563; X32-SSE41-NEXT:    movl %eax, %ecx
3564; X32-SSE41-NEXT:    shll $21, %ecx
3565; X32-SSE41-NEXT:    sarl $31, %ecx
3566; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
3567; X32-SSE41-NEXT:    movl %eax, %ecx
3568; X32-SSE41-NEXT:    shll $20, %ecx
3569; X32-SSE41-NEXT:    sarl $31, %ecx
3570; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
3571; X32-SSE41-NEXT:    movl %eax, %ecx
3572; X32-SSE41-NEXT:    shll $19, %ecx
3573; X32-SSE41-NEXT:    sarl $31, %ecx
3574; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
3575; X32-SSE41-NEXT:    movl %eax, %ecx
3576; X32-SSE41-NEXT:    shll $18, %ecx
3577; X32-SSE41-NEXT:    sarl $31, %ecx
3578; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
3579; X32-SSE41-NEXT:    movl %eax, %ecx
3580; X32-SSE41-NEXT:    shll $17, %ecx
3581; X32-SSE41-NEXT:    sarl $31, %ecx
3582; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
3583; X32-SSE41-NEXT:    shrl $15, %eax
3584; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm0
3585; X32-SSE41-NEXT:    retl
3586entry:
3587 %X = load <16 x i1>, <16 x i1>* %ptr
3588 %Y = sext <16 x i1> %X to <16 x i8>
3589 ret <16 x i8> %Y
3590}
3591
3592define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
3593; SSE2-LABEL: load_sext_16i1_to_16i16:
3594; SSE2:       # %bb.0: # %entry
3595; SSE2-NEXT:    movzwl (%rdi), %eax
3596; SSE2-NEXT:    movl %eax, %ecx
3597; SSE2-NEXT:    shrl $15, %ecx
3598; SSE2-NEXT:    movd %ecx, %xmm0
3599; SSE2-NEXT:    movl %eax, %ecx
3600; SSE2-NEXT:    shrl $14, %ecx
3601; SSE2-NEXT:    andl $1, %ecx
3602; SSE2-NEXT:    movd %ecx, %xmm1
3603; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3604; SSE2-NEXT:    movl %eax, %ecx
3605; SSE2-NEXT:    shrl $13, %ecx
3606; SSE2-NEXT:    andl $1, %ecx
3607; SSE2-NEXT:    movd %ecx, %xmm0
3608; SSE2-NEXT:    movl %eax, %ecx
3609; SSE2-NEXT:    shrl $12, %ecx
3610; SSE2-NEXT:    andl $1, %ecx
3611; SSE2-NEXT:    movd %ecx, %xmm2
3612; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3613; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3614; SSE2-NEXT:    movl %eax, %ecx
3615; SSE2-NEXT:    shrl $11, %ecx
3616; SSE2-NEXT:    andl $1, %ecx
3617; SSE2-NEXT:    movd %ecx, %xmm0
3618; SSE2-NEXT:    movl %eax, %ecx
3619; SSE2-NEXT:    shrl $10, %ecx
3620; SSE2-NEXT:    andl $1, %ecx
3621; SSE2-NEXT:    movd %ecx, %xmm3
3622; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3623; SSE2-NEXT:    movl %eax, %ecx
3624; SSE2-NEXT:    shrl $9, %ecx
3625; SSE2-NEXT:    andl $1, %ecx
3626; SSE2-NEXT:    movd %ecx, %xmm0
3627; SSE2-NEXT:    movl %eax, %ecx
3628; SSE2-NEXT:    shrl $8, %ecx
3629; SSE2-NEXT:    andl $1, %ecx
3630; SSE2-NEXT:    movd %ecx, %xmm1
3631; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3632; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
3633; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3634; SSE2-NEXT:    movl %eax, %ecx
3635; SSE2-NEXT:    shrl $7, %ecx
3636; SSE2-NEXT:    andl $1, %ecx
3637; SSE2-NEXT:    movd %ecx, %xmm0
3638; SSE2-NEXT:    movl %eax, %ecx
3639; SSE2-NEXT:    shrl $6, %ecx
3640; SSE2-NEXT:    andl $1, %ecx
3641; SSE2-NEXT:    movd %ecx, %xmm2
3642; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3643; SSE2-NEXT:    movl %eax, %ecx
3644; SSE2-NEXT:    shrl $5, %ecx
3645; SSE2-NEXT:    andl $1, %ecx
3646; SSE2-NEXT:    movd %ecx, %xmm0
3647; SSE2-NEXT:    movl %eax, %ecx
3648; SSE2-NEXT:    shrl $4, %ecx
3649; SSE2-NEXT:    andl $1, %ecx
3650; SSE2-NEXT:    movd %ecx, %xmm3
3651; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3652; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
3653; SSE2-NEXT:    movl %eax, %ecx
3654; SSE2-NEXT:    shrl $3, %ecx
3655; SSE2-NEXT:    andl $1, %ecx
3656; SSE2-NEXT:    movd %ecx, %xmm0
3657; SSE2-NEXT:    movl %eax, %ecx
3658; SSE2-NEXT:    shrl $2, %ecx
3659; SSE2-NEXT:    andl $1, %ecx
3660; SSE2-NEXT:    movd %ecx, %xmm2
3661; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3662; SSE2-NEXT:    movl %eax, %ecx
3663; SSE2-NEXT:    andl $1, %ecx
3664; SSE2-NEXT:    movd %ecx, %xmm0
3665; SSE2-NEXT:    shrl %eax
3666; SSE2-NEXT:    andl $1, %eax
3667; SSE2-NEXT:    movd %eax, %xmm4
3668; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3669; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3670; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3671; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3672; SSE2-NEXT:    psllw $15, %xmm0
3673; SSE2-NEXT:    psraw $15, %xmm0
3674; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3675; SSE2-NEXT:    psllw $15, %xmm1
3676; SSE2-NEXT:    psraw $15, %xmm1
3677; SSE2-NEXT:    retq
3678;
3679; SSSE3-LABEL: load_sext_16i1_to_16i16:
3680; SSSE3:       # %bb.0: # %entry
3681; SSSE3-NEXT:    movzwl (%rdi), %eax
3682; SSSE3-NEXT:    movl %eax, %ecx
3683; SSSE3-NEXT:    shrl $15, %ecx
3684; SSSE3-NEXT:    movd %ecx, %xmm0
3685; SSSE3-NEXT:    movl %eax, %ecx
3686; SSSE3-NEXT:    shrl $14, %ecx
3687; SSSE3-NEXT:    andl $1, %ecx
3688; SSSE3-NEXT:    movd %ecx, %xmm1
3689; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3690; SSSE3-NEXT:    movl %eax, %ecx
3691; SSSE3-NEXT:    shrl $13, %ecx
3692; SSSE3-NEXT:    andl $1, %ecx
3693; SSSE3-NEXT:    movd %ecx, %xmm0
3694; SSSE3-NEXT:    movl %eax, %ecx
3695; SSSE3-NEXT:    shrl $12, %ecx
3696; SSSE3-NEXT:    andl $1, %ecx
3697; SSSE3-NEXT:    movd %ecx, %xmm2
3698; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3699; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3700; SSSE3-NEXT:    movl %eax, %ecx
3701; SSSE3-NEXT:    shrl $11, %ecx
3702; SSSE3-NEXT:    andl $1, %ecx
3703; SSSE3-NEXT:    movd %ecx, %xmm0
3704; SSSE3-NEXT:    movl %eax, %ecx
3705; SSSE3-NEXT:    shrl $10, %ecx
3706; SSSE3-NEXT:    andl $1, %ecx
3707; SSSE3-NEXT:    movd %ecx, %xmm3
3708; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3709; SSSE3-NEXT:    movl %eax, %ecx
3710; SSSE3-NEXT:    shrl $9, %ecx
3711; SSSE3-NEXT:    andl $1, %ecx
3712; SSSE3-NEXT:    movd %ecx, %xmm0
3713; SSSE3-NEXT:    movl %eax, %ecx
3714; SSSE3-NEXT:    shrl $8, %ecx
3715; SSSE3-NEXT:    andl $1, %ecx
3716; SSSE3-NEXT:    movd %ecx, %xmm1
3717; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3718; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
3719; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3720; SSSE3-NEXT:    movl %eax, %ecx
3721; SSSE3-NEXT:    shrl $7, %ecx
3722; SSSE3-NEXT:    andl $1, %ecx
3723; SSSE3-NEXT:    movd %ecx, %xmm0
3724; SSSE3-NEXT:    movl %eax, %ecx
3725; SSSE3-NEXT:    shrl $6, %ecx
3726; SSSE3-NEXT:    andl $1, %ecx
3727; SSSE3-NEXT:    movd %ecx, %xmm2
3728; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3729; SSSE3-NEXT:    movl %eax, %ecx
3730; SSSE3-NEXT:    shrl $5, %ecx
3731; SSSE3-NEXT:    andl $1, %ecx
3732; SSSE3-NEXT:    movd %ecx, %xmm0
3733; SSSE3-NEXT:    movl %eax, %ecx
3734; SSSE3-NEXT:    shrl $4, %ecx
3735; SSSE3-NEXT:    andl $1, %ecx
3736; SSSE3-NEXT:    movd %ecx, %xmm3
3737; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3738; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
3739; SSSE3-NEXT:    movl %eax, %ecx
3740; SSSE3-NEXT:    shrl $3, %ecx
3741; SSSE3-NEXT:    andl $1, %ecx
3742; SSSE3-NEXT:    movd %ecx, %xmm0
3743; SSSE3-NEXT:    movl %eax, %ecx
3744; SSSE3-NEXT:    shrl $2, %ecx
3745; SSSE3-NEXT:    andl $1, %ecx
3746; SSSE3-NEXT:    movd %ecx, %xmm2
3747; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3748; SSSE3-NEXT:    movl %eax, %ecx
3749; SSSE3-NEXT:    andl $1, %ecx
3750; SSSE3-NEXT:    movd %ecx, %xmm0
3751; SSSE3-NEXT:    shrl %eax
3752; SSSE3-NEXT:    andl $1, %eax
3753; SSSE3-NEXT:    movd %eax, %xmm4
3754; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3755; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3756; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3757; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3758; SSSE3-NEXT:    psllw $15, %xmm0
3759; SSSE3-NEXT:    psraw $15, %xmm0
3760; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3761; SSSE3-NEXT:    psllw $15, %xmm1
3762; SSSE3-NEXT:    psraw $15, %xmm1
3763; SSSE3-NEXT:    retq
3764;
3765; SSE41-LABEL: load_sext_16i1_to_16i16:
3766; SSE41:       # %bb.0: # %entry
3767; SSE41-NEXT:    movzwl (%rdi), %eax
3768; SSE41-NEXT:    movl %eax, %ecx
3769; SSE41-NEXT:    shrl %ecx
3770; SSE41-NEXT:    andl $1, %ecx
3771; SSE41-NEXT:    movl %eax, %edx
3772; SSE41-NEXT:    andl $1, %edx
3773; SSE41-NEXT:    movd %edx, %xmm1
3774; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
3775; SSE41-NEXT:    movl %eax, %ecx
3776; SSE41-NEXT:    shrl $2, %ecx
3777; SSE41-NEXT:    andl $1, %ecx
3778; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
3779; SSE41-NEXT:    movl %eax, %ecx
3780; SSE41-NEXT:    shrl $3, %ecx
3781; SSE41-NEXT:    andl $1, %ecx
3782; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
3783; SSE41-NEXT:    movl %eax, %ecx
3784; SSE41-NEXT:    shrl $4, %ecx
3785; SSE41-NEXT:    andl $1, %ecx
3786; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
3787; SSE41-NEXT:    movl %eax, %ecx
3788; SSE41-NEXT:    shrl $5, %ecx
3789; SSE41-NEXT:    andl $1, %ecx
3790; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
3791; SSE41-NEXT:    movl %eax, %ecx
3792; SSE41-NEXT:    shrl $6, %ecx
3793; SSE41-NEXT:    andl $1, %ecx
3794; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
3795; SSE41-NEXT:    movl %eax, %ecx
3796; SSE41-NEXT:    shrl $7, %ecx
3797; SSE41-NEXT:    andl $1, %ecx
3798; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
3799; SSE41-NEXT:    movl %eax, %ecx
3800; SSE41-NEXT:    shrl $8, %ecx
3801; SSE41-NEXT:    andl $1, %ecx
3802; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
3803; SSE41-NEXT:    movl %eax, %ecx
3804; SSE41-NEXT:    shrl $9, %ecx
3805; SSE41-NEXT:    andl $1, %ecx
3806; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
3807; SSE41-NEXT:    movl %eax, %ecx
3808; SSE41-NEXT:    shrl $10, %ecx
3809; SSE41-NEXT:    andl $1, %ecx
3810; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
3811; SSE41-NEXT:    movl %eax, %ecx
3812; SSE41-NEXT:    shrl $11, %ecx
3813; SSE41-NEXT:    andl $1, %ecx
3814; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
3815; SSE41-NEXT:    movl %eax, %ecx
3816; SSE41-NEXT:    shrl $12, %ecx
3817; SSE41-NEXT:    andl $1, %ecx
3818; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
3819; SSE41-NEXT:    movl %eax, %ecx
3820; SSE41-NEXT:    shrl $13, %ecx
3821; SSE41-NEXT:    andl $1, %ecx
3822; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
3823; SSE41-NEXT:    movl %eax, %ecx
3824; SSE41-NEXT:    shrl $14, %ecx
3825; SSE41-NEXT:    andl $1, %ecx
3826; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
3827; SSE41-NEXT:    shrl $15, %eax
3828; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
3829; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3830; SSE41-NEXT:    psllw $15, %xmm0
3831; SSE41-NEXT:    psraw $15, %xmm0
3832; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
3833; SSE41-NEXT:    psllw $15, %xmm1
3834; SSE41-NEXT:    psraw $15, %xmm1
3835; SSE41-NEXT:    retq
3836;
3837; AVX1-LABEL: load_sext_16i1_to_16i16:
3838; AVX1:       # %bb.0: # %entry
3839; AVX1-NEXT:    pushq %rbp
3840; AVX1-NEXT:    .cfi_def_cfa_offset 16
3841; AVX1-NEXT:    pushq %r15
3842; AVX1-NEXT:    .cfi_def_cfa_offset 24
3843; AVX1-NEXT:    pushq %r14
3844; AVX1-NEXT:    .cfi_def_cfa_offset 32
3845; AVX1-NEXT:    pushq %r13
3846; AVX1-NEXT:    .cfi_def_cfa_offset 40
3847; AVX1-NEXT:    pushq %r12
3848; AVX1-NEXT:    .cfi_def_cfa_offset 48
3849; AVX1-NEXT:    pushq %rbx
3850; AVX1-NEXT:    .cfi_def_cfa_offset 56
3851; AVX1-NEXT:    .cfi_offset %rbx, -56
3852; AVX1-NEXT:    .cfi_offset %r12, -48
3853; AVX1-NEXT:    .cfi_offset %r13, -40
3854; AVX1-NEXT:    .cfi_offset %r14, -32
3855; AVX1-NEXT:    .cfi_offset %r15, -24
3856; AVX1-NEXT:    .cfi_offset %rbp, -16
3857; AVX1-NEXT:    movswq (%rdi), %rax
3858; AVX1-NEXT:    movq %rax, %rcx
3859; AVX1-NEXT:    shlq $55, %rcx
3860; AVX1-NEXT:    sarq $63, %rcx
3861; AVX1-NEXT:    vmovd %ecx, %xmm0
3862; AVX1-NEXT:    movq %rax, %r8
3863; AVX1-NEXT:    movq %rax, %r10
3864; AVX1-NEXT:    movq %rax, %r11
3865; AVX1-NEXT:    movq %rax, %r14
3866; AVX1-NEXT:    movq %rax, %r15
3867; AVX1-NEXT:    movq %rax, %r9
3868; AVX1-NEXT:    movq %rax, %r12
3869; AVX1-NEXT:    movq %rax, %r13
3870; AVX1-NEXT:    movq %rax, %rbx
3871; AVX1-NEXT:    movq %rax, %rdi
3872; AVX1-NEXT:    movq %rax, %rcx
3873; AVX1-NEXT:    movq %rax, %rdx
3874; AVX1-NEXT:    movq %rax, %rsi
3875; AVX1-NEXT:    movsbq %al, %rbp
3876; AVX1-NEXT:    shlq $54, %rax
3877; AVX1-NEXT:    sarq $63, %rax
3878; AVX1-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
3879; AVX1-NEXT:    shlq $53, %r8
3880; AVX1-NEXT:    sarq $63, %r8
3881; AVX1-NEXT:    vpinsrw $2, %r8d, %xmm0, %xmm0
3882; AVX1-NEXT:    shlq $52, %r10
3883; AVX1-NEXT:    sarq $63, %r10
3884; AVX1-NEXT:    vpinsrw $3, %r10d, %xmm0, %xmm0
3885; AVX1-NEXT:    shlq $51, %r11
3886; AVX1-NEXT:    sarq $63, %r11
3887; AVX1-NEXT:    vpinsrw $4, %r11d, %xmm0, %xmm0
3888; AVX1-NEXT:    shlq $50, %r14
3889; AVX1-NEXT:    sarq $63, %r14
3890; AVX1-NEXT:    vpinsrw $5, %r14d, %xmm0, %xmm0
3891; AVX1-NEXT:    shlq $49, %r15
3892; AVX1-NEXT:    sarq $63, %r15
3893; AVX1-NEXT:    vpinsrw $6, %r15d, %xmm0, %xmm0
3894; AVX1-NEXT:    shrq $15, %r9
3895; AVX1-NEXT:    vpinsrw $7, %r9d, %xmm0, %xmm0
3896; AVX1-NEXT:    shlq $63, %r13
3897; AVX1-NEXT:    sarq $63, %r13
3898; AVX1-NEXT:    vmovd %r13d, %xmm1
3899; AVX1-NEXT:    shlq $62, %r12
3900; AVX1-NEXT:    sarq $63, %r12
3901; AVX1-NEXT:    vpinsrw $1, %r12d, %xmm1, %xmm1
3902; AVX1-NEXT:    shlq $61, %rbx
3903; AVX1-NEXT:    sarq $63, %rbx
3904; AVX1-NEXT:    vpinsrw $2, %ebx, %xmm1, %xmm1
3905; AVX1-NEXT:    shlq $60, %rdi
3906; AVX1-NEXT:    sarq $63, %rdi
3907; AVX1-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
3908; AVX1-NEXT:    shlq $59, %rcx
3909; AVX1-NEXT:    sarq $63, %rcx
3910; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
3911; AVX1-NEXT:    shlq $58, %rdx
3912; AVX1-NEXT:    sarq $63, %rdx
3913; AVX1-NEXT:    vpinsrw $5, %edx, %xmm1, %xmm1
3914; AVX1-NEXT:    shlq $57, %rsi
3915; AVX1-NEXT:    sarq $63, %rsi
3916; AVX1-NEXT:    vpinsrw $6, %esi, %xmm1, %xmm1
3917; AVX1-NEXT:    shrq $7, %rbp
3918; AVX1-NEXT:    vpinsrw $7, %ebp, %xmm1, %xmm1
3919; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
3920; AVX1-NEXT:    popq %rbx
3921; AVX1-NEXT:    .cfi_def_cfa_offset 48
3922; AVX1-NEXT:    popq %r12
3923; AVX1-NEXT:    .cfi_def_cfa_offset 40
3924; AVX1-NEXT:    popq %r13
3925; AVX1-NEXT:    .cfi_def_cfa_offset 32
3926; AVX1-NEXT:    popq %r14
3927; AVX1-NEXT:    .cfi_def_cfa_offset 24
3928; AVX1-NEXT:    popq %r15
3929; AVX1-NEXT:    .cfi_def_cfa_offset 16
3930; AVX1-NEXT:    popq %rbp
3931; AVX1-NEXT:    .cfi_def_cfa_offset 8
3932; AVX1-NEXT:    retq
3933;
3934; AVX2-LABEL: load_sext_16i1_to_16i16:
3935; AVX2:       # %bb.0: # %entry
3936; AVX2-NEXT:    pushq %rbp
3937; AVX2-NEXT:    .cfi_def_cfa_offset 16
3938; AVX2-NEXT:    pushq %r15
3939; AVX2-NEXT:    .cfi_def_cfa_offset 24
3940; AVX2-NEXT:    pushq %r14
3941; AVX2-NEXT:    .cfi_def_cfa_offset 32
3942; AVX2-NEXT:    pushq %r13
3943; AVX2-NEXT:    .cfi_def_cfa_offset 40
3944; AVX2-NEXT:    pushq %r12
3945; AVX2-NEXT:    .cfi_def_cfa_offset 48
3946; AVX2-NEXT:    pushq %rbx
3947; AVX2-NEXT:    .cfi_def_cfa_offset 56
3948; AVX2-NEXT:    .cfi_offset %rbx, -56
3949; AVX2-NEXT:    .cfi_offset %r12, -48
3950; AVX2-NEXT:    .cfi_offset %r13, -40
3951; AVX2-NEXT:    .cfi_offset %r14, -32
3952; AVX2-NEXT:    .cfi_offset %r15, -24
3953; AVX2-NEXT:    .cfi_offset %rbp, -16
3954; AVX2-NEXT:    movswq (%rdi), %rax
3955; AVX2-NEXT:    movq %rax, %rcx
3956; AVX2-NEXT:    shlq $55, %rcx
3957; AVX2-NEXT:    sarq $63, %rcx
3958; AVX2-NEXT:    vmovd %ecx, %xmm0
3959; AVX2-NEXT:    movq %rax, %r8
3960; AVX2-NEXT:    movq %rax, %r10
3961; AVX2-NEXT:    movq %rax, %r11
3962; AVX2-NEXT:    movq %rax, %r14
3963; AVX2-NEXT:    movq %rax, %r15
3964; AVX2-NEXT:    movq %rax, %r9
3965; AVX2-NEXT:    movq %rax, %r12
3966; AVX2-NEXT:    movq %rax, %r13
3967; AVX2-NEXT:    movq %rax, %rbx
3968; AVX2-NEXT:    movq %rax, %rdi
3969; AVX2-NEXT:    movq %rax, %rcx
3970; AVX2-NEXT:    movq %rax, %rdx
3971; AVX2-NEXT:    movq %rax, %rsi
3972; AVX2-NEXT:    movsbq %al, %rbp
3973; AVX2-NEXT:    shlq $54, %rax
3974; AVX2-NEXT:    sarq $63, %rax
3975; AVX2-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
3976; AVX2-NEXT:    shlq $53, %r8
3977; AVX2-NEXT:    sarq $63, %r8
3978; AVX2-NEXT:    vpinsrw $2, %r8d, %xmm0, %xmm0
3979; AVX2-NEXT:    shlq $52, %r10
3980; AVX2-NEXT:    sarq $63, %r10
3981; AVX2-NEXT:    vpinsrw $3, %r10d, %xmm0, %xmm0
3982; AVX2-NEXT:    shlq $51, %r11
3983; AVX2-NEXT:    sarq $63, %r11
3984; AVX2-NEXT:    vpinsrw $4, %r11d, %xmm0, %xmm0
3985; AVX2-NEXT:    shlq $50, %r14
3986; AVX2-NEXT:    sarq $63, %r14
3987; AVX2-NEXT:    vpinsrw $5, %r14d, %xmm0, %xmm0
3988; AVX2-NEXT:    shlq $49, %r15
3989; AVX2-NEXT:    sarq $63, %r15
3990; AVX2-NEXT:    vpinsrw $6, %r15d, %xmm0, %xmm0
3991; AVX2-NEXT:    shrq $15, %r9
3992; AVX2-NEXT:    vpinsrw $7, %r9d, %xmm0, %xmm0
3993; AVX2-NEXT:    shlq $63, %r13
3994; AVX2-NEXT:    sarq $63, %r13
3995; AVX2-NEXT:    vmovd %r13d, %xmm1
3996; AVX2-NEXT:    shlq $62, %r12
3997; AVX2-NEXT:    sarq $63, %r12
3998; AVX2-NEXT:    vpinsrw $1, %r12d, %xmm1, %xmm1
3999; AVX2-NEXT:    shlq $61, %rbx
4000; AVX2-NEXT:    sarq $63, %rbx
4001; AVX2-NEXT:    vpinsrw $2, %ebx, %xmm1, %xmm1
4002; AVX2-NEXT:    shlq $60, %rdi
4003; AVX2-NEXT:    sarq $63, %rdi
4004; AVX2-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
4005; AVX2-NEXT:    shlq $59, %rcx
4006; AVX2-NEXT:    sarq $63, %rcx
4007; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
4008; AVX2-NEXT:    shlq $58, %rdx
4009; AVX2-NEXT:    sarq $63, %rdx
4010; AVX2-NEXT:    vpinsrw $5, %edx, %xmm1, %xmm1
4011; AVX2-NEXT:    shlq $57, %rsi
4012; AVX2-NEXT:    sarq $63, %rsi
4013; AVX2-NEXT:    vpinsrw $6, %esi, %xmm1, %xmm1
4014; AVX2-NEXT:    shrq $7, %rbp
4015; AVX2-NEXT:    vpinsrw $7, %ebp, %xmm1, %xmm1
4016; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
4017; AVX2-NEXT:    popq %rbx
4018; AVX2-NEXT:    .cfi_def_cfa_offset 48
4019; AVX2-NEXT:    popq %r12
4020; AVX2-NEXT:    .cfi_def_cfa_offset 40
4021; AVX2-NEXT:    popq %r13
4022; AVX2-NEXT:    .cfi_def_cfa_offset 32
4023; AVX2-NEXT:    popq %r14
4024; AVX2-NEXT:    .cfi_def_cfa_offset 24
4025; AVX2-NEXT:    popq %r15
4026; AVX2-NEXT:    .cfi_def_cfa_offset 16
4027; AVX2-NEXT:    popq %rbp
4028; AVX2-NEXT:    .cfi_def_cfa_offset 8
4029; AVX2-NEXT:    retq
4030;
4031; AVX512F-LABEL: load_sext_16i1_to_16i16:
4032; AVX512F:       # %bb.0: # %entry
4033; AVX512F-NEXT:    kmovw (%rdi), %k1
4034; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
4035; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
4036; AVX512F-NEXT:    retq
4037;
4038; AVX512BW-LABEL: load_sext_16i1_to_16i16:
4039; AVX512BW:       # %bb.0: # %entry
4040; AVX512BW-NEXT:    kmovw (%rdi), %k0
4041; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
4042; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
4043; AVX512BW-NEXT:    retq
4044;
4045; X32-SSE2-LABEL: load_sext_16i1_to_16i16:
4046; X32-SSE2:       # %bb.0: # %entry
4047; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
4048; X32-SSE2-NEXT:    movzwl (%eax), %eax
4049; X32-SSE2-NEXT:    movl %eax, %ecx
4050; X32-SSE2-NEXT:    shrl $15, %ecx
4051; X32-SSE2-NEXT:    movd %ecx, %xmm0
4052; X32-SSE2-NEXT:    movl %eax, %ecx
4053; X32-SSE2-NEXT:    shrl $14, %ecx
4054; X32-SSE2-NEXT:    andl $1, %ecx
4055; X32-SSE2-NEXT:    movd %ecx, %xmm1
4056; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
4057; X32-SSE2-NEXT:    movl %eax, %ecx
4058; X32-SSE2-NEXT:    shrl $13, %ecx
4059; X32-SSE2-NEXT:    andl $1, %ecx
4060; X32-SSE2-NEXT:    movd %ecx, %xmm0
4061; X32-SSE2-NEXT:    movl %eax, %ecx
4062; X32-SSE2-NEXT:    shrl $12, %ecx
4063; X32-SSE2-NEXT:    andl $1, %ecx
4064; X32-SSE2-NEXT:    movd %ecx, %xmm2
4065; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
4066; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4067; X32-SSE2-NEXT:    movl %eax, %ecx
4068; X32-SSE2-NEXT:    shrl $11, %ecx
4069; X32-SSE2-NEXT:    andl $1, %ecx
4070; X32-SSE2-NEXT:    movd %ecx, %xmm0
4071; X32-SSE2-NEXT:    movl %eax, %ecx
4072; X32-SSE2-NEXT:    shrl $10, %ecx
4073; X32-SSE2-NEXT:    andl $1, %ecx
4074; X32-SSE2-NEXT:    movd %ecx, %xmm3
4075; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
4076; X32-SSE2-NEXT:    movl %eax, %ecx
4077; X32-SSE2-NEXT:    shrl $9, %ecx
4078; X32-SSE2-NEXT:    andl $1, %ecx
4079; X32-SSE2-NEXT:    movd %ecx, %xmm0
4080; X32-SSE2-NEXT:    movl %eax, %ecx
4081; X32-SSE2-NEXT:    shrl $8, %ecx
4082; X32-SSE2-NEXT:    andl $1, %ecx
4083; X32-SSE2-NEXT:    movd %ecx, %xmm1
4084; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
4085; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
4086; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
4087; X32-SSE2-NEXT:    movl %eax, %ecx
4088; X32-SSE2-NEXT:    shrl $7, %ecx
4089; X32-SSE2-NEXT:    andl $1, %ecx
4090; X32-SSE2-NEXT:    movd %ecx, %xmm0
4091; X32-SSE2-NEXT:    movl %eax, %ecx
4092; X32-SSE2-NEXT:    shrl $6, %ecx
4093; X32-SSE2-NEXT:    andl $1, %ecx
4094; X32-SSE2-NEXT:    movd %ecx, %xmm2
4095; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
4096; X32-SSE2-NEXT:    movl %eax, %ecx
4097; X32-SSE2-NEXT:    shrl $5, %ecx
4098; X32-SSE2-NEXT:    andl $1, %ecx
4099; X32-SSE2-NEXT:    movd %ecx, %xmm0
4100; X32-SSE2-NEXT:    movl %eax, %ecx
4101; X32-SSE2-NEXT:    shrl $4, %ecx
4102; X32-SSE2-NEXT:    andl $1, %ecx
4103; X32-SSE2-NEXT:    movd %ecx, %xmm3
4104; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
4105; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
4106; X32-SSE2-NEXT:    movl %eax, %ecx
4107; X32-SSE2-NEXT:    shrl $3, %ecx
4108; X32-SSE2-NEXT:    andl $1, %ecx
4109; X32-SSE2-NEXT:    movd %ecx, %xmm0
4110; X32-SSE2-NEXT:    movl %eax, %ecx
4111; X32-SSE2-NEXT:    shrl $2, %ecx
4112; X32-SSE2-NEXT:    andl $1, %ecx
4113; X32-SSE2-NEXT:    movd %ecx, %xmm2
4114; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
4115; X32-SSE2-NEXT:    movl %eax, %ecx
4116; X32-SSE2-NEXT:    andl $1, %ecx
4117; X32-SSE2-NEXT:    movd %ecx, %xmm0
4118; X32-SSE2-NEXT:    shrl %eax
4119; X32-SSE2-NEXT:    andl $1, %eax
4120; X32-SSE2-NEXT:    movd %eax, %xmm4
4121; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
4122; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
4123; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
4124; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
4125; X32-SSE2-NEXT:    psllw $15, %xmm0
4126; X32-SSE2-NEXT:    psraw $15, %xmm0
4127; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
4128; X32-SSE2-NEXT:    psllw $15, %xmm1
4129; X32-SSE2-NEXT:    psraw $15, %xmm1
4130; X32-SSE2-NEXT:    retl
4131;
4132; X32-SSE41-LABEL: load_sext_16i1_to_16i16:
4133; X32-SSE41:       # %bb.0: # %entry
4134; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
4135; X32-SSE41-NEXT:    movzwl (%eax), %eax
4136; X32-SSE41-NEXT:    movl %eax, %ecx
4137; X32-SSE41-NEXT:    shrl %ecx
4138; X32-SSE41-NEXT:    andl $1, %ecx
4139; X32-SSE41-NEXT:    movl %eax, %edx
4140; X32-SSE41-NEXT:    andl $1, %edx
4141; X32-SSE41-NEXT:    movd %edx, %xmm1
4142; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
4143; X32-SSE41-NEXT:    movl %eax, %ecx
4144; X32-SSE41-NEXT:    shrl $2, %ecx
4145; X32-SSE41-NEXT:    andl $1, %ecx
4146; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
4147; X32-SSE41-NEXT:    movl %eax, %ecx
4148; X32-SSE41-NEXT:    shrl $3, %ecx
4149; X32-SSE41-NEXT:    andl $1, %ecx
4150; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
4151; X32-SSE41-NEXT:    movl %eax, %ecx
4152; X32-SSE41-NEXT:    shrl $4, %ecx
4153; X32-SSE41-NEXT:    andl $1, %ecx
4154; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
4155; X32-SSE41-NEXT:    movl %eax, %ecx
4156; X32-SSE41-NEXT:    shrl $5, %ecx
4157; X32-SSE41-NEXT:    andl $1, %ecx
4158; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
4159; X32-SSE41-NEXT:    movl %eax, %ecx
4160; X32-SSE41-NEXT:    shrl $6, %ecx
4161; X32-SSE41-NEXT:    andl $1, %ecx
4162; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
4163; X32-SSE41-NEXT:    movl %eax, %ecx
4164; X32-SSE41-NEXT:    shrl $7, %ecx
4165; X32-SSE41-NEXT:    andl $1, %ecx
4166; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
4167; X32-SSE41-NEXT:    movl %eax, %ecx
4168; X32-SSE41-NEXT:    shrl $8, %ecx
4169; X32-SSE41-NEXT:    andl $1, %ecx
4170; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
4171; X32-SSE41-NEXT:    movl %eax, %ecx
4172; X32-SSE41-NEXT:    shrl $9, %ecx
4173; X32-SSE41-NEXT:    andl $1, %ecx
4174; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
4175; X32-SSE41-NEXT:    movl %eax, %ecx
4176; X32-SSE41-NEXT:    shrl $10, %ecx
4177; X32-SSE41-NEXT:    andl $1, %ecx
4178; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
4179; X32-SSE41-NEXT:    movl %eax, %ecx
4180; X32-SSE41-NEXT:    shrl $11, %ecx
4181; X32-SSE41-NEXT:    andl $1, %ecx
4182; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
4183; X32-SSE41-NEXT:    movl %eax, %ecx
4184; X32-SSE41-NEXT:    shrl $12, %ecx
4185; X32-SSE41-NEXT:    andl $1, %ecx
4186; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
4187; X32-SSE41-NEXT:    movl %eax, %ecx
4188; X32-SSE41-NEXT:    shrl $13, %ecx
4189; X32-SSE41-NEXT:    andl $1, %ecx
4190; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
4191; X32-SSE41-NEXT:    movl %eax, %ecx
4192; X32-SSE41-NEXT:    shrl $14, %ecx
4193; X32-SSE41-NEXT:    andl $1, %ecx
4194; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
4195; X32-SSE41-NEXT:    shrl $15, %eax
4196; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm1
4197; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
4198; X32-SSE41-NEXT:    psllw $15, %xmm0
4199; X32-SSE41-NEXT:    psraw $15, %xmm0
4200; X32-SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
4201; X32-SSE41-NEXT:    psllw $15, %xmm1
4202; X32-SSE41-NEXT:    psraw $15, %xmm1
4203; X32-SSE41-NEXT:    retl
4204entry:
4205 %X = load <16 x i1>, <16 x i1>* %ptr
4206 %Y = sext <16 x i1> %X to <16 x i16>
4207 ret <16 x i16> %Y
4208}
4209
4210define <32 x i8> @load_sext_32i1_to_32i8(<32 x i1> *%ptr) nounwind readnone {
4211; SSE2-LABEL: load_sext_32i1_to_32i8:
4212; SSE2:       # %bb.0: # %entry
4213; SSE2-NEXT:    pushq %rbp
4214; SSE2-NEXT:    pushq %r15
4215; SSE2-NEXT:    pushq %r14
4216; SSE2-NEXT:    pushq %r13
4217; SSE2-NEXT:    pushq %r12
4218; SSE2-NEXT:    pushq %rbx
4219; SSE2-NEXT:    movswq (%rdi), %rax
4220; SSE2-NEXT:    movq %rax, %r10
4221; SSE2-NEXT:    movq %rax, %r8
4222; SSE2-NEXT:    movq %rax, %r9
4223; SSE2-NEXT:    movq %rax, %r11
4224; SSE2-NEXT:    movq %rax, %r14
4225; SSE2-NEXT:    movq %rax, %r15
4226; SSE2-NEXT:    movq %rax, %r12
4227; SSE2-NEXT:    movq %rax, %r13
4228; SSE2-NEXT:    movq %rax, %rdx
4229; SSE2-NEXT:    movq %rax, %rsi
4230; SSE2-NEXT:    movq %rax, %rcx
4231; SSE2-NEXT:    movq %rax, %rbp
4232; SSE2-NEXT:    movq %rax, %rbx
4233; SSE2-NEXT:    shrq $15, %rbx
4234; SSE2-NEXT:    movd %ebx, %xmm0
4235; SSE2-NEXT:    movq %rax, %rbx
4236; SSE2-NEXT:    shlq $49, %r10
4237; SSE2-NEXT:    sarq $63, %r10
4238; SSE2-NEXT:    movd %r10d, %xmm15
4239; SSE2-NEXT:    movq %rax, %r10
4240; SSE2-NEXT:    movsbq %al, %rax
4241; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
4242; SSE2-NEXT:    shlq $50, %r8
4243; SSE2-NEXT:    sarq $63, %r8
4244; SSE2-NEXT:    movd %r8d, %xmm8
4245; SSE2-NEXT:    shlq $51, %r9
4246; SSE2-NEXT:    sarq $63, %r9
4247; SSE2-NEXT:    movd %r9d, %xmm3
4248; SSE2-NEXT:    shlq $52, %r11
4249; SSE2-NEXT:    sarq $63, %r11
4250; SSE2-NEXT:    movd %r11d, %xmm9
4251; SSE2-NEXT:    shlq $53, %r14
4252; SSE2-NEXT:    sarq $63, %r14
4253; SSE2-NEXT:    movd %r14d, %xmm6
4254; SSE2-NEXT:    shlq $54, %r15
4255; SSE2-NEXT:    sarq $63, %r15
4256; SSE2-NEXT:    movd %r15d, %xmm10
4257; SSE2-NEXT:    shlq $55, %r12
4258; SSE2-NEXT:    sarq $63, %r12
4259; SSE2-NEXT:    movd %r12d, %xmm2
4260; SSE2-NEXT:    shlq $60, %r13
4261; SSE2-NEXT:    sarq $63, %r13
4262; SSE2-NEXT:    movd %r13d, %xmm11
4263; SSE2-NEXT:    shlq $61, %rdx
4264; SSE2-NEXT:    sarq $63, %rdx
4265; SSE2-NEXT:    movd %edx, %xmm5
4266; SSE2-NEXT:    shlq $62, %rsi
4267; SSE2-NEXT:    sarq $63, %rsi
4268; SSE2-NEXT:    movd %esi, %xmm12
4269; SSE2-NEXT:    shlq $63, %rcx
4270; SSE2-NEXT:    sarq $63, %rcx
4271; SSE2-NEXT:    movd %ecx, %xmm0
4272; SSE2-NEXT:    shlq $58, %rbp
4273; SSE2-NEXT:    sarq $63, %rbp
4274; SSE2-NEXT:    movd %ebp, %xmm13
4275; SSE2-NEXT:    shlq $59, %rbx
4276; SSE2-NEXT:    sarq $63, %rbx
4277; SSE2-NEXT:    movd %ebx, %xmm7
4278; SSE2-NEXT:    shlq $57, %r10
4279; SSE2-NEXT:    sarq $63, %r10
4280; SSE2-NEXT:    movd %r10d, %xmm4
4281; SSE2-NEXT:    shrq $7, %rax
4282; SSE2-NEXT:    movd %eax, %xmm14
4283; SSE2-NEXT:    movswq 2(%rdi), %rsi
4284; SSE2-NEXT:    movq %rsi, %r8
4285; SSE2-NEXT:    movq %rsi, %r9
4286; SSE2-NEXT:    movq %rsi, %r10
4287; SSE2-NEXT:    movq %rsi, %r11
4288; SSE2-NEXT:    movq %rsi, %r14
4289; SSE2-NEXT:    movq %rsi, %r15
4290; SSE2-NEXT:    movq %rsi, %r12
4291; SSE2-NEXT:    movq %rsi, %r13
4292; SSE2-NEXT:    movq %rsi, %rbx
4293; SSE2-NEXT:    movq %rsi, %rax
4294; SSE2-NEXT:    movq %rsi, %rcx
4295; SSE2-NEXT:    movq %rsi, %rdx
4296; SSE2-NEXT:    movq %rsi, %rdi
4297; SSE2-NEXT:    movq %rsi, %rbp
4298; SSE2-NEXT:    shrq $15, %rbp
4299; SSE2-NEXT:    movd %ebp, %xmm1
4300; SSE2-NEXT:    movq %rsi, %rbp
4301; SSE2-NEXT:    movsbq %sil, %rsi
4302; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
4303; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
4304; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7]
4305; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]
4306; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
4307; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4308; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3],xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]
4309; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
4310; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
4311; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
4312; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
4313; SSE2-NEXT:    shlq $49, %r8
4314; SSE2-NEXT:    sarq $63, %r8
4315; SSE2-NEXT:    movd %r8d, %xmm3
4316; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
4317; SSE2-NEXT:    shlq $50, %r9
4318; SSE2-NEXT:    sarq $63, %r9
4319; SSE2-NEXT:    movd %r9d, %xmm4
4320; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
4321; SSE2-NEXT:    shlq $51, %r10
4322; SSE2-NEXT:    sarq $63, %r10
4323; SSE2-NEXT:    movd %r10d, %xmm5
4324; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
4325; SSE2-NEXT:    shlq $52, %r11
4326; SSE2-NEXT:    sarq $63, %r11
4327; SSE2-NEXT:    movd %r11d, %xmm2
4328; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
4329; SSE2-NEXT:    shlq $53, %r14
4330; SSE2-NEXT:    sarq $63, %r14
4331; SSE2-NEXT:    movd %r14d, %xmm1
4332; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
4333; SSE2-NEXT:    shlq $54, %r15
4334; SSE2-NEXT:    sarq $63, %r15
4335; SSE2-NEXT:    movd %r15d, %xmm4
4336; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
4337; SSE2-NEXT:    shlq $55, %r12
4338; SSE2-NEXT:    sarq $63, %r12
4339; SSE2-NEXT:    movd %r12d, %xmm3
4340; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
4341; SSE2-NEXT:    shlq $60, %r13
4342; SSE2-NEXT:    sarq $63, %r13
4343; SSE2-NEXT:    movd %r13d, %xmm2
4344; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
4345; SSE2-NEXT:    shlq $61, %rbx
4346; SSE2-NEXT:    sarq $63, %rbx
4347; SSE2-NEXT:    movd %ebx, %xmm4
4348; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
4349; SSE2-NEXT:    shlq $62, %rax
4350; SSE2-NEXT:    sarq $63, %rax
4351; SSE2-NEXT:    movd %eax, %xmm6
4352; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
4353; SSE2-NEXT:    shlq $63, %rcx
4354; SSE2-NEXT:    sarq $63, %rcx
4355; SSE2-NEXT:    movd %ecx, %xmm1
4356; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4357; SSE2-NEXT:    shlq $58, %rdx
4358; SSE2-NEXT:    sarq $63, %rdx
4359; SSE2-NEXT:    movd %edx, %xmm2
4360; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
4361; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
4362; SSE2-NEXT:    shlq $59, %rdi
4363; SSE2-NEXT:    sarq $63, %rdi
4364; SSE2-NEXT:    movd %edi, %xmm4
4365; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4366; SSE2-NEXT:    shlq $57, %rbp
4367; SSE2-NEXT:    sarq $63, %rbp
4368; SSE2-NEXT:    movd %ebp, %xmm2
4369; SSE2-NEXT:    shrq $7, %rsi
4370; SSE2-NEXT:    movd %esi, %xmm5
4371; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4372; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
4373; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
4374; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
4375; SSE2-NEXT:    popq %rbx
4376; SSE2-NEXT:    popq %r12
4377; SSE2-NEXT:    popq %r13
4378; SSE2-NEXT:    popq %r14
4379; SSE2-NEXT:    popq %r15
4380; SSE2-NEXT:    popq %rbp
4381; SSE2-NEXT:    retq
4382;
4383; SSSE3-LABEL: load_sext_32i1_to_32i8:
4384; SSSE3:       # %bb.0: # %entry
4385; SSSE3-NEXT:    pushq %rbp
4386; SSSE3-NEXT:    pushq %r15
4387; SSSE3-NEXT:    pushq %r14
4388; SSSE3-NEXT:    pushq %r13
4389; SSSE3-NEXT:    pushq %r12
4390; SSSE3-NEXT:    pushq %rbx
4391; SSSE3-NEXT:    movswq (%rdi), %rax
4392; SSSE3-NEXT:    movq %rax, %r10
4393; SSSE3-NEXT:    movq %rax, %r8
4394; SSSE3-NEXT:    movq %rax, %r9
4395; SSSE3-NEXT:    movq %rax, %r11
4396; SSSE3-NEXT:    movq %rax, %r14
4397; SSSE3-NEXT:    movq %rax, %r15
4398; SSSE3-NEXT:    movq %rax, %r12
4399; SSSE3-NEXT:    movq %rax, %r13
4400; SSSE3-NEXT:    movq %rax, %rdx
4401; SSSE3-NEXT:    movq %rax, %rsi
4402; SSSE3-NEXT:    movq %rax, %rcx
4403; SSSE3-NEXT:    movq %rax, %rbp
4404; SSSE3-NEXT:    movq %rax, %rbx
4405; SSSE3-NEXT:    shrq $15, %rbx
4406; SSSE3-NEXT:    movd %ebx, %xmm0
4407; SSSE3-NEXT:    movq %rax, %rbx
4408; SSSE3-NEXT:    shlq $49, %r10
4409; SSSE3-NEXT:    sarq $63, %r10
4410; SSSE3-NEXT:    movd %r10d, %xmm15
4411; SSSE3-NEXT:    movq %rax, %r10
4412; SSSE3-NEXT:    movsbq %al, %rax
4413; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
4414; SSSE3-NEXT:    shlq $50, %r8
4415; SSSE3-NEXT:    sarq $63, %r8
4416; SSSE3-NEXT:    movd %r8d, %xmm8
4417; SSSE3-NEXT:    shlq $51, %r9
4418; SSSE3-NEXT:    sarq $63, %r9
4419; SSSE3-NEXT:    movd %r9d, %xmm3
4420; SSSE3-NEXT:    shlq $52, %r11
4421; SSSE3-NEXT:    sarq $63, %r11
4422; SSSE3-NEXT:    movd %r11d, %xmm9
4423; SSSE3-NEXT:    shlq $53, %r14
4424; SSSE3-NEXT:    sarq $63, %r14
4425; SSSE3-NEXT:    movd %r14d, %xmm6
4426; SSSE3-NEXT:    shlq $54, %r15
4427; SSSE3-NEXT:    sarq $63, %r15
4428; SSSE3-NEXT:    movd %r15d, %xmm10
4429; SSSE3-NEXT:    shlq $55, %r12
4430; SSSE3-NEXT:    sarq $63, %r12
4431; SSSE3-NEXT:    movd %r12d, %xmm2
4432; SSSE3-NEXT:    shlq $60, %r13
4433; SSSE3-NEXT:    sarq $63, %r13
4434; SSSE3-NEXT:    movd %r13d, %xmm11
4435; SSSE3-NEXT:    shlq $61, %rdx
4436; SSSE3-NEXT:    sarq $63, %rdx
4437; SSSE3-NEXT:    movd %edx, %xmm5
4438; SSSE3-NEXT:    shlq $62, %rsi
4439; SSSE3-NEXT:    sarq $63, %rsi
4440; SSSE3-NEXT:    movd %esi, %xmm12
4441; SSSE3-NEXT:    shlq $63, %rcx
4442; SSSE3-NEXT:    sarq $63, %rcx
4443; SSSE3-NEXT:    movd %ecx, %xmm0
4444; SSSE3-NEXT:    shlq $58, %rbp
4445; SSSE3-NEXT:    sarq $63, %rbp
4446; SSSE3-NEXT:    movd %ebp, %xmm13
4447; SSSE3-NEXT:    shlq $59, %rbx
4448; SSSE3-NEXT:    sarq $63, %rbx
4449; SSSE3-NEXT:    movd %ebx, %xmm7
4450; SSSE3-NEXT:    shlq $57, %r10
4451; SSSE3-NEXT:    sarq $63, %r10
4452; SSSE3-NEXT:    movd %r10d, %xmm4
4453; SSSE3-NEXT:    shrq $7, %rax
4454; SSSE3-NEXT:    movd %eax, %xmm14
4455; SSSE3-NEXT:    movswq 2(%rdi), %rsi
4456; SSSE3-NEXT:    movq %rsi, %r8
4457; SSSE3-NEXT:    movq %rsi, %r9
4458; SSSE3-NEXT:    movq %rsi, %r10
4459; SSSE3-NEXT:    movq %rsi, %r11
4460; SSSE3-NEXT:    movq %rsi, %r14
4461; SSSE3-NEXT:    movq %rsi, %r15
4462; SSSE3-NEXT:    movq %rsi, %r12
4463; SSSE3-NEXT:    movq %rsi, %r13
4464; SSSE3-NEXT:    movq %rsi, %rbx
4465; SSSE3-NEXT:    movq %rsi, %rax
4466; SSSE3-NEXT:    movq %rsi, %rcx
4467; SSSE3-NEXT:    movq %rsi, %rdx
4468; SSSE3-NEXT:    movq %rsi, %rdi
4469; SSSE3-NEXT:    movq %rsi, %rbp
4470; SSSE3-NEXT:    shrq $15, %rbp
4471; SSSE3-NEXT:    movd %ebp, %xmm1
4472; SSSE3-NEXT:    movq %rsi, %rbp
4473; SSSE3-NEXT:    movsbq %sil, %rsi
4474; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
4475; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
4476; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7]
4477; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]
4478; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
4479; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4480; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3],xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]
4481; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
4482; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
4483; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
4484; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
4485; SSSE3-NEXT:    shlq $49, %r8
4486; SSSE3-NEXT:    sarq $63, %r8
4487; SSSE3-NEXT:    movd %r8d, %xmm3
4488; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
4489; SSSE3-NEXT:    shlq $50, %r9
4490; SSSE3-NEXT:    sarq $63, %r9
4491; SSSE3-NEXT:    movd %r9d, %xmm4
4492; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
4493; SSSE3-NEXT:    shlq $51, %r10
4494; SSSE3-NEXT:    sarq $63, %r10
4495; SSSE3-NEXT:    movd %r10d, %xmm5
4496; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
4497; SSSE3-NEXT:    shlq $52, %r11
4498; SSSE3-NEXT:    sarq $63, %r11
4499; SSSE3-NEXT:    movd %r11d, %xmm2
4500; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
4501; SSSE3-NEXT:    shlq $53, %r14
4502; SSSE3-NEXT:    sarq $63, %r14
4503; SSSE3-NEXT:    movd %r14d, %xmm1
4504; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
4505; SSSE3-NEXT:    shlq $54, %r15
4506; SSSE3-NEXT:    sarq $63, %r15
4507; SSSE3-NEXT:    movd %r15d, %xmm4
4508; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
4509; SSSE3-NEXT:    shlq $55, %r12
4510; SSSE3-NEXT:    sarq $63, %r12
4511; SSSE3-NEXT:    movd %r12d, %xmm3
4512; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
4513; SSSE3-NEXT:    shlq $60, %r13
4514; SSSE3-NEXT:    sarq $63, %r13
4515; SSSE3-NEXT:    movd %r13d, %xmm2
4516; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
4517; SSSE3-NEXT:    shlq $61, %rbx
4518; SSSE3-NEXT:    sarq $63, %rbx
4519; SSSE3-NEXT:    movd %ebx, %xmm4
4520; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
4521; SSSE3-NEXT:    shlq $62, %rax
4522; SSSE3-NEXT:    sarq $63, %rax
4523; SSSE3-NEXT:    movd %eax, %xmm6
4524; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
4525; SSSE3-NEXT:    shlq $63, %rcx
4526; SSSE3-NEXT:    sarq $63, %rcx
4527; SSSE3-NEXT:    movd %ecx, %xmm1
4528; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4529; SSSE3-NEXT:    shlq $58, %rdx
4530; SSSE3-NEXT:    sarq $63, %rdx
4531; SSSE3-NEXT:    movd %edx, %xmm2
4532; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
4533; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
4534; SSSE3-NEXT:    shlq $59, %rdi
4535; SSSE3-NEXT:    sarq $63, %rdi
4536; SSSE3-NEXT:    movd %edi, %xmm4
4537; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4538; SSSE3-NEXT:    shlq $57, %rbp
4539; SSSE3-NEXT:    sarq $63, %rbp
4540; SSSE3-NEXT:    movd %ebp, %xmm2
4541; SSSE3-NEXT:    shrq $7, %rsi
4542; SSSE3-NEXT:    movd %esi, %xmm5
4543; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4544; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
4545; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
4546; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
4547; SSSE3-NEXT:    popq %rbx
4548; SSSE3-NEXT:    popq %r12
4549; SSSE3-NEXT:    popq %r13
4550; SSSE3-NEXT:    popq %r14
4551; SSSE3-NEXT:    popq %r15
4552; SSSE3-NEXT:    popq %rbp
4553; SSSE3-NEXT:    retq
4554;
4555; SSE41-LABEL: load_sext_32i1_to_32i8:
4556; SSE41:       # %bb.0: # %entry
4557; SSE41-NEXT:    movswq (%rdi), %rax
4558; SSE41-NEXT:    movq %rax, %rcx
4559; SSE41-NEXT:    shlq $62, %rcx
4560; SSE41-NEXT:    sarq $63, %rcx
4561; SSE41-NEXT:    movq %rax, %rdx
4562; SSE41-NEXT:    shlq $63, %rdx
4563; SSE41-NEXT:    sarq $63, %rdx
4564; SSE41-NEXT:    movd %edx, %xmm0
4565; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
4566; SSE41-NEXT:    movq %rax, %rcx
4567; SSE41-NEXT:    shlq $61, %rcx
4568; SSE41-NEXT:    sarq $63, %rcx
4569; SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
4570; SSE41-NEXT:    movq %rax, %rcx
4571; SSE41-NEXT:    shlq $60, %rcx
4572; SSE41-NEXT:    sarq $63, %rcx
4573; SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
4574; SSE41-NEXT:    movq %rax, %rcx
4575; SSE41-NEXT:    shlq $59, %rcx
4576; SSE41-NEXT:    sarq $63, %rcx
4577; SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
4578; SSE41-NEXT:    movq %rax, %rcx
4579; SSE41-NEXT:    shlq $58, %rcx
4580; SSE41-NEXT:    sarq $63, %rcx
4581; SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
4582; SSE41-NEXT:    movq %rax, %rcx
4583; SSE41-NEXT:    shlq $57, %rcx
4584; SSE41-NEXT:    sarq $63, %rcx
4585; SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
4586; SSE41-NEXT:    movsbq %al, %rcx
4587; SSE41-NEXT:    shrq $7, %rcx
4588; SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
4589; SSE41-NEXT:    movq %rax, %rcx
4590; SSE41-NEXT:    shlq $55, %rcx
4591; SSE41-NEXT:    sarq $63, %rcx
4592; SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
4593; SSE41-NEXT:    movq %rax, %rcx
4594; SSE41-NEXT:    shlq $54, %rcx
4595; SSE41-NEXT:    sarq $63, %rcx
4596; SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
4597; SSE41-NEXT:    movq %rax, %rcx
4598; SSE41-NEXT:    shlq $53, %rcx
4599; SSE41-NEXT:    sarq $63, %rcx
4600; SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
4601; SSE41-NEXT:    movq %rax, %rcx
4602; SSE41-NEXT:    shlq $52, %rcx
4603; SSE41-NEXT:    sarq $63, %rcx
4604; SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
4605; SSE41-NEXT:    movq %rax, %rcx
4606; SSE41-NEXT:    shlq $51, %rcx
4607; SSE41-NEXT:    sarq $63, %rcx
4608; SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
4609; SSE41-NEXT:    movq %rax, %rcx
4610; SSE41-NEXT:    shlq $50, %rcx
4611; SSE41-NEXT:    sarq $63, %rcx
4612; SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
4613; SSE41-NEXT:    movq %rax, %rcx
4614; SSE41-NEXT:    shlq $49, %rcx
4615; SSE41-NEXT:    sarq $63, %rcx
4616; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
4617; SSE41-NEXT:    shrq $15, %rax
4618; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
4619; SSE41-NEXT:    movswq 2(%rdi), %rax
4620; SSE41-NEXT:    movq %rax, %rcx
4621; SSE41-NEXT:    shlq $62, %rcx
4622; SSE41-NEXT:    sarq $63, %rcx
4623; SSE41-NEXT:    movq %rax, %rdx
4624; SSE41-NEXT:    shlq $63, %rdx
4625; SSE41-NEXT:    sarq $63, %rdx
4626; SSE41-NEXT:    movd %edx, %xmm1
4627; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
4628; SSE41-NEXT:    movq %rax, %rcx
4629; SSE41-NEXT:    shlq $61, %rcx
4630; SSE41-NEXT:    sarq $63, %rcx
4631; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
4632; SSE41-NEXT:    movq %rax, %rcx
4633; SSE41-NEXT:    shlq $60, %rcx
4634; SSE41-NEXT:    sarq $63, %rcx
4635; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
4636; SSE41-NEXT:    movq %rax, %rcx
4637; SSE41-NEXT:    shlq $59, %rcx
4638; SSE41-NEXT:    sarq $63, %rcx
4639; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
4640; SSE41-NEXT:    movq %rax, %rcx
4641; SSE41-NEXT:    shlq $58, %rcx
4642; SSE41-NEXT:    sarq $63, %rcx
4643; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
4644; SSE41-NEXT:    movq %rax, %rcx
4645; SSE41-NEXT:    shlq $57, %rcx
4646; SSE41-NEXT:    sarq $63, %rcx
4647; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
4648; SSE41-NEXT:    movsbq %al, %rcx
4649; SSE41-NEXT:    shrq $7, %rcx
4650; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
4651; SSE41-NEXT:    movq %rax, %rcx
4652; SSE41-NEXT:    shlq $55, %rcx
4653; SSE41-NEXT:    sarq $63, %rcx
4654; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
4655; SSE41-NEXT:    movq %rax, %rcx
4656; SSE41-NEXT:    shlq $54, %rcx
4657; SSE41-NEXT:    sarq $63, %rcx
4658; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
4659; SSE41-NEXT:    movq %rax, %rcx
4660; SSE41-NEXT:    shlq $53, %rcx
4661; SSE41-NEXT:    sarq $63, %rcx
4662; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
4663; SSE41-NEXT:    movq %rax, %rcx
4664; SSE41-NEXT:    shlq $52, %rcx
4665; SSE41-NEXT:    sarq $63, %rcx
4666; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
4667; SSE41-NEXT:    movq %rax, %rcx
4668; SSE41-NEXT:    shlq $51, %rcx
4669; SSE41-NEXT:    sarq $63, %rcx
4670; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
4671; SSE41-NEXT:    movq %rax, %rcx
4672; SSE41-NEXT:    shlq $50, %rcx
4673; SSE41-NEXT:    sarq $63, %rcx
4674; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
4675; SSE41-NEXT:    movq %rax, %rcx
4676; SSE41-NEXT:    shlq $49, %rcx
4677; SSE41-NEXT:    sarq $63, %rcx
4678; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
4679; SSE41-NEXT:    shrq $15, %rax
4680; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
4681; SSE41-NEXT:    retq
4682;
4683; AVX1-LABEL: load_sext_32i1_to_32i8:
4684; AVX1:       # %bb.0: # %entry
4685; AVX1-NEXT:    pushq %rbp
4686; AVX1-NEXT:    pushq %r15
4687; AVX1-NEXT:    pushq %r14
4688; AVX1-NEXT:    pushq %r13
4689; AVX1-NEXT:    pushq %r12
4690; AVX1-NEXT:    pushq %rbx
4691; AVX1-NEXT:    movslq (%rdi), %rax
4692; AVX1-NEXT:    movq %rax, %rcx
4693; AVX1-NEXT:    shlq $47, %rcx
4694; AVX1-NEXT:    sarq $63, %rcx
4695; AVX1-NEXT:    vmovd %ecx, %xmm0
4696; AVX1-NEXT:    movq %rax, %r8
4697; AVX1-NEXT:    movq %rax, %rdx
4698; AVX1-NEXT:    movq %rax, %rcx
4699; AVX1-NEXT:    movq %rax, %rdi
4700; AVX1-NEXT:    movq %rax, %r13
4701; AVX1-NEXT:    movq %rax, %rsi
4702; AVX1-NEXT:    movq %rax, %r10
4703; AVX1-NEXT:    movq %rax, %r11
4704; AVX1-NEXT:    movq %rax, %r9
4705; AVX1-NEXT:    movq %rax, %rbx
4706; AVX1-NEXT:    movq %rax, %r14
4707; AVX1-NEXT:    movq %rax, %r15
4708; AVX1-NEXT:    movq %rax, %r12
4709; AVX1-NEXT:    movq %rax, %rbp
4710; AVX1-NEXT:    shlq $46, %rbp
4711; AVX1-NEXT:    sarq $63, %rbp
4712; AVX1-NEXT:    vpinsrb $1, %ebp, %xmm0, %xmm0
4713; AVX1-NEXT:    movq %rax, %rbp
4714; AVX1-NEXT:    shlq $45, %r8
4715; AVX1-NEXT:    sarq $63, %r8
4716; AVX1-NEXT:    vpinsrb $2, %r8d, %xmm0, %xmm0
4717; AVX1-NEXT:    movq %rax, %r8
4718; AVX1-NEXT:    shlq $44, %rdx
4719; AVX1-NEXT:    sarq $63, %rdx
4720; AVX1-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
4721; AVX1-NEXT:    movq %rax, %rdx
4722; AVX1-NEXT:    shlq $43, %rcx
4723; AVX1-NEXT:    sarq $63, %rcx
4724; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
4725; AVX1-NEXT:    movq %rax, %rcx
4726; AVX1-NEXT:    shlq $42, %rdi
4727; AVX1-NEXT:    sarq $63, %rdi
4728; AVX1-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
4729; AVX1-NEXT:    movq %rax, %rdi
4730; AVX1-NEXT:    shlq $41, %r13
4731; AVX1-NEXT:    sarq $63, %r13
4732; AVX1-NEXT:    vpinsrb $6, %r13d, %xmm0, %xmm0
4733; AVX1-NEXT:    movq %rax, %r13
4734; AVX1-NEXT:    shlq $40, %rsi
4735; AVX1-NEXT:    sarq $63, %rsi
4736; AVX1-NEXT:    vpinsrb $7, %esi, %xmm0, %xmm0
4737; AVX1-NEXT:    movq %rax, %rsi
4738; AVX1-NEXT:    shlq $39, %r10
4739; AVX1-NEXT:    sarq $63, %r10
4740; AVX1-NEXT:    vpinsrb $8, %r10d, %xmm0, %xmm0
4741; AVX1-NEXT:    movq %rax, %r10
4742; AVX1-NEXT:    shlq $38, %r11
4743; AVX1-NEXT:    sarq $63, %r11
4744; AVX1-NEXT:    vpinsrb $9, %r11d, %xmm0, %xmm0
4745; AVX1-NEXT:    movsbq %al, %r11
4746; AVX1-NEXT:    shlq $37, %r9
4747; AVX1-NEXT:    sarq $63, %r9
4748; AVX1-NEXT:    vpinsrb $10, %r9d, %xmm0, %xmm0
4749; AVX1-NEXT:    movq %rax, %r9
4750; AVX1-NEXT:    shlq $36, %rbx
4751; AVX1-NEXT:    sarq $63, %rbx
4752; AVX1-NEXT:    vpinsrb $11, %ebx, %xmm0, %xmm0
4753; AVX1-NEXT:    movq %rax, %rbx
4754; AVX1-NEXT:    shlq $35, %r14
4755; AVX1-NEXT:    sarq $63, %r14
4756; AVX1-NEXT:    vpinsrb $12, %r14d, %xmm0, %xmm0
4757; AVX1-NEXT:    movq %rax, %r14
4758; AVX1-NEXT:    shlq $34, %r15
4759; AVX1-NEXT:    sarq $63, %r15
4760; AVX1-NEXT:    vpinsrb $13, %r15d, %xmm0, %xmm0
4761; AVX1-NEXT:    movq %rax, %r15
4762; AVX1-NEXT:    shlq $33, %r12
4763; AVX1-NEXT:    sarq $63, %r12
4764; AVX1-NEXT:    vpinsrb $14, %r12d, %xmm0, %xmm0
4765; AVX1-NEXT:    movq %rax, %r12
4766; AVX1-NEXT:    shrq $31, %rbp
4767; AVX1-NEXT:    vpinsrb $15, %ebp, %xmm0, %xmm0
4768; AVX1-NEXT:    movq %rax, %rbp
4769; AVX1-NEXT:    shlq $63, %rdx
4770; AVX1-NEXT:    sarq $63, %rdx
4771; AVX1-NEXT:    vmovd %edx, %xmm1
4772; AVX1-NEXT:    movq %rax, %rdx
4773; AVX1-NEXT:    movswq %ax, %rax
4774; AVX1-NEXT:    shlq $62, %r8
4775; AVX1-NEXT:    sarq $63, %r8
4776; AVX1-NEXT:    vpinsrb $1, %r8d, %xmm1, %xmm1
4777; AVX1-NEXT:    shlq $61, %rcx
4778; AVX1-NEXT:    sarq $63, %rcx
4779; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
4780; AVX1-NEXT:    shlq $60, %rdi
4781; AVX1-NEXT:    sarq $63, %rdi
4782; AVX1-NEXT:    vpinsrb $3, %edi, %xmm1, %xmm1
4783; AVX1-NEXT:    shlq $59, %r13
4784; AVX1-NEXT:    sarq $63, %r13
4785; AVX1-NEXT:    vpinsrb $4, %r13d, %xmm1, %xmm1
4786; AVX1-NEXT:    shlq $58, %rsi
4787; AVX1-NEXT:    sarq $63, %rsi
4788; AVX1-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
4789; AVX1-NEXT:    shlq $57, %r10
4790; AVX1-NEXT:    sarq $63, %r10
4791; AVX1-NEXT:    vpinsrb $6, %r10d, %xmm1, %xmm1
4792; AVX1-NEXT:    shrq $7, %r11
4793; AVX1-NEXT:    vpinsrb $7, %r11d, %xmm1, %xmm1
4794; AVX1-NEXT:    shlq $55, %r9
4795; AVX1-NEXT:    sarq $63, %r9
4796; AVX1-NEXT:    vpinsrb $8, %r9d, %xmm1, %xmm1
4797; AVX1-NEXT:    shlq $54, %rbx
4798; AVX1-NEXT:    sarq $63, %rbx
4799; AVX1-NEXT:    vpinsrb $9, %ebx, %xmm1, %xmm1
4800; AVX1-NEXT:    shlq $53, %r14
4801; AVX1-NEXT:    sarq $63, %r14
4802; AVX1-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
4803; AVX1-NEXT:    shlq $52, %r15
4804; AVX1-NEXT:    sarq $63, %r15
4805; AVX1-NEXT:    vpinsrb $11, %r15d, %xmm1, %xmm1
4806; AVX1-NEXT:    shlq $51, %r12
4807; AVX1-NEXT:    sarq $63, %r12
4808; AVX1-NEXT:    vpinsrb $12, %r12d, %xmm1, %xmm1
4809; AVX1-NEXT:    shlq $50, %rbp
4810; AVX1-NEXT:    sarq $63, %rbp
4811; AVX1-NEXT:    vpinsrb $13, %ebp, %xmm1, %xmm1
4812; AVX1-NEXT:    shlq $49, %rdx
4813; AVX1-NEXT:    sarq $63, %rdx
4814; AVX1-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
4815; AVX1-NEXT:    shrq $15, %rax
4816; AVX1-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
4817; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
4818; AVX1-NEXT:    popq %rbx
4819; AVX1-NEXT:    popq %r12
4820; AVX1-NEXT:    popq %r13
4821; AVX1-NEXT:    popq %r14
4822; AVX1-NEXT:    popq %r15
4823; AVX1-NEXT:    popq %rbp
4824; AVX1-NEXT:    retq
4825;
4826; AVX2-LABEL: load_sext_32i1_to_32i8:
4827; AVX2:       # %bb.0: # %entry
4828; AVX2-NEXT:    pushq %rbp
4829; AVX2-NEXT:    pushq %r15
4830; AVX2-NEXT:    pushq %r14
4831; AVX2-NEXT:    pushq %r13
4832; AVX2-NEXT:    pushq %r12
4833; AVX2-NEXT:    pushq %rbx
4834; AVX2-NEXT:    movslq (%rdi), %rax
4835; AVX2-NEXT:    movq %rax, %rcx
4836; AVX2-NEXT:    shlq $47, %rcx
4837; AVX2-NEXT:    sarq $63, %rcx
4838; AVX2-NEXT:    vmovd %ecx, %xmm0
4839; AVX2-NEXT:    movq %rax, %r8
4840; AVX2-NEXT:    movq %rax, %rdx
4841; AVX2-NEXT:    movq %rax, %rcx
4842; AVX2-NEXT:    movq %rax, %rdi
4843; AVX2-NEXT:    movq %rax, %r13
4844; AVX2-NEXT:    movq %rax, %rsi
4845; AVX2-NEXT:    movq %rax, %r10
4846; AVX2-NEXT:    movq %rax, %r11
4847; AVX2-NEXT:    movq %rax, %r9
4848; AVX2-NEXT:    movq %rax, %rbx
4849; AVX2-NEXT:    movq %rax, %r14
4850; AVX2-NEXT:    movq %rax, %r15
4851; AVX2-NEXT:    movq %rax, %r12
4852; AVX2-NEXT:    movq %rax, %rbp
4853; AVX2-NEXT:    shlq $46, %rbp
4854; AVX2-NEXT:    sarq $63, %rbp
4855; AVX2-NEXT:    vpinsrb $1, %ebp, %xmm0, %xmm0
4856; AVX2-NEXT:    movq %rax, %rbp
4857; AVX2-NEXT:    shlq $45, %r8
4858; AVX2-NEXT:    sarq $63, %r8
4859; AVX2-NEXT:    vpinsrb $2, %r8d, %xmm0, %xmm0
4860; AVX2-NEXT:    movq %rax, %r8
4861; AVX2-NEXT:    shlq $44, %rdx
4862; AVX2-NEXT:    sarq $63, %rdx
4863; AVX2-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
4864; AVX2-NEXT:    movq %rax, %rdx
4865; AVX2-NEXT:    shlq $43, %rcx
4866; AVX2-NEXT:    sarq $63, %rcx
4867; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
4868; AVX2-NEXT:    movq %rax, %rcx
4869; AVX2-NEXT:    shlq $42, %rdi
4870; AVX2-NEXT:    sarq $63, %rdi
4871; AVX2-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
4872; AVX2-NEXT:    movq %rax, %rdi
4873; AVX2-NEXT:    shlq $41, %r13
4874; AVX2-NEXT:    sarq $63, %r13
4875; AVX2-NEXT:    vpinsrb $6, %r13d, %xmm0, %xmm0
4876; AVX2-NEXT:    movq %rax, %r13
4877; AVX2-NEXT:    shlq $40, %rsi
4878; AVX2-NEXT:    sarq $63, %rsi
4879; AVX2-NEXT:    vpinsrb $7, %esi, %xmm0, %xmm0
4880; AVX2-NEXT:    movq %rax, %rsi
4881; AVX2-NEXT:    shlq $39, %r10
4882; AVX2-NEXT:    sarq $63, %r10
4883; AVX2-NEXT:    vpinsrb $8, %r10d, %xmm0, %xmm0
4884; AVX2-NEXT:    movq %rax, %r10
4885; AVX2-NEXT:    shlq $38, %r11
4886; AVX2-NEXT:    sarq $63, %r11
4887; AVX2-NEXT:    vpinsrb $9, %r11d, %xmm0, %xmm0
4888; AVX2-NEXT:    movsbq %al, %r11
4889; AVX2-NEXT:    shlq $37, %r9
4890; AVX2-NEXT:    sarq $63, %r9
4891; AVX2-NEXT:    vpinsrb $10, %r9d, %xmm0, %xmm0
4892; AVX2-NEXT:    movq %rax, %r9
4893; AVX2-NEXT:    shlq $36, %rbx
4894; AVX2-NEXT:    sarq $63, %rbx
4895; AVX2-NEXT:    vpinsrb $11, %ebx, %xmm0, %xmm0
4896; AVX2-NEXT:    movq %rax, %rbx
4897; AVX2-NEXT:    shlq $35, %r14
4898; AVX2-NEXT:    sarq $63, %r14
4899; AVX2-NEXT:    vpinsrb $12, %r14d, %xmm0, %xmm0
4900; AVX2-NEXT:    movq %rax, %r14
4901; AVX2-NEXT:    shlq $34, %r15
4902; AVX2-NEXT:    sarq $63, %r15
4903; AVX2-NEXT:    vpinsrb $13, %r15d, %xmm0, %xmm0
4904; AVX2-NEXT:    movq %rax, %r15
4905; AVX2-NEXT:    shlq $33, %r12
4906; AVX2-NEXT:    sarq $63, %r12
4907; AVX2-NEXT:    vpinsrb $14, %r12d, %xmm0, %xmm0
4908; AVX2-NEXT:    movq %rax, %r12
4909; AVX2-NEXT:    shrq $31, %rbp
4910; AVX2-NEXT:    vpinsrb $15, %ebp, %xmm0, %xmm0
4911; AVX2-NEXT:    movq %rax, %rbp
4912; AVX2-NEXT:    shlq $63, %rdx
4913; AVX2-NEXT:    sarq $63, %rdx
4914; AVX2-NEXT:    vmovd %edx, %xmm1
4915; AVX2-NEXT:    movq %rax, %rdx
4916; AVX2-NEXT:    movswq %ax, %rax
4917; AVX2-NEXT:    shlq $62, %r8
4918; AVX2-NEXT:    sarq $63, %r8
4919; AVX2-NEXT:    vpinsrb $1, %r8d, %xmm1, %xmm1
4920; AVX2-NEXT:    shlq $61, %rcx
4921; AVX2-NEXT:    sarq $63, %rcx
4922; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
4923; AVX2-NEXT:    shlq $60, %rdi
4924; AVX2-NEXT:    sarq $63, %rdi
4925; AVX2-NEXT:    vpinsrb $3, %edi, %xmm1, %xmm1
4926; AVX2-NEXT:    shlq $59, %r13
4927; AVX2-NEXT:    sarq $63, %r13
4928; AVX2-NEXT:    vpinsrb $4, %r13d, %xmm1, %xmm1
4929; AVX2-NEXT:    shlq $58, %rsi
4930; AVX2-NEXT:    sarq $63, %rsi
4931; AVX2-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
4932; AVX2-NEXT:    shlq $57, %r10
4933; AVX2-NEXT:    sarq $63, %r10
4934; AVX2-NEXT:    vpinsrb $6, %r10d, %xmm1, %xmm1
4935; AVX2-NEXT:    shrq $7, %r11
4936; AVX2-NEXT:    vpinsrb $7, %r11d, %xmm1, %xmm1
4937; AVX2-NEXT:    shlq $55, %r9
4938; AVX2-NEXT:    sarq $63, %r9
4939; AVX2-NEXT:    vpinsrb $8, %r9d, %xmm1, %xmm1
4940; AVX2-NEXT:    shlq $54, %rbx
4941; AVX2-NEXT:    sarq $63, %rbx
4942; AVX2-NEXT:    vpinsrb $9, %ebx, %xmm1, %xmm1
4943; AVX2-NEXT:    shlq $53, %r14
4944; AVX2-NEXT:    sarq $63, %r14
4945; AVX2-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
4946; AVX2-NEXT:    shlq $52, %r15
4947; AVX2-NEXT:    sarq $63, %r15
4948; AVX2-NEXT:    vpinsrb $11, %r15d, %xmm1, %xmm1
4949; AVX2-NEXT:    shlq $51, %r12
4950; AVX2-NEXT:    sarq $63, %r12
4951; AVX2-NEXT:    vpinsrb $12, %r12d, %xmm1, %xmm1
4952; AVX2-NEXT:    shlq $50, %rbp
4953; AVX2-NEXT:    sarq $63, %rbp
4954; AVX2-NEXT:    vpinsrb $13, %ebp, %xmm1, %xmm1
4955; AVX2-NEXT:    shlq $49, %rdx
4956; AVX2-NEXT:    sarq $63, %rdx
4957; AVX2-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
4958; AVX2-NEXT:    shrq $15, %rax
4959; AVX2-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
4960; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
4961; AVX2-NEXT:    popq %rbx
4962; AVX2-NEXT:    popq %r12
4963; AVX2-NEXT:    popq %r13
4964; AVX2-NEXT:    popq %r14
4965; AVX2-NEXT:    popq %r15
4966; AVX2-NEXT:    popq %rbp
4967; AVX2-NEXT:    retq
4968;
4969; AVX512F-LABEL: load_sext_32i1_to_32i8:
4970; AVX512F:       # %bb.0: # %entry
4971; AVX512F-NEXT:    kmovw (%rdi), %k1
4972; AVX512F-NEXT:    kmovw 2(%rdi), %k2
4973; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
4974; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
4975; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
4976; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
4977; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
4978; AVX512F-NEXT:    retq
4979;
4980; AVX512BW-LABEL: load_sext_32i1_to_32i8:
4981; AVX512BW:       # %bb.0: # %entry
4982; AVX512BW-NEXT:    kmovd (%rdi), %k0
4983; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
4984; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
4985; AVX512BW-NEXT:    retq
4986;
4987; X32-SSE2-LABEL: load_sext_32i1_to_32i8:
4988; X32-SSE2:       # %bb.0: # %entry
4989; X32-SSE2-NEXT:    pushl %ebp
4990; X32-SSE2-NEXT:    pushl %ebx
4991; X32-SSE2-NEXT:    pushl %edi
4992; X32-SSE2-NEXT:    pushl %esi
4993; X32-SSE2-NEXT:    subl $28, %esp
4994; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
4995; X32-SSE2-NEXT:    movswl (%eax), %edx
4996; X32-SSE2-NEXT:    movl %edx, %ebp
4997; X32-SSE2-NEXT:    movl %edx, %esi
4998; X32-SSE2-NEXT:    movl %edx, %edi
4999; X32-SSE2-NEXT:    movl %edx, %ebx
5000; X32-SSE2-NEXT:    movl %edx, %ecx
5001; X32-SSE2-NEXT:    shrl $15, %ecx
5002; X32-SSE2-NEXT:    movd %ecx, %xmm0
5003; X32-SSE2-NEXT:    movl %edx, %ecx
5004; X32-SSE2-NEXT:    shll $17, %ebp
5005; X32-SSE2-NEXT:    sarl $31, %ebp
5006; X32-SSE2-NEXT:    movd %ebp, %xmm4
5007; X32-SSE2-NEXT:    movl %edx, %ebp
5008; X32-SSE2-NEXT:    shll $18, %esi
5009; X32-SSE2-NEXT:    sarl $31, %esi
5010; X32-SSE2-NEXT:    movd %esi, %xmm1
5011; X32-SSE2-NEXT:    movl %edx, %esi
5012; X32-SSE2-NEXT:    shll $19, %edi
5013; X32-SSE2-NEXT:    sarl $31, %edi
5014; X32-SSE2-NEXT:    movd %edi, %xmm2
5015; X32-SSE2-NEXT:    movl %edx, %edi
5016; X32-SSE2-NEXT:    shll $20, %ebx
5017; X32-SSE2-NEXT:    sarl $31, %ebx
5018; X32-SSE2-NEXT:    movd %ebx, %xmm5
5019; X32-SSE2-NEXT:    movl %edx, %ebx
5020; X32-SSE2-NEXT:    shll $21, %ecx
5021; X32-SSE2-NEXT:    sarl $31, %ecx
5022; X32-SSE2-NEXT:    movd %ecx, %xmm6
5023; X32-SSE2-NEXT:    movl %edx, %ecx
5024; X32-SSE2-NEXT:    shll $22, %ebp
5025; X32-SSE2-NEXT:    sarl $31, %ebp
5026; X32-SSE2-NEXT:    movd %ebp, %xmm7
5027; X32-SSE2-NEXT:    movl %edx, %ebp
5028; X32-SSE2-NEXT:    shll $23, %esi
5029; X32-SSE2-NEXT:    sarl $31, %esi
5030; X32-SSE2-NEXT:    movd %esi, %xmm3
5031; X32-SSE2-NEXT:    movl %edx, %esi
5032; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
5033; X32-SSE2-NEXT:    shll $28, %edi
5034; X32-SSE2-NEXT:    sarl $31, %edi
5035; X32-SSE2-NEXT:    movd %edi, %xmm0
5036; X32-SSE2-NEXT:    movl %edx, %edi
5037; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
5038; X32-SSE2-NEXT:    shll $29, %ebx
5039; X32-SSE2-NEXT:    sarl $31, %ebx
5040; X32-SSE2-NEXT:    movd %ebx, %xmm1
5041; X32-SSE2-NEXT:    movl %edx, %ebx
5042; X32-SSE2-NEXT:    movsbl %dl, %edx
5043; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5044; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
5045; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
5046; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
5047; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
5048; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
5049; X32-SSE2-NEXT:    shll $30, %ecx
5050; X32-SSE2-NEXT:    sarl $31, %ecx
5051; X32-SSE2-NEXT:    movd %ecx, %xmm2
5052; X32-SSE2-NEXT:    shll $31, %ebp
5053; X32-SSE2-NEXT:    sarl $31, %ebp
5054; X32-SSE2-NEXT:    movd %ebp, %xmm0
5055; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
5056; X32-SSE2-NEXT:    shll $26, %esi
5057; X32-SSE2-NEXT:    sarl $31, %esi
5058; X32-SSE2-NEXT:    movd %esi, %xmm7
5059; X32-SSE2-NEXT:    shll $27, %edi
5060; X32-SSE2-NEXT:    sarl $31, %edi
5061; X32-SSE2-NEXT:    movd %edi, %xmm2
5062; X32-SSE2-NEXT:    shll $25, %ebx
5063; X32-SSE2-NEXT:    sarl $31, %ebx
5064; X32-SSE2-NEXT:    movd %ebx, %xmm6
5065; X32-SSE2-NEXT:    shrl $7, %edx
5066; X32-SSE2-NEXT:    movd %edx, %xmm5
5067; X32-SSE2-NEXT:    movswl 2(%eax), %eax
5068; X32-SSE2-NEXT:    movl %eax, %edx
5069; X32-SSE2-NEXT:    movl %eax, %ebp
5070; X32-SSE2-NEXT:    movl %eax, %ecx
5071; X32-SSE2-NEXT:    movl %eax, %esi
5072; X32-SSE2-NEXT:    movl %eax, %edi
5073; X32-SSE2-NEXT:    movl %eax, %ebx
5074; X32-SSE2-NEXT:    shrl $15, %ebx
5075; X32-SSE2-NEXT:    movd %ebx, %xmm4
5076; X32-SSE2-NEXT:    movdqu %xmm4, (%esp) # 16-byte Spill
5077; X32-SSE2-NEXT:    movl %eax, %ebx
5078; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5079; X32-SSE2-NEXT:    shll $17, %edx
5080; X32-SSE2-NEXT:    sarl $31, %edx
5081; X32-SSE2-NEXT:    movd %edx, %xmm4
5082; X32-SSE2-NEXT:    movl %eax, %edx
5083; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3],xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
5084; X32-SSE2-NEXT:    shll $18, %ebp
5085; X32-SSE2-NEXT:    sarl $31, %ebp
5086; X32-SSE2-NEXT:    movd %ebp, %xmm7
5087; X32-SSE2-NEXT:    movl %eax, %ebp
5088; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
5089; X32-SSE2-NEXT:    shll $19, %ecx
5090; X32-SSE2-NEXT:    sarl $31, %ecx
5091; X32-SSE2-NEXT:    movd %ecx, %xmm5
5092; X32-SSE2-NEXT:    movl %eax, %ecx
5093; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
5094; X32-SSE2-NEXT:    shll $20, %esi
5095; X32-SSE2-NEXT:    sarl $31, %esi
5096; X32-SSE2-NEXT:    movd %esi, %xmm6
5097; X32-SSE2-NEXT:    movl %eax, %esi
5098; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
5099; X32-SSE2-NEXT:    shll $21, %edi
5100; X32-SSE2-NEXT:    sarl $31, %edi
5101; X32-SSE2-NEXT:    movd %edi, %xmm1
5102; X32-SSE2-NEXT:    movl %eax, %edi
5103; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
5104; X32-SSE2-NEXT:    shll $22, %ebx
5105; X32-SSE2-NEXT:    sarl $31, %ebx
5106; X32-SSE2-NEXT:    movd %ebx, %xmm3
5107; X32-SSE2-NEXT:    movl %eax, %ebx
5108; X32-SSE2-NEXT:    movdqu (%esp), %xmm2 # 16-byte Reload
5109; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
5110; X32-SSE2-NEXT:    shll $23, %edx
5111; X32-SSE2-NEXT:    sarl $31, %edx
5112; X32-SSE2-NEXT:    movd %edx, %xmm2
5113; X32-SSE2-NEXT:    movl %eax, %edx
5114; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
5115; X32-SSE2-NEXT:    shll $28, %ebp
5116; X32-SSE2-NEXT:    sarl $31, %ebp
5117; X32-SSE2-NEXT:    movd %ebp, %xmm7
5118; X32-SSE2-NEXT:    movl %eax, %ebp
5119; X32-SSE2-NEXT:    movsbl %al, %eax
5120; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5121; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
5122; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
5123; X32-SSE2-NEXT:    shll $29, %ecx
5124; X32-SSE2-NEXT:    sarl $31, %ecx
5125; X32-SSE2-NEXT:    movd %ecx, %xmm3
5126; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5127; X32-SSE2-NEXT:    shll $30, %esi
5128; X32-SSE2-NEXT:    sarl $31, %esi
5129; X32-SSE2-NEXT:    movd %esi, %xmm4
5130; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
5131; X32-SSE2-NEXT:    shll $31, %edi
5132; X32-SSE2-NEXT:    sarl $31, %edi
5133; X32-SSE2-NEXT:    movd %edi, %xmm1
5134; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
5135; X32-SSE2-NEXT:    shll $26, %ebx
5136; X32-SSE2-NEXT:    sarl $31, %ebx
5137; X32-SSE2-NEXT:    movd %ebx, %xmm5
5138; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
5139; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
5140; X32-SSE2-NEXT:    shll $27, %edx
5141; X32-SSE2-NEXT:    sarl $31, %edx
5142; X32-SSE2-NEXT:    movd %edx, %xmm3
5143; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
5144; X32-SSE2-NEXT:    shll $25, %ebp
5145; X32-SSE2-NEXT:    sarl $31, %ebp
5146; X32-SSE2-NEXT:    movd %ebp, %xmm4
5147; X32-SSE2-NEXT:    shrl $7, %eax
5148; X32-SSE2-NEXT:    movd %eax, %xmm5
5149; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
5150; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
5151; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
5152; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5153; X32-SSE2-NEXT:    addl $28, %esp
5154; X32-SSE2-NEXT:    popl %esi
5155; X32-SSE2-NEXT:    popl %edi
5156; X32-SSE2-NEXT:    popl %ebx
5157; X32-SSE2-NEXT:    popl %ebp
5158; X32-SSE2-NEXT:    retl
5159;
5160; X32-SSE41-LABEL: load_sext_32i1_to_32i8:
5161; X32-SSE41:       # %bb.0: # %entry
5162; X32-SSE41-NEXT:    pushl %esi
5163; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5164; X32-SSE41-NEXT:    movswl (%eax), %ecx
5165; X32-SSE41-NEXT:    movl %ecx, %edx
5166; X32-SSE41-NEXT:    shll $30, %edx
5167; X32-SSE41-NEXT:    sarl $31, %edx
5168; X32-SSE41-NEXT:    movl %ecx, %esi
5169; X32-SSE41-NEXT:    shll $31, %esi
5170; X32-SSE41-NEXT:    sarl $31, %esi
5171; X32-SSE41-NEXT:    movd %esi, %xmm0
5172; X32-SSE41-NEXT:    pinsrb $1, %edx, %xmm0
5173; X32-SSE41-NEXT:    movl %ecx, %edx
5174; X32-SSE41-NEXT:    shll $29, %edx
5175; X32-SSE41-NEXT:    sarl $31, %edx
5176; X32-SSE41-NEXT:    pinsrb $2, %edx, %xmm0
5177; X32-SSE41-NEXT:    movl %ecx, %edx
5178; X32-SSE41-NEXT:    shll $28, %edx
5179; X32-SSE41-NEXT:    sarl $31, %edx
5180; X32-SSE41-NEXT:    pinsrb $3, %edx, %xmm0
5181; X32-SSE41-NEXT:    movl %ecx, %edx
5182; X32-SSE41-NEXT:    shll $27, %edx
5183; X32-SSE41-NEXT:    sarl $31, %edx
5184; X32-SSE41-NEXT:    pinsrb $4, %edx, %xmm0
5185; X32-SSE41-NEXT:    movl %ecx, %edx
5186; X32-SSE41-NEXT:    shll $26, %edx
5187; X32-SSE41-NEXT:    sarl $31, %edx
5188; X32-SSE41-NEXT:    pinsrb $5, %edx, %xmm0
5189; X32-SSE41-NEXT:    movl %ecx, %edx
5190; X32-SSE41-NEXT:    shll $25, %edx
5191; X32-SSE41-NEXT:    sarl $31, %edx
5192; X32-SSE41-NEXT:    pinsrb $6, %edx, %xmm0
5193; X32-SSE41-NEXT:    movsbl %cl, %edx
5194; X32-SSE41-NEXT:    shrl $7, %edx
5195; X32-SSE41-NEXT:    pinsrb $7, %edx, %xmm0
5196; X32-SSE41-NEXT:    movl %ecx, %edx
5197; X32-SSE41-NEXT:    shll $23, %edx
5198; X32-SSE41-NEXT:    sarl $31, %edx
5199; X32-SSE41-NEXT:    pinsrb $8, %edx, %xmm0
5200; X32-SSE41-NEXT:    movl %ecx, %edx
5201; X32-SSE41-NEXT:    shll $22, %edx
5202; X32-SSE41-NEXT:    sarl $31, %edx
5203; X32-SSE41-NEXT:    pinsrb $9, %edx, %xmm0
5204; X32-SSE41-NEXT:    movl %ecx, %edx
5205; X32-SSE41-NEXT:    shll $21, %edx
5206; X32-SSE41-NEXT:    sarl $31, %edx
5207; X32-SSE41-NEXT:    pinsrb $10, %edx, %xmm0
5208; X32-SSE41-NEXT:    movl %ecx, %edx
5209; X32-SSE41-NEXT:    shll $20, %edx
5210; X32-SSE41-NEXT:    sarl $31, %edx
5211; X32-SSE41-NEXT:    pinsrb $11, %edx, %xmm0
5212; X32-SSE41-NEXT:    movl %ecx, %edx
5213; X32-SSE41-NEXT:    shll $19, %edx
5214; X32-SSE41-NEXT:    sarl $31, %edx
5215; X32-SSE41-NEXT:    pinsrb $12, %edx, %xmm0
5216; X32-SSE41-NEXT:    movl %ecx, %edx
5217; X32-SSE41-NEXT:    shll $18, %edx
5218; X32-SSE41-NEXT:    sarl $31, %edx
5219; X32-SSE41-NEXT:    pinsrb $13, %edx, %xmm0
5220; X32-SSE41-NEXT:    movl %ecx, %edx
5221; X32-SSE41-NEXT:    shll $17, %edx
5222; X32-SSE41-NEXT:    sarl $31, %edx
5223; X32-SSE41-NEXT:    pinsrb $14, %edx, %xmm0
5224; X32-SSE41-NEXT:    shrl $15, %ecx
5225; X32-SSE41-NEXT:    pinsrb $15, %ecx, %xmm0
5226; X32-SSE41-NEXT:    movswl 2(%eax), %eax
5227; X32-SSE41-NEXT:    movl %eax, %ecx
5228; X32-SSE41-NEXT:    shll $30, %ecx
5229; X32-SSE41-NEXT:    sarl $31, %ecx
5230; X32-SSE41-NEXT:    movl %eax, %edx
5231; X32-SSE41-NEXT:    shll $31, %edx
5232; X32-SSE41-NEXT:    sarl $31, %edx
5233; X32-SSE41-NEXT:    movd %edx, %xmm1
5234; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
5235; X32-SSE41-NEXT:    movl %eax, %ecx
5236; X32-SSE41-NEXT:    shll $29, %ecx
5237; X32-SSE41-NEXT:    sarl $31, %ecx
5238; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
5239; X32-SSE41-NEXT:    movl %eax, %ecx
5240; X32-SSE41-NEXT:    shll $28, %ecx
5241; X32-SSE41-NEXT:    sarl $31, %ecx
5242; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
5243; X32-SSE41-NEXT:    movl %eax, %ecx
5244; X32-SSE41-NEXT:    shll $27, %ecx
5245; X32-SSE41-NEXT:    sarl $31, %ecx
5246; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
5247; X32-SSE41-NEXT:    movl %eax, %ecx
5248; X32-SSE41-NEXT:    shll $26, %ecx
5249; X32-SSE41-NEXT:    sarl $31, %ecx
5250; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
5251; X32-SSE41-NEXT:    movl %eax, %ecx
5252; X32-SSE41-NEXT:    shll $25, %ecx
5253; X32-SSE41-NEXT:    sarl $31, %ecx
5254; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
5255; X32-SSE41-NEXT:    movsbl %al, %ecx
5256; X32-SSE41-NEXT:    shrl $7, %ecx
5257; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
5258; X32-SSE41-NEXT:    movl %eax, %ecx
5259; X32-SSE41-NEXT:    shll $23, %ecx
5260; X32-SSE41-NEXT:    sarl $31, %ecx
5261; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
5262; X32-SSE41-NEXT:    movl %eax, %ecx
5263; X32-SSE41-NEXT:    shll $22, %ecx
5264; X32-SSE41-NEXT:    sarl $31, %ecx
5265; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
5266; X32-SSE41-NEXT:    movl %eax, %ecx
5267; X32-SSE41-NEXT:    shll $21, %ecx
5268; X32-SSE41-NEXT:    sarl $31, %ecx
5269; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
5270; X32-SSE41-NEXT:    movl %eax, %ecx
5271; X32-SSE41-NEXT:    shll $20, %ecx
5272; X32-SSE41-NEXT:    sarl $31, %ecx
5273; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
5274; X32-SSE41-NEXT:    movl %eax, %ecx
5275; X32-SSE41-NEXT:    shll $19, %ecx
5276; X32-SSE41-NEXT:    sarl $31, %ecx
5277; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
5278; X32-SSE41-NEXT:    movl %eax, %ecx
5279; X32-SSE41-NEXT:    shll $18, %ecx
5280; X32-SSE41-NEXT:    sarl $31, %ecx
5281; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
5282; X32-SSE41-NEXT:    movl %eax, %ecx
5283; X32-SSE41-NEXT:    shll $17, %ecx
5284; X32-SSE41-NEXT:    sarl $31, %ecx
5285; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
5286; X32-SSE41-NEXT:    shrl $15, %eax
5287; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm1
5288; X32-SSE41-NEXT:    popl %esi
5289; X32-SSE41-NEXT:    retl
5290entry:
5291 %X = load <32 x i1>, <32 x i1>* %ptr
5292 %Y = sext <32 x i1> %X to <32 x i8>
5293 ret <32 x i8> %Y
5294}
5295
5296define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
5297; SSE2-LABEL: load_sext_16i8_to_16i16:
5298; SSE2:       # %bb.0: # %entry
5299; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5300; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5301; SSE2-NEXT:    psraw $8, %xmm0
5302; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5303; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5304; SSE2-NEXT:    psraw $8, %xmm1
5305; SSE2-NEXT:    retq
5306;
5307; SSSE3-LABEL: load_sext_16i8_to_16i16:
5308; SSSE3:       # %bb.0: # %entry
5309; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5310; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5311; SSSE3-NEXT:    psraw $8, %xmm0
5312; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5313; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5314; SSSE3-NEXT:    psraw $8, %xmm1
5315; SSSE3-NEXT:    retq
5316;
5317; SSE41-LABEL: load_sext_16i8_to_16i16:
5318; SSE41:       # %bb.0: # %entry
5319; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
5320; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
5321; SSE41-NEXT:    retq
5322;
5323; AVX1-LABEL: load_sext_16i8_to_16i16:
5324; AVX1:       # %bb.0: # %entry
5325; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm0
5326; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm1
5327; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5328; AVX1-NEXT:    retq
5329;
5330; AVX2-LABEL: load_sext_16i8_to_16i16:
5331; AVX2:       # %bb.0: # %entry
5332; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
5333; AVX2-NEXT:    retq
5334;
5335; AVX512-LABEL: load_sext_16i8_to_16i16:
5336; AVX512:       # %bb.0: # %entry
5337; AVX512-NEXT:    vpmovsxbw (%rdi), %ymm0
5338; AVX512-NEXT:    retq
5339;
5340; X32-SSE2-LABEL: load_sext_16i8_to_16i16:
5341; X32-SSE2:       # %bb.0: # %entry
5342; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5343; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5344; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5345; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5346; X32-SSE2-NEXT:    psraw $8, %xmm0
5347; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5348; X32-SSE2-NEXT:    psraw $8, %xmm1
5349; X32-SSE2-NEXT:    retl
5350;
5351; X32-SSE41-LABEL: load_sext_16i8_to_16i16:
5352; X32-SSE41:       # %bb.0: # %entry
5353; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5354; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
5355; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
5356; X32-SSE41-NEXT:    retl
5357entry:
5358 %X = load <16 x i8>, <16 x i8>* %ptr
5359 %Y = sext <16 x i8> %X to <16 x i16>
5360 ret <16 x i16> %Y
5361}
5362
5363define <2 x i64> @load_sext_2i16_to_2i64(<2 x i16> *%ptr) {
5364; SSE2-LABEL: load_sext_2i16_to_2i64:
5365; SSE2:       # %bb.0: # %entry
5366; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5367; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5368; SSE2-NEXT:    pxor %xmm1, %xmm1
5369; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5370; SSE2-NEXT:    psrad $16, %xmm0
5371; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5372; SSE2-NEXT:    retq
5373;
5374; SSSE3-LABEL: load_sext_2i16_to_2i64:
5375; SSSE3:       # %bb.0: # %entry
5376; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5377; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5378; SSSE3-NEXT:    pxor %xmm1, %xmm1
5379; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
5380; SSSE3-NEXT:    psrad $16, %xmm0
5381; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5382; SSSE3-NEXT:    retq
5383;
5384; SSE41-LABEL: load_sext_2i16_to_2i64:
5385; SSE41:       # %bb.0: # %entry
5386; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
5387; SSE41-NEXT:    retq
5388;
5389; AVX-LABEL: load_sext_2i16_to_2i64:
5390; AVX:       # %bb.0: # %entry
5391; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
5392; AVX-NEXT:    retq
5393;
5394; X32-SSE2-LABEL: load_sext_2i16_to_2i64:
5395; X32-SSE2:       # %bb.0: # %entry
5396; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5397; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5398; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5399; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
5400; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5401; X32-SSE2-NEXT:    psrad $16, %xmm0
5402; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5403; X32-SSE2-NEXT:    retl
5404;
5405; X32-SSE41-LABEL: load_sext_2i16_to_2i64:
5406; X32-SSE41:       # %bb.0: # %entry
5407; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5408; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
5409; X32-SSE41-NEXT:    retl
5410entry:
5411 %X = load <2 x i16>, <2 x i16>* %ptr
5412 %Y = sext <2 x i16> %X to <2 x i64>
5413 ret <2 x i64> %Y
5414}
5415
5416define <4 x i32> @load_sext_4i16_to_4i32(<4 x i16> *%ptr) {
5417; SSE2-LABEL: load_sext_4i16_to_4i32:
5418; SSE2:       # %bb.0: # %entry
5419; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5420; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5421; SSE2-NEXT:    psrad $16, %xmm0
5422; SSE2-NEXT:    retq
5423;
5424; SSSE3-LABEL: load_sext_4i16_to_4i32:
5425; SSSE3:       # %bb.0: # %entry
5426; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5427; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5428; SSSE3-NEXT:    psrad $16, %xmm0
5429; SSSE3-NEXT:    retq
5430;
5431; SSE41-LABEL: load_sext_4i16_to_4i32:
5432; SSE41:       # %bb.0: # %entry
5433; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
5434; SSE41-NEXT:    retq
5435;
5436; AVX-LABEL: load_sext_4i16_to_4i32:
5437; AVX:       # %bb.0: # %entry
5438; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
5439; AVX-NEXT:    retq
5440;
5441; X32-SSE2-LABEL: load_sext_4i16_to_4i32:
5442; X32-SSE2:       # %bb.0: # %entry
5443; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5444; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5445; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5446; X32-SSE2-NEXT:    psrad $16, %xmm0
5447; X32-SSE2-NEXT:    retl
5448;
5449; X32-SSE41-LABEL: load_sext_4i16_to_4i32:
5450; X32-SSE41:       # %bb.0: # %entry
5451; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5452; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
5453; X32-SSE41-NEXT:    retl
5454entry:
5455 %X = load <4 x i16>, <4 x i16>* %ptr
5456 %Y = sext <4 x i16> %X to <4 x i32>
5457 ret <4 x i32> %Y
5458}
5459
5460define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
5461; SSE2-LABEL: load_sext_4i16_to_4i64:
5462; SSE2:       # %bb.0: # %entry
5463; SSE2-NEXT:    movswq 2(%rdi), %rax
5464; SSE2-NEXT:    movq %rax, %xmm1
5465; SSE2-NEXT:    movswq (%rdi), %rax
5466; SSE2-NEXT:    movq %rax, %xmm0
5467; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
5468; SSE2-NEXT:    movswq 6(%rdi), %rax
5469; SSE2-NEXT:    movq %rax, %xmm2
5470; SSE2-NEXT:    movswq 4(%rdi), %rax
5471; SSE2-NEXT:    movq %rax, %xmm1
5472; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5473; SSE2-NEXT:    retq
5474;
5475; SSSE3-LABEL: load_sext_4i16_to_4i64:
5476; SSSE3:       # %bb.0: # %entry
5477; SSSE3-NEXT:    movswq 2(%rdi), %rax
5478; SSSE3-NEXT:    movq %rax, %xmm1
5479; SSSE3-NEXT:    movswq (%rdi), %rax
5480; SSSE3-NEXT:    movq %rax, %xmm0
5481; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
5482; SSSE3-NEXT:    movswq 6(%rdi), %rax
5483; SSSE3-NEXT:    movq %rax, %xmm2
5484; SSSE3-NEXT:    movswq 4(%rdi), %rax
5485; SSSE3-NEXT:    movq %rax, %xmm1
5486; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5487; SSSE3-NEXT:    retq
5488;
5489; SSE41-LABEL: load_sext_4i16_to_4i64:
5490; SSE41:       # %bb.0: # %entry
5491; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
5492; SSE41-NEXT:    pmovsxwq 4(%rdi), %xmm1
5493; SSE41-NEXT:    retq
5494;
5495; AVX1-LABEL: load_sext_4i16_to_4i64:
5496; AVX1:       # %bb.0: # %entry
5497; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm0
5498; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
5499; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5500; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
5501; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5502; AVX1-NEXT:    retq
5503;
5504; AVX2-LABEL: load_sext_4i16_to_4i64:
5505; AVX2:       # %bb.0: # %entry
5506; AVX2-NEXT:    vpmovsxwq (%rdi), %ymm0
5507; AVX2-NEXT:    retq
5508;
5509; AVX512-LABEL: load_sext_4i16_to_4i64:
5510; AVX512:       # %bb.0: # %entry
5511; AVX512-NEXT:    vpmovsxwq (%rdi), %ymm0
5512; AVX512-NEXT:    retq
5513;
5514; X32-SSE2-LABEL: load_sext_4i16_to_4i64:
5515; X32-SSE2:       # %bb.0: # %entry
5516; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5517; X32-SSE2-NEXT:    movswl 2(%eax), %ecx
5518; X32-SSE2-NEXT:    movd %ecx, %xmm1
5519; X32-SSE2-NEXT:    sarl $31, %ecx
5520; X32-SSE2-NEXT:    movd %ecx, %xmm0
5521; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
5522; X32-SSE2-NEXT:    movswl (%eax), %ecx
5523; X32-SSE2-NEXT:    movd %ecx, %xmm0
5524; X32-SSE2-NEXT:    sarl $31, %ecx
5525; X32-SSE2-NEXT:    movd %ecx, %xmm2
5526; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
5527; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
5528; X32-SSE2-NEXT:    movswl 6(%eax), %ecx
5529; X32-SSE2-NEXT:    movd %ecx, %xmm2
5530; X32-SSE2-NEXT:    sarl $31, %ecx
5531; X32-SSE2-NEXT:    movd %ecx, %xmm1
5532; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
5533; X32-SSE2-NEXT:    movswl 4(%eax), %eax
5534; X32-SSE2-NEXT:    movd %eax, %xmm1
5535; X32-SSE2-NEXT:    sarl $31, %eax
5536; X32-SSE2-NEXT:    movd %eax, %xmm3
5537; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
5538; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5539; X32-SSE2-NEXT:    retl
5540;
5541; X32-SSE41-LABEL: load_sext_4i16_to_4i64:
5542; X32-SSE41:       # %bb.0: # %entry
5543; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5544; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
5545; X32-SSE41-NEXT:    pmovsxwq 4(%eax), %xmm1
5546; X32-SSE41-NEXT:    retl
5547entry:
5548 %X = load <4 x i16>, <4 x i16>* %ptr
5549 %Y = sext <4 x i16> %X to <4 x i64>
5550 ret <4 x i64> %Y
5551}
5552
5553define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
5554; SSE2-LABEL: load_sext_8i16_to_8i32:
5555; SSE2:       # %bb.0: # %entry
5556; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5557; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5558; SSE2-NEXT:    psrad $16, %xmm0
5559; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5560; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
5561; SSE2-NEXT:    psrad $16, %xmm1
5562; SSE2-NEXT:    retq
5563;
5564; SSSE3-LABEL: load_sext_8i16_to_8i32:
5565; SSSE3:       # %bb.0: # %entry
5566; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5567; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5568; SSSE3-NEXT:    psrad $16, %xmm0
5569; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5570; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
5571; SSSE3-NEXT:    psrad $16, %xmm1
5572; SSSE3-NEXT:    retq
5573;
5574; SSE41-LABEL: load_sext_8i16_to_8i32:
5575; SSE41:       # %bb.0: # %entry
5576; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
5577; SSE41-NEXT:    pmovsxwd 8(%rdi), %xmm1
5578; SSE41-NEXT:    retq
5579;
5580; AVX1-LABEL: load_sext_8i16_to_8i32:
5581; AVX1:       # %bb.0: # %entry
5582; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm0
5583; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm1
5584; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5585; AVX1-NEXT:    retq
5586;
5587; AVX2-LABEL: load_sext_8i16_to_8i32:
5588; AVX2:       # %bb.0: # %entry
5589; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm0
5590; AVX2-NEXT:    retq
5591;
5592; AVX512-LABEL: load_sext_8i16_to_8i32:
5593; AVX512:       # %bb.0: # %entry
5594; AVX512-NEXT:    vpmovsxwd (%rdi), %ymm0
5595; AVX512-NEXT:    retq
5596;
5597; X32-SSE2-LABEL: load_sext_8i16_to_8i32:
5598; X32-SSE2:       # %bb.0: # %entry
5599; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5600; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5601; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
5602; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5603; X32-SSE2-NEXT:    psrad $16, %xmm0
5604; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
5605; X32-SSE2-NEXT:    psrad $16, %xmm1
5606; X32-SSE2-NEXT:    retl
5607;
5608; X32-SSE41-LABEL: load_sext_8i16_to_8i32:
5609; X32-SSE41:       # %bb.0: # %entry
5610; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5611; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
5612; X32-SSE41-NEXT:    pmovsxwd 8(%eax), %xmm1
5613; X32-SSE41-NEXT:    retl
5614entry:
5615 %X = load <8 x i16>, <8 x i16>* %ptr
5616 %Y = sext <8 x i16> %X to <8 x i32>
5617 ret <8 x i32> %Y
5618}
5619
5620define <2 x i64> @load_sext_2i32_to_2i64(<2 x i32> *%ptr) {
5621; SSE2-LABEL: load_sext_2i32_to_2i64:
5622; SSE2:       # %bb.0: # %entry
5623; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5624; SSE2-NEXT:    pxor %xmm1, %xmm1
5625; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5626; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5627; SSE2-NEXT:    retq
5628;
5629; SSSE3-LABEL: load_sext_2i32_to_2i64:
5630; SSSE3:       # %bb.0: # %entry
5631; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5632; SSSE3-NEXT:    pxor %xmm1, %xmm1
5633; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
5634; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5635; SSSE3-NEXT:    retq
5636;
5637; SSE41-LABEL: load_sext_2i32_to_2i64:
5638; SSE41:       # %bb.0: # %entry
5639; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
5640; SSE41-NEXT:    retq
5641;
5642; AVX-LABEL: load_sext_2i32_to_2i64:
5643; AVX:       # %bb.0: # %entry
5644; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
5645; AVX-NEXT:    retq
5646;
5647; X32-SSE2-LABEL: load_sext_2i32_to_2i64:
5648; X32-SSE2:       # %bb.0: # %entry
5649; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5650; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5651; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
5652; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5653; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5654; X32-SSE2-NEXT:    retl
5655;
5656; X32-SSE41-LABEL: load_sext_2i32_to_2i64:
5657; X32-SSE41:       # %bb.0: # %entry
5658; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5659; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
5660; X32-SSE41-NEXT:    retl
5661entry:
5662 %X = load <2 x i32>, <2 x i32>* %ptr
5663 %Y = sext <2 x i32> %X to <2 x i64>
5664 ret <2 x i64> %Y
5665}
5666
5667define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
5668; SSE2-LABEL: load_sext_4i32_to_4i64:
5669; SSE2:       # %bb.0: # %entry
5670; SSE2-NEXT:    movdqa (%rdi), %xmm0
5671; SSE2-NEXT:    pxor %xmm2, %xmm2
5672; SSE2-NEXT:    pxor %xmm3, %xmm3
5673; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5674; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5675; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5676; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5677; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5678; SSE2-NEXT:    retq
5679;
5680; SSSE3-LABEL: load_sext_4i32_to_4i64:
5681; SSSE3:       # %bb.0: # %entry
5682; SSSE3-NEXT:    movdqa (%rdi), %xmm0
5683; SSSE3-NEXT:    pxor %xmm2, %xmm2
5684; SSSE3-NEXT:    pxor %xmm3, %xmm3
5685; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5686; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5687; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5688; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5689; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5690; SSSE3-NEXT:    retq
5691;
5692; SSE41-LABEL: load_sext_4i32_to_4i64:
5693; SSE41:       # %bb.0: # %entry
5694; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
5695; SSE41-NEXT:    pmovsxdq 8(%rdi), %xmm1
5696; SSE41-NEXT:    retq
5697;
5698; AVX1-LABEL: load_sext_4i32_to_4i64:
5699; AVX1:       # %bb.0: # %entry
5700; AVX1-NEXT:    vpmovsxdq 8(%rdi), %xmm0
5701; AVX1-NEXT:    vpmovsxdq (%rdi), %xmm1
5702; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5703; AVX1-NEXT:    retq
5704;
5705; AVX2-LABEL: load_sext_4i32_to_4i64:
5706; AVX2:       # %bb.0: # %entry
5707; AVX2-NEXT:    vpmovsxdq (%rdi), %ymm0
5708; AVX2-NEXT:    retq
5709;
5710; AVX512-LABEL: load_sext_4i32_to_4i64:
5711; AVX512:       # %bb.0: # %entry
5712; AVX512-NEXT:    vpmovsxdq (%rdi), %ymm0
5713; AVX512-NEXT:    retq
5714;
5715; X32-SSE2-LABEL: load_sext_4i32_to_4i64:
5716; X32-SSE2:       # %bb.0: # %entry
5717; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5718; X32-SSE2-NEXT:    movdqa (%eax), %xmm0
5719; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5720; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5721; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5722; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5723; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5724; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5725; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5726; X32-SSE2-NEXT:    retl
5727;
5728; X32-SSE41-LABEL: load_sext_4i32_to_4i64:
5729; X32-SSE41:       # %bb.0: # %entry
5730; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5731; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
5732; X32-SSE41-NEXT:    pmovsxdq 8(%eax), %xmm1
5733; X32-SSE41-NEXT:    retl
5734entry:
5735 %X = load <4 x i32>, <4 x i32>* %ptr
5736 %Y = sext <4 x i32> %X to <4 x i64>
5737 ret <4 x i64> %Y
5738}
5739
5740define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
5741; SSE2-LABEL: sext_2i8_to_i32:
5742; SSE2:       # %bb.0: # %entry
5743; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5744; SSE2-NEXT:    psraw $8, %xmm0
5745; SSE2-NEXT:    movd %xmm0, %eax
5746; SSE2-NEXT:    retq
5747;
5748; SSSE3-LABEL: sext_2i8_to_i32:
5749; SSSE3:       # %bb.0: # %entry
5750; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5751; SSSE3-NEXT:    psraw $8, %xmm0
5752; SSSE3-NEXT:    movd %xmm0, %eax
5753; SSSE3-NEXT:    retq
5754;
5755; SSE41-LABEL: sext_2i8_to_i32:
5756; SSE41:       # %bb.0: # %entry
5757; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
5758; SSE41-NEXT:    movd %xmm0, %eax
5759; SSE41-NEXT:    retq
5760;
5761; AVX-LABEL: sext_2i8_to_i32:
5762; AVX:       # %bb.0: # %entry
5763; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
5764; AVX-NEXT:    vmovd %xmm0, %eax
5765; AVX-NEXT:    retq
5766;
5767; X32-SSE2-LABEL: sext_2i8_to_i32:
5768; X32-SSE2:       # %bb.0: # %entry
5769; X32-SSE2-NEXT:    pushl %eax
5770; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
5771; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5772; X32-SSE2-NEXT:    psraw $8, %xmm0
5773; X32-SSE2-NEXT:    movd %xmm0, %eax
5774; X32-SSE2-NEXT:    popl %ecx
5775; X32-SSE2-NEXT:    .cfi_def_cfa_offset 4
5776; X32-SSE2-NEXT:    retl
5777;
5778; X32-SSE41-LABEL: sext_2i8_to_i32:
5779; X32-SSE41:       # %bb.0: # %entry
5780; X32-SSE41-NEXT:    pushl %eax
5781; X32-SSE41-NEXT:    .cfi_def_cfa_offset 8
5782; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
5783; X32-SSE41-NEXT:    movd %xmm0, %eax
5784; X32-SSE41-NEXT:    popl %ecx
5785; X32-SSE41-NEXT:    .cfi_def_cfa_offset 4
5786; X32-SSE41-NEXT:    retl
5787entry:
5788  %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
5789  %Ex = sext <2 x i8> %Shuf to <2 x i16>
5790  %Bc = bitcast <2 x i16> %Ex to i32
5791  ret i32 %Bc
5792}
5793
5794define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
5795; SSE2-LABEL: sext_4i1_to_4i64:
5796; SSE2:       # %bb.0:
5797; SSE2-NEXT:    pslld $31, %xmm0
5798; SSE2-NEXT:    psrad $31, %xmm0
5799; SSE2-NEXT:    pxor %xmm2, %xmm2
5800; SSE2-NEXT:    pxor %xmm3, %xmm3
5801; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5802; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5803; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5804; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5805; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5806; SSE2-NEXT:    retq
5807;
5808; SSSE3-LABEL: sext_4i1_to_4i64:
5809; SSSE3:       # %bb.0:
5810; SSSE3-NEXT:    pslld $31, %xmm0
5811; SSSE3-NEXT:    psrad $31, %xmm0
5812; SSSE3-NEXT:    pxor %xmm2, %xmm2
5813; SSSE3-NEXT:    pxor %xmm3, %xmm3
5814; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5815; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5816; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5817; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5818; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5819; SSSE3-NEXT:    retq
5820;
5821; SSE41-LABEL: sext_4i1_to_4i64:
5822; SSE41:       # %bb.0:
5823; SSE41-NEXT:    pslld $31, %xmm0
5824; SSE41-NEXT:    psrad $31, %xmm0
5825; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5826; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5827; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5828; SSE41-NEXT:    movdqa %xmm2, %xmm0
5829; SSE41-NEXT:    retq
5830;
5831; AVX1-LABEL: sext_4i1_to_4i64:
5832; AVX1:       # %bb.0:
5833; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
5834; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
5835; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
5836; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5837; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
5838; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5839; AVX1-NEXT:    retq
5840;
5841; AVX2-LABEL: sext_4i1_to_4i64:
5842; AVX2:       # %bb.0:
5843; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
5844; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
5845; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
5846; AVX2-NEXT:    retq
5847;
5848; AVX512-LABEL: sext_4i1_to_4i64:
5849; AVX512:       # %bb.0:
5850; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
5851; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm0
5852; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
5853; AVX512-NEXT:    retq
5854;
5855; X32-SSE2-LABEL: sext_4i1_to_4i64:
5856; X32-SSE2:       # %bb.0:
5857; X32-SSE2-NEXT:    pslld $31, %xmm0
5858; X32-SSE2-NEXT:    psrad $31, %xmm0
5859; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5860; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5861; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5862; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5863; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5864; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5865; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5866; X32-SSE2-NEXT:    retl
5867;
5868; X32-SSE41-LABEL: sext_4i1_to_4i64:
5869; X32-SSE41:       # %bb.0:
5870; X32-SSE41-NEXT:    pslld $31, %xmm0
5871; X32-SSE41-NEXT:    psrad $31, %xmm0
5872; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5873; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5874; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5875; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
5876; X32-SSE41-NEXT:    retl
5877  %extmask = sext <4 x i1> %mask to <4 x i64>
5878  ret <4 x i64> %extmask
5879}
5880
5881define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
5882; SSE2-LABEL: sext_4i8_to_4i64:
5883; SSE2:       # %bb.0:
5884; SSE2-NEXT:    pslld $24, %xmm0
5885; SSE2-NEXT:    psrad $24, %xmm0
5886; SSE2-NEXT:    pxor %xmm2, %xmm2
5887; SSE2-NEXT:    pxor %xmm3, %xmm3
5888; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5889; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5890; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5891; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5892; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5893; SSE2-NEXT:    retq
5894;
5895; SSSE3-LABEL: sext_4i8_to_4i64:
5896; SSSE3:       # %bb.0:
5897; SSSE3-NEXT:    pslld $24, %xmm0
5898; SSSE3-NEXT:    psrad $24, %xmm0
5899; SSSE3-NEXT:    pxor %xmm2, %xmm2
5900; SSSE3-NEXT:    pxor %xmm3, %xmm3
5901; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5902; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5903; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5904; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5905; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5906; SSSE3-NEXT:    retq
5907;
5908; SSE41-LABEL: sext_4i8_to_4i64:
5909; SSE41:       # %bb.0:
5910; SSE41-NEXT:    pslld $24, %xmm0
5911; SSE41-NEXT:    psrad $24, %xmm0
5912; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5913; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5914; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5915; SSE41-NEXT:    movdqa %xmm2, %xmm0
5916; SSE41-NEXT:    retq
5917;
5918; AVX1-LABEL: sext_4i8_to_4i64:
5919; AVX1:       # %bb.0:
5920; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
5921; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
5922; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
5923; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5924; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
5925; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5926; AVX1-NEXT:    retq
5927;
5928; AVX2-LABEL: sext_4i8_to_4i64:
5929; AVX2:       # %bb.0:
5930; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
5931; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
5932; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
5933; AVX2-NEXT:    retq
5934;
5935; AVX512-LABEL: sext_4i8_to_4i64:
5936; AVX512:       # %bb.0:
5937; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
5938; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
5939; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
5940; AVX512-NEXT:    retq
5941;
5942; X32-SSE2-LABEL: sext_4i8_to_4i64:
5943; X32-SSE2:       # %bb.0:
5944; X32-SSE2-NEXT:    pslld $24, %xmm0
5945; X32-SSE2-NEXT:    psrad $24, %xmm0
5946; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5947; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5948; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5949; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5950; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5951; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5952; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5953; X32-SSE2-NEXT:    retl
5954;
5955; X32-SSE41-LABEL: sext_4i8_to_4i64:
5956; X32-SSE41:       # %bb.0:
5957; X32-SSE41-NEXT:    pslld $24, %xmm0
5958; X32-SSE41-NEXT:    psrad $24, %xmm0
5959; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5960; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5961; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5962; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
5963; X32-SSE41-NEXT:    retl
5964  %extmask = sext <4 x i8> %mask to <4 x i64>
5965  ret <4 x i64> %extmask
5966}
5967
5968define <32 x i8> @sext_32xi1_to_32xi8(<32 x i16> %c1, <32 x i16> %c2)nounwind {
5969; SSE-LABEL: sext_32xi1_to_32xi8:
5970; SSE:       # %bb.0:
5971; SSE-NEXT:    pcmpeqw %xmm5, %xmm1
5972; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
5973; SSE-NEXT:    packsswb %xmm1, %xmm0
5974; SSE-NEXT:    pcmpeqw %xmm7, %xmm3
5975; SSE-NEXT:    pcmpeqw %xmm6, %xmm2
5976; SSE-NEXT:    packsswb %xmm3, %xmm2
5977; SSE-NEXT:    movdqa %xmm2, %xmm1
5978; SSE-NEXT:    retq
5979;
5980; AVX1-LABEL: sext_32xi1_to_32xi8:
5981; AVX1:       # %bb.0:
5982; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
5983; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
5984; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm5, %xmm4
5985; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
5986; AVX1-NEXT:    vpacksswb %xmm4, %xmm1, %xmm1
5987; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
5988; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
5989; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm4, %xmm3
5990; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
5991; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
5992; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
5993; AVX1-NEXT:    retq
5994;
5995; AVX2-LABEL: sext_32xi1_to_32xi8:
5996; AVX2:       # %bb.0:
5997; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
5998; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
5999; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
6000; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
6001; AVX2-NEXT:    retq
6002;
6003; AVX512F-LABEL: sext_32xi1_to_32xi8:
6004; AVX512F:       # %bb.0:
6005; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
6006; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
6007; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
6008; AVX512F-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
6009; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
6010; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
6011; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
6012; AVX512F-NEXT:    retq
6013;
6014; AVX512BW-LABEL: sext_32xi1_to_32xi8:
6015; AVX512BW:       # %bb.0:
6016; AVX512BW-NEXT:    vpcmpeqw %zmm1, %zmm0, %k0
6017; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
6018; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
6019; AVX512BW-NEXT:    retq
6020;
6021; X32-SSE2-LABEL: sext_32xi1_to_32xi8:
6022; X32-SSE2:       # %bb.0:
6023; X32-SSE2-NEXT:    pushl %ebp
6024; X32-SSE2-NEXT:    movl %esp, %ebp
6025; X32-SSE2-NEXT:    andl $-16, %esp
6026; X32-SSE2-NEXT:    subl $16, %esp
6027; X32-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
6028; X32-SSE2-NEXT:    pcmpeqw 40(%ebp), %xmm1
6029; X32-SSE2-NEXT:    pcmpeqw 24(%ebp), %xmm0
6030; X32-SSE2-NEXT:    packsswb %xmm1, %xmm0
6031; X32-SSE2-NEXT:    pcmpeqw 72(%ebp), %xmm3
6032; X32-SSE2-NEXT:    pcmpeqw 56(%ebp), %xmm2
6033; X32-SSE2-NEXT:    packsswb %xmm3, %xmm2
6034; X32-SSE2-NEXT:    movdqa %xmm2, %xmm1
6035; X32-SSE2-NEXT:    movl %ebp, %esp
6036; X32-SSE2-NEXT:    popl %ebp
6037; X32-SSE2-NEXT:    retl
6038;
6039; X32-SSE41-LABEL: sext_32xi1_to_32xi8:
6040; X32-SSE41:       # %bb.0:
6041; X32-SSE41-NEXT:    pushl %ebp
6042; X32-SSE41-NEXT:    movl %esp, %ebp
6043; X32-SSE41-NEXT:    andl $-16, %esp
6044; X32-SSE41-NEXT:    subl $16, %esp
6045; X32-SSE41-NEXT:    movdqa 8(%ebp), %xmm3
6046; X32-SSE41-NEXT:    pcmpeqw 40(%ebp), %xmm1
6047; X32-SSE41-NEXT:    pcmpeqw 24(%ebp), %xmm0
6048; X32-SSE41-NEXT:    packsswb %xmm1, %xmm0
6049; X32-SSE41-NEXT:    pcmpeqw 72(%ebp), %xmm3
6050; X32-SSE41-NEXT:    pcmpeqw 56(%ebp), %xmm2
6051; X32-SSE41-NEXT:    packsswb %xmm3, %xmm2
6052; X32-SSE41-NEXT:    movdqa %xmm2, %xmm1
6053; X32-SSE41-NEXT:    movl %ebp, %esp
6054; X32-SSE41-NEXT:    popl %ebp
6055; X32-SSE41-NEXT:    retl
6056  %a = icmp eq <32 x i16> %c1, %c2
6057  %b = sext <32 x i1> %a to <32 x i8>
6058  ret <32 x i8> %b
6059}
6060
6061define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
6062; SSE2-LABEL: sext_2i8_to_2i32:
6063; SSE2:       # %bb.0:
6064; SSE2-NEXT:    movzwl (%rdi), %eax
6065; SSE2-NEXT:    movd %eax, %xmm0
6066; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
6067; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
6068; SSE2-NEXT:    psrad $24, %xmm0
6069; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
6070; SSE2-NEXT:    paddq %xmm0, %xmm0
6071; SSE2-NEXT:    retq
6072;
6073; SSSE3-LABEL: sext_2i8_to_2i32:
6074; SSSE3:       # %bb.0:
6075; SSSE3-NEXT:    movzwl (%rdi), %eax
6076; SSSE3-NEXT:    movd %eax, %xmm0
6077; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
6078; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
6079; SSSE3-NEXT:    psrad $24, %xmm0
6080; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
6081; SSSE3-NEXT:    paddq %xmm0, %xmm0
6082; SSSE3-NEXT:    retq
6083;
6084; SSE41-LABEL: sext_2i8_to_2i32:
6085; SSE41:       # %bb.0:
6086; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
6087; SSE41-NEXT:    paddq %xmm0, %xmm0
6088; SSE41-NEXT:    retq
6089;
6090; AVX-LABEL: sext_2i8_to_2i32:
6091; AVX:       # %bb.0:
6092; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
6093; AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
6094; AVX-NEXT:    retq
6095;
6096; X32-SSE2-LABEL: sext_2i8_to_2i32:
6097; X32-SSE2:       # %bb.0:
6098; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
6099; X32-SSE2-NEXT:    movzwl (%eax), %eax
6100; X32-SSE2-NEXT:    movd %eax, %xmm0
6101; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
6102; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
6103; X32-SSE2-NEXT:    psrad $24, %xmm0
6104; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
6105; X32-SSE2-NEXT:    paddq %xmm0, %xmm0
6106; X32-SSE2-NEXT:    retl
6107;
6108; X32-SSE41-LABEL: sext_2i8_to_2i32:
6109; X32-SSE41:       # %bb.0:
6110; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
6111; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
6112; X32-SSE41-NEXT:    paddq %xmm0, %xmm0
6113; X32-SSE41-NEXT:    retl
6114  %x = load <2 x i8>, <2 x i8>* %addr, align 1
6115  %y = sext <2 x i8> %x to <2 x i32>
6116  %z = add <2 x i32>%y, %y
6117  ret <2 x i32>%z
6118}
6119
6120