1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
9;
10; Just two 32-bit runs to make sure we do reasonable things there.
11; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE2
12; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41
13
14define <8 x i16> @sext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
15; SSE2-LABEL: sext_16i8_to_8i16:
16; SSE2:       # %bb.0: # %entry
17; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
18; SSE2-NEXT:    psraw $8, %xmm0
19; SSE2-NEXT:    retq
20;
21; SSSE3-LABEL: sext_16i8_to_8i16:
22; SSSE3:       # %bb.0: # %entry
23; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
24; SSSE3-NEXT:    psraw $8, %xmm0
25; SSSE3-NEXT:    retq
26;
27; SSE41-LABEL: sext_16i8_to_8i16:
28; SSE41:       # %bb.0: # %entry
29; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
30; SSE41-NEXT:    retq
31;
32; AVX-LABEL: sext_16i8_to_8i16:
33; AVX:       # %bb.0: # %entry
34; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
35; AVX-NEXT:    retq
36;
37; X32-SSE2-LABEL: sext_16i8_to_8i16:
38; X32-SSE2:       # %bb.0: # %entry
39; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
40; X32-SSE2-NEXT:    psraw $8, %xmm0
41; X32-SSE2-NEXT:    retl
42;
43; X32-SSE41-LABEL: sext_16i8_to_8i16:
44; X32-SSE41:       # %bb.0: # %entry
45; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
46; X32-SSE41-NEXT:    retl
47entry:
48  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
49  %C = sext <8 x i8> %B to <8 x i16>
50  ret <8 x i16> %C
51}
52
53define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ssp {
54; SSE2-LABEL: sext_16i8_to_16i16:
55; SSE2:       # %bb.0: # %entry
56; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
57; SSE2-NEXT:    psraw $8, %xmm2
58; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
59; SSE2-NEXT:    psraw $8, %xmm1
60; SSE2-NEXT:    movdqa %xmm2, %xmm0
61; SSE2-NEXT:    retq
62;
63; SSSE3-LABEL: sext_16i8_to_16i16:
64; SSSE3:       # %bb.0: # %entry
65; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
66; SSSE3-NEXT:    psraw $8, %xmm2
67; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
68; SSSE3-NEXT:    psraw $8, %xmm1
69; SSSE3-NEXT:    movdqa %xmm2, %xmm0
70; SSSE3-NEXT:    retq
71;
72; SSE41-LABEL: sext_16i8_to_16i16:
73; SSE41:       # %bb.0: # %entry
74; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
75; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
76; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
77; SSE41-NEXT:    movdqa %xmm2, %xmm0
78; SSE41-NEXT:    retq
79;
80; AVX1-LABEL: sext_16i8_to_16i16:
81; AVX1:       # %bb.0: # %entry
82; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
83; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
84; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
85; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
86; AVX1-NEXT:    retq
87;
88; AVX2-LABEL: sext_16i8_to_16i16:
89; AVX2:       # %bb.0: # %entry
90; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
91; AVX2-NEXT:    retq
92;
93; AVX512-LABEL: sext_16i8_to_16i16:
94; AVX512:       # %bb.0: # %entry
95; AVX512-NEXT:    vpmovsxbw %xmm0, %ymm0
96; AVX512-NEXT:    retq
97;
98; X32-SSE2-LABEL: sext_16i8_to_16i16:
99; X32-SSE2:       # %bb.0: # %entry
100; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
101; X32-SSE2-NEXT:    psraw $8, %xmm2
102; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
103; X32-SSE2-NEXT:    psraw $8, %xmm1
104; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
105; X32-SSE2-NEXT:    retl
106;
107; X32-SSE41-LABEL: sext_16i8_to_16i16:
108; X32-SSE41:       # %bb.0: # %entry
109; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
110; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
111; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
112; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
113; X32-SSE41-NEXT:    retl
114entry:
115  %B = sext <16 x i8> %A to <16 x i16>
116  ret <16 x i16> %B
117}
118
119define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ssp {
120; SSE2-LABEL: sext_32i8_to_32i16:
121; SSE2:       # %bb.0: # %entry
122; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
123; SSE2-NEXT:    psraw $8, %xmm4
124; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
125; SSE2-NEXT:    psraw $8, %xmm5
126; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
127; SSE2-NEXT:    psraw $8, %xmm2
128; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
129; SSE2-NEXT:    psraw $8, %xmm3
130; SSE2-NEXT:    movdqa %xmm4, %xmm0
131; SSE2-NEXT:    movdqa %xmm5, %xmm1
132; SSE2-NEXT:    retq
133;
134; SSSE3-LABEL: sext_32i8_to_32i16:
135; SSSE3:       # %bb.0: # %entry
136; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
137; SSSE3-NEXT:    psraw $8, %xmm4
138; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
139; SSSE3-NEXT:    psraw $8, %xmm5
140; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
141; SSSE3-NEXT:    psraw $8, %xmm2
142; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
143; SSSE3-NEXT:    psraw $8, %xmm3
144; SSSE3-NEXT:    movdqa %xmm4, %xmm0
145; SSSE3-NEXT:    movdqa %xmm5, %xmm1
146; SSSE3-NEXT:    retq
147;
148; SSE41-LABEL: sext_32i8_to_32i16:
149; SSE41:       # %bb.0: # %entry
150; SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
151; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
152; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
153; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
154; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
155; SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
156; SSE41-NEXT:    movdqa %xmm5, %xmm0
157; SSE41-NEXT:    movdqa %xmm4, %xmm1
158; SSE41-NEXT:    retq
159;
160; AVX1-LABEL: sext_32i8_to_32i16:
161; AVX1:       # %bb.0: # %entry
162; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
163; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
164; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
165; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
166; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
167; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
168; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
169; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
170; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
171; AVX1-NEXT:    vmovaps %ymm2, %ymm0
172; AVX1-NEXT:    retq
173;
174; AVX2-LABEL: sext_32i8_to_32i16:
175; AVX2:       # %bb.0: # %entry
176; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm2
177; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
178; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm1
179; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
180; AVX2-NEXT:    retq
181;
182; AVX512F-LABEL: sext_32i8_to_32i16:
183; AVX512F:       # %bb.0: # %entry
184; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm2
185; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
186; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
187; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
188; AVX512F-NEXT:    retq
189;
190; AVX512BW-LABEL: sext_32i8_to_32i16:
191; AVX512BW:       # %bb.0: # %entry
192; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
193; AVX512BW-NEXT:    retq
194;
195; X32-SSE2-LABEL: sext_32i8_to_32i16:
196; X32-SSE2:       # %bb.0: # %entry
197; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
198; X32-SSE2-NEXT:    psraw $8, %xmm4
199; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
200; X32-SSE2-NEXT:    psraw $8, %xmm5
201; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
202; X32-SSE2-NEXT:    psraw $8, %xmm2
203; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
204; X32-SSE2-NEXT:    psraw $8, %xmm3
205; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
206; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
207; X32-SSE2-NEXT:    retl
208;
209; X32-SSE41-LABEL: sext_32i8_to_32i16:
210; X32-SSE41:       # %bb.0: # %entry
211; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
212; X32-SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
213; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
214; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
215; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
216; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
217; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
218; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
219; X32-SSE41-NEXT:    retl
220entry:
221  %B = sext <32 x i8> %A to <32 x i16>
222  ret <32 x i16> %B
223}
224
225define <4 x i32> @sext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
226; SSE2-LABEL: sext_16i8_to_4i32:
227; SSE2:       # %bb.0: # %entry
228; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
229; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
230; SSE2-NEXT:    psrad $24, %xmm0
231; SSE2-NEXT:    retq
232;
233; SSSE3-LABEL: sext_16i8_to_4i32:
234; SSSE3:       # %bb.0: # %entry
235; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
236; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
237; SSSE3-NEXT:    psrad $24, %xmm0
238; SSSE3-NEXT:    retq
239;
240; SSE41-LABEL: sext_16i8_to_4i32:
241; SSE41:       # %bb.0: # %entry
242; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
243; SSE41-NEXT:    retq
244;
245; AVX-LABEL: sext_16i8_to_4i32:
246; AVX:       # %bb.0: # %entry
247; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
248; AVX-NEXT:    retq
249;
250; X32-SSE2-LABEL: sext_16i8_to_4i32:
251; X32-SSE2:       # %bb.0: # %entry
252; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
253; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
254; X32-SSE2-NEXT:    psrad $24, %xmm0
255; X32-SSE2-NEXT:    retl
256;
257; X32-SSE41-LABEL: sext_16i8_to_4i32:
258; X32-SSE41:       # %bb.0: # %entry
259; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
260; X32-SSE41-NEXT:    retl
261entry:
262  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
263  %C = sext <4 x i8> %B to <4 x i32>
264  ret <4 x i32> %C
265}
266
267define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
268; SSE2-LABEL: sext_16i8_to_8i32:
269; SSE2:       # %bb.0: # %entry
270; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
271; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
272; SSE2-NEXT:    psrad $24, %xmm2
273; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
274; SSE2-NEXT:    psrad $24, %xmm1
275; SSE2-NEXT:    movdqa %xmm2, %xmm0
276; SSE2-NEXT:    retq
277;
278; SSSE3-LABEL: sext_16i8_to_8i32:
279; SSSE3:       # %bb.0: # %entry
280; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
281; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
282; SSSE3-NEXT:    psrad $24, %xmm2
283; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
284; SSSE3-NEXT:    psrad $24, %xmm1
285; SSSE3-NEXT:    movdqa %xmm2, %xmm0
286; SSSE3-NEXT:    retq
287;
288; SSE41-LABEL: sext_16i8_to_8i32:
289; SSE41:       # %bb.0: # %entry
290; SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
291; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
292; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
293; SSE41-NEXT:    movdqa %xmm2, %xmm0
294; SSE41-NEXT:    retq
295;
296; AVX1-LABEL: sext_16i8_to_8i32:
297; AVX1:       # %bb.0: # %entry
298; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
299; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
300; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
301; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
302; AVX1-NEXT:    retq
303;
304; AVX2-LABEL: sext_16i8_to_8i32:
305; AVX2:       # %bb.0: # %entry
306; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
307; AVX2-NEXT:    retq
308;
309; AVX512-LABEL: sext_16i8_to_8i32:
310; AVX512:       # %bb.0: # %entry
311; AVX512-NEXT:    vpmovsxbd %xmm0, %ymm0
312; AVX512-NEXT:    retq
313;
314; X32-SSE2-LABEL: sext_16i8_to_8i32:
315; X32-SSE2:       # %bb.0: # %entry
316; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
317; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
318; X32-SSE2-NEXT:    psrad $24, %xmm2
319; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
320; X32-SSE2-NEXT:    psrad $24, %xmm1
321; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
322; X32-SSE2-NEXT:    retl
323;
324; X32-SSE41-LABEL: sext_16i8_to_8i32:
325; X32-SSE41:       # %bb.0: # %entry
326; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
327; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
328; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
329; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
330; X32-SSE41-NEXT:    retl
331entry:
332  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
333  %C = sext <8 x i8> %B to <8 x i32>
334  ret <8 x i32> %C
335}
336
337define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ssp {
338; SSE2-LABEL: sext_16i8_to_16i32:
339; SSE2:       # %bb.0: # %entry
340; SSE2-NEXT:    movdqa %xmm0, %xmm1
341; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
342; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
343; SSE2-NEXT:    psrad $24, %xmm4
344; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
345; SSE2-NEXT:    psrad $24, %xmm1
346; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
347; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
348; SSE2-NEXT:    psrad $24, %xmm2
349; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
350; SSE2-NEXT:    psrad $24, %xmm3
351; SSE2-NEXT:    movdqa %xmm4, %xmm0
352; SSE2-NEXT:    retq
353;
354; SSSE3-LABEL: sext_16i8_to_16i32:
355; SSSE3:       # %bb.0: # %entry
356; SSSE3-NEXT:    movdqa %xmm0, %xmm1
357; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
358; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
359; SSSE3-NEXT:    psrad $24, %xmm4
360; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
361; SSSE3-NEXT:    psrad $24, %xmm1
362; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
363; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
364; SSSE3-NEXT:    psrad $24, %xmm2
365; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
366; SSSE3-NEXT:    psrad $24, %xmm3
367; SSSE3-NEXT:    movdqa %xmm4, %xmm0
368; SSSE3-NEXT:    retq
369;
370; SSE41-LABEL: sext_16i8_to_16i32:
371; SSE41:       # %bb.0: # %entry
372; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
373; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
374; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
375; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
376; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
377; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
378; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
379; SSE41-NEXT:    movdqa %xmm4, %xmm0
380; SSE41-NEXT:    retq
381;
382; AVX1-LABEL: sext_16i8_to_16i32:
383; AVX1:       # %bb.0: # %entry
384; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
385; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
386; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
387; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
388; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
389; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
390; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
391; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
392; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
393; AVX1-NEXT:    vmovaps %ymm2, %ymm0
394; AVX1-NEXT:    retq
395;
396; AVX2-LABEL: sext_16i8_to_16i32:
397; AVX2:       # %bb.0: # %entry
398; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm2
399; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
400; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm1
401; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
402; AVX2-NEXT:    retq
403;
404; AVX512-LABEL: sext_16i8_to_16i32:
405; AVX512:       # %bb.0: # %entry
406; AVX512-NEXT:    vpmovsxbd %xmm0, %zmm0
407; AVX512-NEXT:    retq
408;
409; X32-SSE2-LABEL: sext_16i8_to_16i32:
410; X32-SSE2:       # %bb.0: # %entry
411; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
412; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
413; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
414; X32-SSE2-NEXT:    psrad $24, %xmm4
415; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
416; X32-SSE2-NEXT:    psrad $24, %xmm1
417; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
418; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
419; X32-SSE2-NEXT:    psrad $24, %xmm2
420; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
421; X32-SSE2-NEXT:    psrad $24, %xmm3
422; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
423; X32-SSE2-NEXT:    retl
424;
425; X32-SSE41-LABEL: sext_16i8_to_16i32:
426; X32-SSE41:       # %bb.0: # %entry
427; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
428; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
429; X32-SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
430; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
431; X32-SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
432; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
433; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
434; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
435; X32-SSE41-NEXT:    retl
436entry:
437  %B = sext <16 x i8> %A to <16 x i32>
438  ret <16 x i32> %B
439}
440
441define <2 x i64> @sext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
442; SSE2-LABEL: sext_16i8_to_2i64:
443; SSE2:       # %bb.0: # %entry
444; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
445; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
446; SSE2-NEXT:    pxor %xmm1, %xmm1
447; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
448; SSE2-NEXT:    psrad $24, %xmm0
449; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
450; SSE2-NEXT:    retq
451;
452; SSSE3-LABEL: sext_16i8_to_2i64:
453; SSSE3:       # %bb.0: # %entry
454; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
455; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
456; SSSE3-NEXT:    pxor %xmm1, %xmm1
457; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
458; SSSE3-NEXT:    psrad $24, %xmm0
459; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
460; SSSE3-NEXT:    retq
461;
462; SSE41-LABEL: sext_16i8_to_2i64:
463; SSE41:       # %bb.0: # %entry
464; SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
465; SSE41-NEXT:    retq
466;
467; AVX-LABEL: sext_16i8_to_2i64:
468; AVX:       # %bb.0: # %entry
469; AVX-NEXT:    vpmovsxbq %xmm0, %xmm0
470; AVX-NEXT:    retq
471;
472; X32-SSE2-LABEL: sext_16i8_to_2i64:
473; X32-SSE2:       # %bb.0: # %entry
474; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
475; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
476; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
477; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
478; X32-SSE2-NEXT:    psrad $24, %xmm0
479; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
480; X32-SSE2-NEXT:    retl
481;
482; X32-SSE41-LABEL: sext_16i8_to_2i64:
483; X32-SSE41:       # %bb.0: # %entry
484; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
485; X32-SSE41-NEXT:    retl
486entry:
487  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
488  %C = sext <2 x i8> %B to <2 x i64>
489  ret <2 x i64> %C
490}
491
492define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
493; SSE2-LABEL: sext_16i8_to_4i64:
494; SSE2:       # %bb.0: # %entry
495; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
496; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
497; SSE2-NEXT:    psrad $24, %xmm1
498; SSE2-NEXT:    pxor %xmm2, %xmm2
499; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
500; SSE2-NEXT:    movdqa %xmm1, %xmm0
501; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
502; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
503; SSE2-NEXT:    retq
504;
505; SSSE3-LABEL: sext_16i8_to_4i64:
506; SSSE3:       # %bb.0: # %entry
507; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
508; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
509; SSSE3-NEXT:    psrad $24, %xmm1
510; SSSE3-NEXT:    pxor %xmm2, %xmm2
511; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
512; SSSE3-NEXT:    movdqa %xmm1, %xmm0
513; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
514; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
515; SSSE3-NEXT:    retq
516;
517; SSE41-LABEL: sext_16i8_to_4i64:
518; SSE41:       # %bb.0: # %entry
519; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
520; SSE41-NEXT:    psrld $16, %xmm0
521; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
522; SSE41-NEXT:    movdqa %xmm2, %xmm0
523; SSE41-NEXT:    retq
524;
525; AVX1-LABEL: sext_16i8_to_4i64:
526; AVX1:       # %bb.0: # %entry
527; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
528; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
529; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
530; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
531; AVX1-NEXT:    retq
532;
533; AVX2-LABEL: sext_16i8_to_4i64:
534; AVX2:       # %bb.0: # %entry
535; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm0
536; AVX2-NEXT:    retq
537;
538; AVX512-LABEL: sext_16i8_to_4i64:
539; AVX512:       # %bb.0: # %entry
540; AVX512-NEXT:    vpmovsxbq %xmm0, %ymm0
541; AVX512-NEXT:    retq
542;
543; X32-SSE2-LABEL: sext_16i8_to_4i64:
544; X32-SSE2:       # %bb.0: # %entry
545; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
546; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
547; X32-SSE2-NEXT:    psrad $24, %xmm1
548; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
549; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
550; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
551; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
552; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
553; X32-SSE2-NEXT:    retl
554;
555; X32-SSE41-LABEL: sext_16i8_to_4i64:
556; X32-SSE41:       # %bb.0: # %entry
557; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
558; X32-SSE41-NEXT:    psrld $16, %xmm0
559; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
560; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
561; X32-SSE41-NEXT:    retl
562entry:
563  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
564  %C = sext <4 x i8> %B to <4 x i64>
565  ret <4 x i64> %C
566}
567
568define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
569; SSE2-LABEL: sext_16i8_to_8i64:
570; SSE2:       # %bb.0: # %entry
571; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
572; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
573; SSE2-NEXT:    psrad $24, %xmm1
574; SSE2-NEXT:    pxor %xmm5, %xmm5
575; SSE2-NEXT:    pxor %xmm2, %xmm2
576; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
577; SSE2-NEXT:    movdqa %xmm1, %xmm4
578; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
579; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
580; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
581; SSE2-NEXT:    psrad $24, %xmm3
582; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
583; SSE2-NEXT:    movdqa %xmm3, %xmm2
584; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
585; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
586; SSE2-NEXT:    movdqa %xmm4, %xmm0
587; SSE2-NEXT:    retq
588;
589; SSSE3-LABEL: sext_16i8_to_8i64:
590; SSSE3:       # %bb.0: # %entry
591; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
592; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
593; SSSE3-NEXT:    psrad $24, %xmm1
594; SSSE3-NEXT:    pxor %xmm5, %xmm5
595; SSSE3-NEXT:    pxor %xmm2, %xmm2
596; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
597; SSSE3-NEXT:    movdqa %xmm1, %xmm4
598; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
599; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
600; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
601; SSSE3-NEXT:    psrad $24, %xmm3
602; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
603; SSSE3-NEXT:    movdqa %xmm3, %xmm2
604; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
605; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
606; SSSE3-NEXT:    movdqa %xmm4, %xmm0
607; SSSE3-NEXT:    retq
608;
609; SSE41-LABEL: sext_16i8_to_8i64:
610; SSE41:       # %bb.0: # %entry
611; SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
612; SSE41-NEXT:    movdqa %xmm0, %xmm1
613; SSE41-NEXT:    psrld $16, %xmm1
614; SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
615; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
616; SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
617; SSE41-NEXT:    psrlq $48, %xmm0
618; SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
619; SSE41-NEXT:    movdqa %xmm4, %xmm0
620; SSE41-NEXT:    retq
621;
622; AVX1-LABEL: sext_16i8_to_8i64:
623; AVX1:       # %bb.0: # %entry
624; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
625; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
626; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
627; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
628; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
629; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
630; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
631; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
632; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
633; AVX1-NEXT:    vmovaps %ymm2, %ymm0
634; AVX1-NEXT:    retq
635;
636; AVX2-LABEL: sext_16i8_to_8i64:
637; AVX2:       # %bb.0: # %entry
638; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm2
639; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
640; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm1
641; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
642; AVX2-NEXT:    retq
643;
644; AVX512-LABEL: sext_16i8_to_8i64:
645; AVX512:       # %bb.0: # %entry
646; AVX512-NEXT:    vpmovsxbq %xmm0, %zmm0
647; AVX512-NEXT:    retq
648;
649; X32-SSE2-LABEL: sext_16i8_to_8i64:
650; X32-SSE2:       # %bb.0: # %entry
651; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
652; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
653; X32-SSE2-NEXT:    psrad $24, %xmm1
654; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
655; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
656; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
657; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
658; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
659; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
660; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
661; X32-SSE2-NEXT:    psrad $24, %xmm3
662; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
663; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
664; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
665; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
666; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
667; X32-SSE2-NEXT:    retl
668;
669; X32-SSE41-LABEL: sext_16i8_to_8i64:
670; X32-SSE41:       # %bb.0: # %entry
671; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm4
672; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
673; X32-SSE41-NEXT:    psrld $16, %xmm1
674; X32-SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
675; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
676; X32-SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
677; X32-SSE41-NEXT:    psrlq $48, %xmm0
678; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
679; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
680; X32-SSE41-NEXT:    retl
681entry:
682  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
683  %C = sext <8 x i8> %B to <8 x i64>
684  ret <8 x i64> %C
685}
686
687define <4 x i32> @sext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
688; SSE2-LABEL: sext_8i16_to_4i32:
689; SSE2:       # %bb.0: # %entry
690; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
691; SSE2-NEXT:    psrad $16, %xmm0
692; SSE2-NEXT:    retq
693;
694; SSSE3-LABEL: sext_8i16_to_4i32:
695; SSSE3:       # %bb.0: # %entry
696; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
697; SSSE3-NEXT:    psrad $16, %xmm0
698; SSSE3-NEXT:    retq
699;
700; SSE41-LABEL: sext_8i16_to_4i32:
701; SSE41:       # %bb.0: # %entry
702; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
703; SSE41-NEXT:    retq
704;
705; AVX-LABEL: sext_8i16_to_4i32:
706; AVX:       # %bb.0: # %entry
707; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
708; AVX-NEXT:    retq
709;
710; X32-SSE2-LABEL: sext_8i16_to_4i32:
711; X32-SSE2:       # %bb.0: # %entry
712; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
713; X32-SSE2-NEXT:    psrad $16, %xmm0
714; X32-SSE2-NEXT:    retl
715;
716; X32-SSE41-LABEL: sext_8i16_to_4i32:
717; X32-SSE41:       # %bb.0: # %entry
718; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
719; X32-SSE41-NEXT:    retl
720entry:
721  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
722  %C = sext <4 x i16> %B to <4 x i32>
723  ret <4 x i32> %C
724}
725
726define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
727; SSE2-LABEL: sext_8i16_to_8i32:
728; SSE2:       # %bb.0: # %entry
729; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
730; SSE2-NEXT:    psrad $16, %xmm2
731; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
732; SSE2-NEXT:    psrad $16, %xmm1
733; SSE2-NEXT:    movdqa %xmm2, %xmm0
734; SSE2-NEXT:    retq
735;
736; SSSE3-LABEL: sext_8i16_to_8i32:
737; SSSE3:       # %bb.0: # %entry
738; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
739; SSSE3-NEXT:    psrad $16, %xmm2
740; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
741; SSSE3-NEXT:    psrad $16, %xmm1
742; SSSE3-NEXT:    movdqa %xmm2, %xmm0
743; SSSE3-NEXT:    retq
744;
745; SSE41-LABEL: sext_8i16_to_8i32:
746; SSE41:       # %bb.0: # %entry
747; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
748; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
749; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
750; SSE41-NEXT:    movdqa %xmm2, %xmm0
751; SSE41-NEXT:    retq
752;
753; AVX1-LABEL: sext_8i16_to_8i32:
754; AVX1:       # %bb.0: # %entry
755; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
756; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
757; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
758; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
759; AVX1-NEXT:    retq
760;
761; AVX2-LABEL: sext_8i16_to_8i32:
762; AVX2:       # %bb.0: # %entry
763; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
764; AVX2-NEXT:    retq
765;
766; AVX512-LABEL: sext_8i16_to_8i32:
767; AVX512:       # %bb.0: # %entry
768; AVX512-NEXT:    vpmovsxwd %xmm0, %ymm0
769; AVX512-NEXT:    retq
770;
771; X32-SSE2-LABEL: sext_8i16_to_8i32:
772; X32-SSE2:       # %bb.0: # %entry
773; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
774; X32-SSE2-NEXT:    psrad $16, %xmm2
775; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
776; X32-SSE2-NEXT:    psrad $16, %xmm1
777; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
778; X32-SSE2-NEXT:    retl
779;
780; X32-SSE41-LABEL: sext_8i16_to_8i32:
781; X32-SSE41:       # %bb.0: # %entry
782; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
783; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
784; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
785; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
786; X32-SSE41-NEXT:    retl
787entry:
788  %B = sext <8 x i16> %A to <8 x i32>
789  ret <8 x i32> %B
790}
791
792define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone ssp {
793; SSE2-LABEL: sext_16i16_to_16i32:
794; SSE2:       # %bb.0: # %entry
795; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
796; SSE2-NEXT:    psrad $16, %xmm4
797; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
798; SSE2-NEXT:    psrad $16, %xmm5
799; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
800; SSE2-NEXT:    psrad $16, %xmm2
801; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
802; SSE2-NEXT:    psrad $16, %xmm3
803; SSE2-NEXT:    movdqa %xmm4, %xmm0
804; SSE2-NEXT:    movdqa %xmm5, %xmm1
805; SSE2-NEXT:    retq
806;
807; SSSE3-LABEL: sext_16i16_to_16i32:
808; SSSE3:       # %bb.0: # %entry
809; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
810; SSSE3-NEXT:    psrad $16, %xmm4
811; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
812; SSSE3-NEXT:    psrad $16, %xmm5
813; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
814; SSSE3-NEXT:    psrad $16, %xmm2
815; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
816; SSSE3-NEXT:    psrad $16, %xmm3
817; SSSE3-NEXT:    movdqa %xmm4, %xmm0
818; SSSE3-NEXT:    movdqa %xmm5, %xmm1
819; SSSE3-NEXT:    retq
820;
821; SSE41-LABEL: sext_16i16_to_16i32:
822; SSE41:       # %bb.0: # %entry
823; SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
824; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
825; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
826; SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
827; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
828; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
829; SSE41-NEXT:    movdqa %xmm5, %xmm0
830; SSE41-NEXT:    movdqa %xmm4, %xmm1
831; SSE41-NEXT:    retq
832;
833; AVX1-LABEL: sext_16i16_to_16i32:
834; AVX1:       # %bb.0: # %entry
835; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
836; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
837; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
838; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
839; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
840; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
841; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
842; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
843; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
844; AVX1-NEXT:    vmovaps %ymm2, %ymm0
845; AVX1-NEXT:    retq
846;
847; AVX2-LABEL: sext_16i16_to_16i32:
848; AVX2:       # %bb.0: # %entry
849; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm2
850; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
851; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm1
852; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
853; AVX2-NEXT:    retq
854;
855; AVX512-LABEL: sext_16i16_to_16i32:
856; AVX512:       # %bb.0: # %entry
857; AVX512-NEXT:    vpmovsxwd %ymm0, %zmm0
858; AVX512-NEXT:    retq
859;
860; X32-SSE2-LABEL: sext_16i16_to_16i32:
861; X32-SSE2:       # %bb.0: # %entry
862; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
863; X32-SSE2-NEXT:    psrad $16, %xmm4
864; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
865; X32-SSE2-NEXT:    psrad $16, %xmm5
866; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
867; X32-SSE2-NEXT:    psrad $16, %xmm2
868; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
869; X32-SSE2-NEXT:    psrad $16, %xmm3
870; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
871; X32-SSE2-NEXT:    movdqa %xmm5, %xmm1
872; X32-SSE2-NEXT:    retl
873;
874; X32-SSE41-LABEL: sext_16i16_to_16i32:
875; X32-SSE41:       # %bb.0: # %entry
876; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
877; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
878; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
879; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
880; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
881; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
882; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
883; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
884; X32-SSE41-NEXT:    retl
885entry:
886  %B = sext <16 x i16> %A to <16 x i32>
887  ret <16 x i32> %B
888}
889
890define <2 x i64> @sext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
891; SSE2-LABEL: sext_8i16_to_2i64:
892; SSE2:       # %bb.0: # %entry
893; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
894; SSE2-NEXT:    pxor %xmm1, %xmm1
895; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
896; SSE2-NEXT:    psrad $16, %xmm0
897; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
898; SSE2-NEXT:    retq
899;
900; SSSE3-LABEL: sext_8i16_to_2i64:
901; SSSE3:       # %bb.0: # %entry
902; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
903; SSSE3-NEXT:    pxor %xmm1, %xmm1
904; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
905; SSSE3-NEXT:    psrad $16, %xmm0
906; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
907; SSSE3-NEXT:    retq
908;
909; SSE41-LABEL: sext_8i16_to_2i64:
910; SSE41:       # %bb.0: # %entry
911; SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
912; SSE41-NEXT:    retq
913;
914; AVX-LABEL: sext_8i16_to_2i64:
915; AVX:       # %bb.0: # %entry
916; AVX-NEXT:    vpmovsxwq %xmm0, %xmm0
917; AVX-NEXT:    retq
918;
919; X32-SSE2-LABEL: sext_8i16_to_2i64:
920; X32-SSE2:       # %bb.0: # %entry
921; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
922; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
923; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
924; X32-SSE2-NEXT:    psrad $16, %xmm0
925; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
926; X32-SSE2-NEXT:    retl
927;
928; X32-SSE41-LABEL: sext_8i16_to_2i64:
929; X32-SSE41:       # %bb.0: # %entry
930; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
931; X32-SSE41-NEXT:    retl
932entry:
933  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
934  %C = sext <2 x i16> %B to <2 x i64>
935  ret <2 x i64> %C
936}
937
938define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
939; SSE2-LABEL: sext_8i16_to_4i64:
940; SSE2:       # %bb.0: # %entry
941; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
942; SSE2-NEXT:    psrad $16, %xmm1
943; SSE2-NEXT:    pxor %xmm2, %xmm2
944; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
945; SSE2-NEXT:    movdqa %xmm1, %xmm0
946; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
947; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
948; SSE2-NEXT:    retq
949;
950; SSSE3-LABEL: sext_8i16_to_4i64:
951; SSSE3:       # %bb.0: # %entry
952; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
953; SSSE3-NEXT:    psrad $16, %xmm1
954; SSSE3-NEXT:    pxor %xmm2, %xmm2
955; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
956; SSSE3-NEXT:    movdqa %xmm1, %xmm0
957; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
958; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
959; SSSE3-NEXT:    retq
960;
961; SSE41-LABEL: sext_8i16_to_4i64:
962; SSE41:       # %bb.0: # %entry
963; SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
964; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
965; SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
966; SSE41-NEXT:    movdqa %xmm2, %xmm0
967; SSE41-NEXT:    retq
968;
969; AVX1-LABEL: sext_8i16_to_4i64:
970; AVX1:       # %bb.0: # %entry
971; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
972; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
973; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
974; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
975; AVX1-NEXT:    retq
976;
977; AVX2-LABEL: sext_8i16_to_4i64:
978; AVX2:       # %bb.0: # %entry
979; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
980; AVX2-NEXT:    retq
981;
982; AVX512-LABEL: sext_8i16_to_4i64:
983; AVX512:       # %bb.0: # %entry
984; AVX512-NEXT:    vpmovsxwq %xmm0, %ymm0
985; AVX512-NEXT:    retq
986;
987; X32-SSE2-LABEL: sext_8i16_to_4i64:
988; X32-SSE2:       # %bb.0: # %entry
989; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
990; X32-SSE2-NEXT:    psrad $16, %xmm1
991; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
992; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
993; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
994; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
995; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
996; X32-SSE2-NEXT:    retl
997;
998; X32-SSE41-LABEL: sext_8i16_to_4i64:
999; X32-SSE41:       # %bb.0: # %entry
1000; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
1001; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1002; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
1003; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
1004; X32-SSE41-NEXT:    retl
1005entry:
1006  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1007  %C = sext <4 x i16> %B to <4 x i64>
1008  ret <4 x i64> %C
1009}
1010
1011define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp {
1012; SSE2-LABEL: sext_8i16_to_8i64:
1013; SSE2:       # %bb.0: # %entry
1014; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1015; SSE2-NEXT:    psrad $16, %xmm1
1016; SSE2-NEXT:    pxor %xmm5, %xmm5
1017; SSE2-NEXT:    pxor %xmm2, %xmm2
1018; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1019; SSE2-NEXT:    movdqa %xmm1, %xmm4
1020; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1021; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1022; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1023; SSE2-NEXT:    psrad $16, %xmm3
1024; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1025; SSE2-NEXT:    movdqa %xmm3, %xmm2
1026; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1027; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1028; SSE2-NEXT:    movdqa %xmm4, %xmm0
1029; SSE2-NEXT:    retq
1030;
1031; SSSE3-LABEL: sext_8i16_to_8i64:
1032; SSSE3:       # %bb.0: # %entry
1033; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1034; SSSE3-NEXT:    psrad $16, %xmm1
1035; SSSE3-NEXT:    pxor %xmm5, %xmm5
1036; SSSE3-NEXT:    pxor %xmm2, %xmm2
1037; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1038; SSSE3-NEXT:    movdqa %xmm1, %xmm4
1039; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1040; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1041; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1042; SSSE3-NEXT:    psrad $16, %xmm3
1043; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
1044; SSSE3-NEXT:    movdqa %xmm3, %xmm2
1045; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1046; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1047; SSSE3-NEXT:    movdqa %xmm4, %xmm0
1048; SSSE3-NEXT:    retq
1049;
1050; SSE41-LABEL: sext_8i16_to_8i64:
1051; SSE41:       # %bb.0: # %entry
1052; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1053; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1054; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1055; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1056; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1057; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1058; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1059; SSE41-NEXT:    movdqa %xmm4, %xmm0
1060; SSE41-NEXT:    retq
1061;
1062; AVX1-LABEL: sext_8i16_to_8i64:
1063; AVX1:       # %bb.0: # %entry
1064; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
1065; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
1066; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
1067; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1068; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1069; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
1070; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
1071; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
1072; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1073; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1074; AVX1-NEXT:    retq
1075;
1076; AVX2-LABEL: sext_8i16_to_8i64:
1077; AVX2:       # %bb.0: # %entry
1078; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm2
1079; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1080; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm1
1081; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1082; AVX2-NEXT:    retq
1083;
1084; AVX512-LABEL: sext_8i16_to_8i64:
1085; AVX512:       # %bb.0: # %entry
1086; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
1087; AVX512-NEXT:    retq
1088;
1089; X32-SSE2-LABEL: sext_8i16_to_8i64:
1090; X32-SSE2:       # %bb.0: # %entry
1091; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1092; X32-SSE2-NEXT:    psrad $16, %xmm1
1093; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1094; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1095; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1096; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
1097; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
1098; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1099; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
1100; X32-SSE2-NEXT:    psrad $16, %xmm3
1101; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
1102; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
1103; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1104; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
1105; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
1106; X32-SSE2-NEXT:    retl
1107;
1108; X32-SSE41-LABEL: sext_8i16_to_8i64:
1109; X32-SSE41:       # %bb.0: # %entry
1110; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
1111; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1112; X32-SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
1113; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1114; X32-SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
1115; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
1116; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
1117; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
1118; X32-SSE41-NEXT:    retl
1119entry:
1120  %B = sext <8 x i16> %A to <8 x i64>
1121  ret <8 x i64> %B
1122}
1123
1124define <2 x i64> @sext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1125; SSE2-LABEL: sext_4i32_to_2i64:
1126; SSE2:       # %bb.0: # %entry
1127; SSE2-NEXT:    pxor %xmm1, %xmm1
1128; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1129; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1130; SSE2-NEXT:    retq
1131;
1132; SSSE3-LABEL: sext_4i32_to_2i64:
1133; SSSE3:       # %bb.0: # %entry
1134; SSSE3-NEXT:    pxor %xmm1, %xmm1
1135; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1136; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1137; SSSE3-NEXT:    retq
1138;
1139; SSE41-LABEL: sext_4i32_to_2i64:
1140; SSE41:       # %bb.0: # %entry
1141; SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1142; SSE41-NEXT:    retq
1143;
1144; AVX-LABEL: sext_4i32_to_2i64:
1145; AVX:       # %bb.0: # %entry
1146; AVX-NEXT:    vpmovsxdq %xmm0, %xmm0
1147; AVX-NEXT:    retq
1148;
1149; X32-SSE2-LABEL: sext_4i32_to_2i64:
1150; X32-SSE2:       # %bb.0: # %entry
1151; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1152; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1153; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1154; X32-SSE2-NEXT:    retl
1155;
1156; X32-SSE41-LABEL: sext_4i32_to_2i64:
1157; X32-SSE41:       # %bb.0: # %entry
1158; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
1159; X32-SSE41-NEXT:    retl
1160entry:
1161  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
1162  %C = sext <2 x i32> %B to <2 x i64>
1163  ret <2 x i64> %C
1164}
1165
1166define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1167; SSE2-LABEL: sext_4i32_to_4i64:
1168; SSE2:       # %bb.0: # %entry
1169; SSE2-NEXT:    pxor %xmm2, %xmm2
1170; SSE2-NEXT:    pxor %xmm3, %xmm3
1171; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1172; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1173; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1174; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1175; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1176; SSE2-NEXT:    retq
1177;
1178; SSSE3-LABEL: sext_4i32_to_4i64:
1179; SSSE3:       # %bb.0: # %entry
1180; SSSE3-NEXT:    pxor %xmm2, %xmm2
1181; SSSE3-NEXT:    pxor %xmm3, %xmm3
1182; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1183; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1184; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1185; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1186; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1187; SSSE3-NEXT:    retq
1188;
1189; SSE41-LABEL: sext_4i32_to_4i64:
1190; SSE41:       # %bb.0: # %entry
1191; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1192; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1193; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1194; SSE41-NEXT:    movdqa %xmm2, %xmm0
1195; SSE41-NEXT:    retq
1196;
1197; AVX1-LABEL: sext_4i32_to_4i64:
1198; AVX1:       # %bb.0: # %entry
1199; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1200; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1201; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1202; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1203; AVX1-NEXT:    retq
1204;
1205; AVX2-LABEL: sext_4i32_to_4i64:
1206; AVX2:       # %bb.0: # %entry
1207; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
1208; AVX2-NEXT:    retq
1209;
1210; AVX512-LABEL: sext_4i32_to_4i64:
1211; AVX512:       # %bb.0: # %entry
1212; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
1213; AVX512-NEXT:    retq
1214;
1215; X32-SSE2-LABEL: sext_4i32_to_4i64:
1216; X32-SSE2:       # %bb.0: # %entry
1217; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1218; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1219; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1220; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1221; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1222; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1223; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1224; X32-SSE2-NEXT:    retl
1225;
1226; X32-SSE41-LABEL: sext_4i32_to_4i64:
1227; X32-SSE41:       # %bb.0: # %entry
1228; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
1229; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1230; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
1231; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
1232; X32-SSE41-NEXT:    retl
1233entry:
1234  %B = sext <4 x i32> %A to <4 x i64>
1235  ret <4 x i64> %B
1236}
1237
1238define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp {
1239; SSE2-LABEL: sext_8i32_to_8i64:
1240; SSE2:       # %bb.0: # %entry
1241; SSE2-NEXT:    movdqa %xmm1, %xmm2
1242; SSE2-NEXT:    pxor %xmm4, %xmm4
1243; SSE2-NEXT:    pxor %xmm3, %xmm3
1244; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1245; SSE2-NEXT:    pxor %xmm5, %xmm5
1246; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1247; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1248; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1249; SSE2-NEXT:    pxor %xmm3, %xmm3
1250; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1251; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1252; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1253; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1254; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1255; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1256; SSE2-NEXT:    retq
1257;
1258; SSSE3-LABEL: sext_8i32_to_8i64:
1259; SSSE3:       # %bb.0: # %entry
1260; SSSE3-NEXT:    movdqa %xmm1, %xmm2
1261; SSSE3-NEXT:    pxor %xmm4, %xmm4
1262; SSSE3-NEXT:    pxor %xmm3, %xmm3
1263; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
1264; SSSE3-NEXT:    pxor %xmm5, %xmm5
1265; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
1266; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1267; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1268; SSSE3-NEXT:    pxor %xmm3, %xmm3
1269; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
1270; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1271; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1272; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1273; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
1274; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1275; SSSE3-NEXT:    retq
1276;
1277; SSE41-LABEL: sext_8i32_to_8i64:
1278; SSE41:       # %bb.0: # %entry
1279; SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1280; SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1281; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1282; SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1283; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1284; SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1285; SSE41-NEXT:    movdqa %xmm5, %xmm0
1286; SSE41-NEXT:    movdqa %xmm4, %xmm1
1287; SSE41-NEXT:    retq
1288;
1289; AVX1-LABEL: sext_8i32_to_8i64:
1290; AVX1:       # %bb.0: # %entry
1291; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1292; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
1293; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
1294; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
1295; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1296; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1297; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1298; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1299; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
1300; AVX1-NEXT:    vmovaps %ymm2, %ymm0
1301; AVX1-NEXT:    retq
1302;
1303; AVX2-LABEL: sext_8i32_to_8i64:
1304; AVX2:       # %bb.0: # %entry
1305; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm2
1306; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1307; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm1
1308; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
1309; AVX2-NEXT:    retq
1310;
1311; AVX512-LABEL: sext_8i32_to_8i64:
1312; AVX512:       # %bb.0: # %entry
1313; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
1314; AVX512-NEXT:    retq
1315;
1316; X32-SSE2-LABEL: sext_8i32_to_8i64:
1317; X32-SSE2:       # %bb.0: # %entry
1318; X32-SSE2-NEXT:    movdqa %xmm1, %xmm2
1319; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
1320; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1321; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
1322; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
1323; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
1324; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1325; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
1326; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
1327; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
1328; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1329; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
1330; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1331; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
1332; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1333; X32-SSE2-NEXT:    retl
1334;
1335; X32-SSE41-LABEL: sext_8i32_to_8i64:
1336; X32-SSE41:       # %bb.0: # %entry
1337; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
1338; X32-SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
1339; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1340; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
1341; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1342; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
1343; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
1344; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
1345; X32-SSE41-NEXT:    retl
1346entry:
1347  %B = sext <8 x i32> %A to <8 x i64>
1348  ret <8 x i64> %B
1349}
1350
1351define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) {
1352; SSE-LABEL: load_sext_2i1_to_2i64:
1353; SSE:       # %bb.0: # %entry
1354; SSE-NEXT:    movzbl (%rdi), %eax
1355; SSE-NEXT:    movq %rax, %rcx
1356; SSE-NEXT:    shlq $62, %rcx
1357; SSE-NEXT:    sarq $63, %rcx
1358; SSE-NEXT:    movq %rcx, %xmm1
1359; SSE-NEXT:    shlq $63, %rax
1360; SSE-NEXT:    sarq $63, %rax
1361; SSE-NEXT:    movq %rax, %xmm0
1362; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1363; SSE-NEXT:    retq
1364;
1365; AVX1-LABEL: load_sext_2i1_to_2i64:
1366; AVX1:       # %bb.0: # %entry
1367; AVX1-NEXT:    movzbl (%rdi), %eax
1368; AVX1-NEXT:    movq %rax, %rcx
1369; AVX1-NEXT:    shlq $62, %rcx
1370; AVX1-NEXT:    sarq $63, %rcx
1371; AVX1-NEXT:    vmovq %rcx, %xmm0
1372; AVX1-NEXT:    shlq $63, %rax
1373; AVX1-NEXT:    sarq $63, %rax
1374; AVX1-NEXT:    vmovq %rax, %xmm1
1375; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1376; AVX1-NEXT:    retq
1377;
1378; AVX2-LABEL: load_sext_2i1_to_2i64:
1379; AVX2:       # %bb.0: # %entry
1380; AVX2-NEXT:    movzbl (%rdi), %eax
1381; AVX2-NEXT:    movq %rax, %rcx
1382; AVX2-NEXT:    shlq $62, %rcx
1383; AVX2-NEXT:    sarq $63, %rcx
1384; AVX2-NEXT:    vmovq %rcx, %xmm0
1385; AVX2-NEXT:    shlq $63, %rax
1386; AVX2-NEXT:    sarq $63, %rax
1387; AVX2-NEXT:    vmovq %rax, %xmm1
1388; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1389; AVX2-NEXT:    retq
1390;
1391; AVX512F-LABEL: load_sext_2i1_to_2i64:
1392; AVX512F:       # %bb.0: # %entry
1393; AVX512F-NEXT:    movzbl (%rdi), %eax
1394; AVX512F-NEXT:    kmovw %eax, %k1
1395; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1396; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1397; AVX512F-NEXT:    vzeroupper
1398; AVX512F-NEXT:    retq
1399;
1400; AVX512BW-LABEL: load_sext_2i1_to_2i64:
1401; AVX512BW:       # %bb.0: # %entry
1402; AVX512BW-NEXT:    movzbl (%rdi), %eax
1403; AVX512BW-NEXT:    kmovd %eax, %k1
1404; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1405; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1406; AVX512BW-NEXT:    vzeroupper
1407; AVX512BW-NEXT:    retq
1408;
1409; X32-SSE2-LABEL: load_sext_2i1_to_2i64:
1410; X32-SSE2:       # %bb.0: # %entry
1411; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1412; X32-SSE2-NEXT:    movzbl (%eax), %eax
1413; X32-SSE2-NEXT:    movl %eax, %ecx
1414; X32-SSE2-NEXT:    shll $30, %ecx
1415; X32-SSE2-NEXT:    sarl $31, %ecx
1416; X32-SSE2-NEXT:    movd %ecx, %xmm0
1417; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]
1418; X32-SSE2-NEXT:    shll $31, %eax
1419; X32-SSE2-NEXT:    sarl $31, %eax
1420; X32-SSE2-NEXT:    movd %eax, %xmm0
1421; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
1422; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1423; X32-SSE2-NEXT:    retl
1424;
1425; X32-SSE41-LABEL: load_sext_2i1_to_2i64:
1426; X32-SSE41:       # %bb.0: # %entry
1427; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1428; X32-SSE41-NEXT:    movzbl (%eax), %eax
1429; X32-SSE41-NEXT:    movl %eax, %ecx
1430; X32-SSE41-NEXT:    shll $31, %ecx
1431; X32-SSE41-NEXT:    sarl $31, %ecx
1432; X32-SSE41-NEXT:    movd %ecx, %xmm0
1433; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1434; X32-SSE41-NEXT:    shll $30, %eax
1435; X32-SSE41-NEXT:    sarl $31, %eax
1436; X32-SSE41-NEXT:    pinsrd $2, %eax, %xmm0
1437; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1438; X32-SSE41-NEXT:    retl
1439entry:
1440 %X = load <2 x i1>, <2 x i1>* %ptr
1441 %Y = sext <2 x i1> %X to <2 x i64>
1442 ret <2 x i64> %Y
1443}
1444
1445define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) {
1446; SSE2-LABEL: load_sext_2i8_to_2i64:
1447; SSE2:       # %bb.0: # %entry
1448; SSE2-NEXT:    movzwl (%rdi), %eax
1449; SSE2-NEXT:    movd %eax, %xmm0
1450; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1451; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1452; SSE2-NEXT:    pxor %xmm1, %xmm1
1453; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1454; SSE2-NEXT:    psrad $24, %xmm0
1455; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1456; SSE2-NEXT:    retq
1457;
1458; SSSE3-LABEL: load_sext_2i8_to_2i64:
1459; SSSE3:       # %bb.0: # %entry
1460; SSSE3-NEXT:    movzwl (%rdi), %eax
1461; SSSE3-NEXT:    movd %eax, %xmm0
1462; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1463; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1464; SSSE3-NEXT:    pxor %xmm1, %xmm1
1465; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
1466; SSSE3-NEXT:    psrad $24, %xmm0
1467; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1468; SSSE3-NEXT:    retq
1469;
1470; SSE41-LABEL: load_sext_2i8_to_2i64:
1471; SSE41:       # %bb.0: # %entry
1472; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
1473; SSE41-NEXT:    retq
1474;
1475; AVX-LABEL: load_sext_2i8_to_2i64:
1476; AVX:       # %bb.0: # %entry
1477; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
1478; AVX-NEXT:    retq
1479;
1480; X32-SSE2-LABEL: load_sext_2i8_to_2i64:
1481; X32-SSE2:       # %bb.0: # %entry
1482; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1483; X32-SSE2-NEXT:    movzwl (%eax), %eax
1484; X32-SSE2-NEXT:    movd %eax, %xmm0
1485; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1486; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1487; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
1488; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
1489; X32-SSE2-NEXT:    psrad $24, %xmm0
1490; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1491; X32-SSE2-NEXT:    retl
1492;
1493; X32-SSE41-LABEL: load_sext_2i8_to_2i64:
1494; X32-SSE41:       # %bb.0: # %entry
1495; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1496; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
1497; X32-SSE41-NEXT:    retl
1498entry:
1499 %X = load <2 x i8>, <2 x i8>* %ptr
1500 %Y = sext <2 x i8> %X to <2 x i64>
1501 ret <2 x i64> %Y
1502}
1503
1504define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) {
1505; SSE2-LABEL: load_sext_4i1_to_4i32:
1506; SSE2:       # %bb.0: # %entry
1507; SSE2-NEXT:    movzbl (%rdi), %eax
1508; SSE2-NEXT:    movq %rax, %rcx
1509; SSE2-NEXT:    shlq $60, %rcx
1510; SSE2-NEXT:    sarq $63, %rcx
1511; SSE2-NEXT:    movd %ecx, %xmm0
1512; SSE2-NEXT:    movq %rax, %rcx
1513; SSE2-NEXT:    shlq $61, %rcx
1514; SSE2-NEXT:    sarq $63, %rcx
1515; SSE2-NEXT:    movd %ecx, %xmm1
1516; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1517; SSE2-NEXT:    movq %rax, %rcx
1518; SSE2-NEXT:    shlq $62, %rcx
1519; SSE2-NEXT:    sarq $63, %rcx
1520; SSE2-NEXT:    movd %ecx, %xmm2
1521; SSE2-NEXT:    shlq $63, %rax
1522; SSE2-NEXT:    sarq $63, %rax
1523; SSE2-NEXT:    movd %eax, %xmm0
1524; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1525; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1526; SSE2-NEXT:    retq
1527;
1528; SSSE3-LABEL: load_sext_4i1_to_4i32:
1529; SSSE3:       # %bb.0: # %entry
1530; SSSE3-NEXT:    movzbl (%rdi), %eax
1531; SSSE3-NEXT:    movq %rax, %rcx
1532; SSSE3-NEXT:    shlq $60, %rcx
1533; SSSE3-NEXT:    sarq $63, %rcx
1534; SSSE3-NEXT:    movd %ecx, %xmm0
1535; SSSE3-NEXT:    movq %rax, %rcx
1536; SSSE3-NEXT:    shlq $61, %rcx
1537; SSSE3-NEXT:    sarq $63, %rcx
1538; SSSE3-NEXT:    movd %ecx, %xmm1
1539; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1540; SSSE3-NEXT:    movq %rax, %rcx
1541; SSSE3-NEXT:    shlq $62, %rcx
1542; SSSE3-NEXT:    sarq $63, %rcx
1543; SSSE3-NEXT:    movd %ecx, %xmm2
1544; SSSE3-NEXT:    shlq $63, %rax
1545; SSSE3-NEXT:    sarq $63, %rax
1546; SSSE3-NEXT:    movd %eax, %xmm0
1547; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1548; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1549; SSSE3-NEXT:    retq
1550;
1551; SSE41-LABEL: load_sext_4i1_to_4i32:
1552; SSE41:       # %bb.0: # %entry
1553; SSE41-NEXT:    movzbl (%rdi), %eax
1554; SSE41-NEXT:    movq %rax, %rcx
1555; SSE41-NEXT:    shlq $62, %rcx
1556; SSE41-NEXT:    sarq $63, %rcx
1557; SSE41-NEXT:    movq %rax, %rdx
1558; SSE41-NEXT:    shlq $63, %rdx
1559; SSE41-NEXT:    sarq $63, %rdx
1560; SSE41-NEXT:    movd %edx, %xmm0
1561; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1562; SSE41-NEXT:    movq %rax, %rcx
1563; SSE41-NEXT:    shlq $61, %rcx
1564; SSE41-NEXT:    sarq $63, %rcx
1565; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1566; SSE41-NEXT:    shlq $60, %rax
1567; SSE41-NEXT:    sarq $63, %rax
1568; SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1569; SSE41-NEXT:    retq
1570;
1571; AVX1-LABEL: load_sext_4i1_to_4i32:
1572; AVX1:       # %bb.0: # %entry
1573; AVX1-NEXT:    movzbl (%rdi), %eax
1574; AVX1-NEXT:    movq %rax, %rcx
1575; AVX1-NEXT:    shlq $62, %rcx
1576; AVX1-NEXT:    sarq $63, %rcx
1577; AVX1-NEXT:    movq %rax, %rdx
1578; AVX1-NEXT:    shlq $63, %rdx
1579; AVX1-NEXT:    sarq $63, %rdx
1580; AVX1-NEXT:    vmovd %edx, %xmm0
1581; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1582; AVX1-NEXT:    movq %rax, %rcx
1583; AVX1-NEXT:    shlq $61, %rcx
1584; AVX1-NEXT:    sarq $63, %rcx
1585; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1586; AVX1-NEXT:    shlq $60, %rax
1587; AVX1-NEXT:    sarq $63, %rax
1588; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1589; AVX1-NEXT:    retq
1590;
1591; AVX2-LABEL: load_sext_4i1_to_4i32:
1592; AVX2:       # %bb.0: # %entry
1593; AVX2-NEXT:    movzbl (%rdi), %eax
1594; AVX2-NEXT:    movq %rax, %rcx
1595; AVX2-NEXT:    shlq $62, %rcx
1596; AVX2-NEXT:    sarq $63, %rcx
1597; AVX2-NEXT:    movq %rax, %rdx
1598; AVX2-NEXT:    shlq $63, %rdx
1599; AVX2-NEXT:    sarq $63, %rdx
1600; AVX2-NEXT:    vmovd %edx, %xmm0
1601; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1602; AVX2-NEXT:    movq %rax, %rcx
1603; AVX2-NEXT:    shlq $61, %rcx
1604; AVX2-NEXT:    sarq $63, %rcx
1605; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1606; AVX2-NEXT:    shlq $60, %rax
1607; AVX2-NEXT:    sarq $63, %rax
1608; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1609; AVX2-NEXT:    retq
1610;
1611; AVX512F-LABEL: load_sext_4i1_to_4i32:
1612; AVX512F:       # %bb.0: # %entry
1613; AVX512F-NEXT:    movzbl (%rdi), %eax
1614; AVX512F-NEXT:    kmovw %eax, %k1
1615; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1616; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1617; AVX512F-NEXT:    vzeroupper
1618; AVX512F-NEXT:    retq
1619;
1620; AVX512BW-LABEL: load_sext_4i1_to_4i32:
1621; AVX512BW:       # %bb.0: # %entry
1622; AVX512BW-NEXT:    movzbl (%rdi), %eax
1623; AVX512BW-NEXT:    kmovd %eax, %k1
1624; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1625; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
1626; AVX512BW-NEXT:    vzeroupper
1627; AVX512BW-NEXT:    retq
1628;
1629; X32-SSE2-LABEL: load_sext_4i1_to_4i32:
1630; X32-SSE2:       # %bb.0: # %entry
1631; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1632; X32-SSE2-NEXT:    movl (%eax), %eax
1633; X32-SSE2-NEXT:    movl %eax, %ecx
1634; X32-SSE2-NEXT:    shll $28, %ecx
1635; X32-SSE2-NEXT:    sarl $31, %ecx
1636; X32-SSE2-NEXT:    movd %ecx, %xmm0
1637; X32-SSE2-NEXT:    movl %eax, %ecx
1638; X32-SSE2-NEXT:    shll $29, %ecx
1639; X32-SSE2-NEXT:    sarl $31, %ecx
1640; X32-SSE2-NEXT:    movd %ecx, %xmm1
1641; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1642; X32-SSE2-NEXT:    movl %eax, %ecx
1643; X32-SSE2-NEXT:    shll $30, %ecx
1644; X32-SSE2-NEXT:    sarl $31, %ecx
1645; X32-SSE2-NEXT:    movd %ecx, %xmm2
1646; X32-SSE2-NEXT:    shll $31, %eax
1647; X32-SSE2-NEXT:    sarl $31, %eax
1648; X32-SSE2-NEXT:    movd %eax, %xmm0
1649; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1650; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1651; X32-SSE2-NEXT:    retl
1652;
1653; X32-SSE41-LABEL: load_sext_4i1_to_4i32:
1654; X32-SSE41:       # %bb.0: # %entry
1655; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1656; X32-SSE41-NEXT:    movl (%eax), %eax
1657; X32-SSE41-NEXT:    movl %eax, %ecx
1658; X32-SSE41-NEXT:    shll $30, %ecx
1659; X32-SSE41-NEXT:    sarl $31, %ecx
1660; X32-SSE41-NEXT:    movl %eax, %edx
1661; X32-SSE41-NEXT:    shll $31, %edx
1662; X32-SSE41-NEXT:    sarl $31, %edx
1663; X32-SSE41-NEXT:    movd %edx, %xmm0
1664; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
1665; X32-SSE41-NEXT:    movl %eax, %ecx
1666; X32-SSE41-NEXT:    shll $29, %ecx
1667; X32-SSE41-NEXT:    sarl $31, %ecx
1668; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
1669; X32-SSE41-NEXT:    shll $28, %eax
1670; X32-SSE41-NEXT:    sarl $31, %eax
1671; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
1672; X32-SSE41-NEXT:    retl
1673entry:
1674 %X = load <4 x i1>, <4 x i1>* %ptr
1675 %Y = sext <4 x i1> %X to <4 x i32>
1676 ret <4 x i32> %Y
1677}
1678
1679define <4 x i32> @load_sext_4i8_to_4i32(<4 x i8> *%ptr) {
1680; SSE2-LABEL: load_sext_4i8_to_4i32:
1681; SSE2:       # %bb.0: # %entry
1682; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1683; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1684; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1685; SSE2-NEXT:    psrad $24, %xmm0
1686; SSE2-NEXT:    retq
1687;
1688; SSSE3-LABEL: load_sext_4i8_to_4i32:
1689; SSSE3:       # %bb.0: # %entry
1690; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1691; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1692; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1693; SSSE3-NEXT:    psrad $24, %xmm0
1694; SSSE3-NEXT:    retq
1695;
1696; SSE41-LABEL: load_sext_4i8_to_4i32:
1697; SSE41:       # %bb.0: # %entry
1698; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
1699; SSE41-NEXT:    retq
1700;
1701; AVX-LABEL: load_sext_4i8_to_4i32:
1702; AVX:       # %bb.0: # %entry
1703; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
1704; AVX-NEXT:    retq
1705;
1706; X32-SSE2-LABEL: load_sext_4i8_to_4i32:
1707; X32-SSE2:       # %bb.0: # %entry
1708; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1709; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1710; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1711; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
1712; X32-SSE2-NEXT:    psrad $24, %xmm0
1713; X32-SSE2-NEXT:    retl
1714;
1715; X32-SSE41-LABEL: load_sext_4i8_to_4i32:
1716; X32-SSE41:       # %bb.0: # %entry
1717; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1718; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
1719; X32-SSE41-NEXT:    retl
1720entry:
1721 %X = load <4 x i8>, <4 x i8>* %ptr
1722 %Y = sext <4 x i8> %X to <4 x i32>
1723 ret <4 x i32> %Y
1724}
1725
1726define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
1727; SSE2-LABEL: load_sext_4i1_to_4i64:
1728; SSE2:       # %bb.0: # %entry
1729; SSE2-NEXT:    movl (%rdi), %eax
1730; SSE2-NEXT:    movl %eax, %ecx
1731; SSE2-NEXT:    shrl $3, %ecx
1732; SSE2-NEXT:    movd %ecx, %xmm0
1733; SSE2-NEXT:    movl %eax, %ecx
1734; SSE2-NEXT:    shrl $2, %ecx
1735; SSE2-NEXT:    movd %ecx, %xmm1
1736; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1737; SSE2-NEXT:    movd %eax, %xmm2
1738; SSE2-NEXT:    shrl %eax
1739; SSE2-NEXT:    movd %eax, %xmm0
1740; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1741; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1742; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
1743; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1744; SSE2-NEXT:    psllq $63, %xmm0
1745; SSE2-NEXT:    psrad $31, %xmm0
1746; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1747; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1748; SSE2-NEXT:    psllq $63, %xmm1
1749; SSE2-NEXT:    psrad $31, %xmm1
1750; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1751; SSE2-NEXT:    retq
1752;
1753; SSSE3-LABEL: load_sext_4i1_to_4i64:
1754; SSSE3:       # %bb.0: # %entry
1755; SSSE3-NEXT:    movl (%rdi), %eax
1756; SSSE3-NEXT:    movl %eax, %ecx
1757; SSSE3-NEXT:    shrl $3, %ecx
1758; SSSE3-NEXT:    movd %ecx, %xmm0
1759; SSSE3-NEXT:    movl %eax, %ecx
1760; SSSE3-NEXT:    shrl $2, %ecx
1761; SSSE3-NEXT:    movd %ecx, %xmm1
1762; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1763; SSSE3-NEXT:    movd %eax, %xmm2
1764; SSSE3-NEXT:    shrl %eax
1765; SSSE3-NEXT:    movd %eax, %xmm0
1766; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1767; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1768; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
1769; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1770; SSSE3-NEXT:    psllq $63, %xmm0
1771; SSSE3-NEXT:    psrad $31, %xmm0
1772; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1773; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1774; SSSE3-NEXT:    psllq $63, %xmm1
1775; SSSE3-NEXT:    psrad $31, %xmm1
1776; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1777; SSSE3-NEXT:    retq
1778;
1779; SSE41-LABEL: load_sext_4i1_to_4i64:
1780; SSE41:       # %bb.0: # %entry
1781; SSE41-NEXT:    movl (%rdi), %eax
1782; SSE41-NEXT:    movl %eax, %ecx
1783; SSE41-NEXT:    shrl %ecx
1784; SSE41-NEXT:    movd %eax, %xmm1
1785; SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1786; SSE41-NEXT:    movl %eax, %ecx
1787; SSE41-NEXT:    shrl $2, %ecx
1788; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1789; SSE41-NEXT:    shrl $3, %eax
1790; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1791; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
1792; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1793; SSE41-NEXT:    psllq $63, %xmm0
1794; SSE41-NEXT:    psrad $31, %xmm0
1795; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1796; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1797; SSE41-NEXT:    psllq $63, %xmm1
1798; SSE41-NEXT:    psrad $31, %xmm1
1799; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1800; SSE41-NEXT:    retq
1801;
1802; AVX1-LABEL: load_sext_4i1_to_4i64:
1803; AVX1:       # %bb.0: # %entry
1804; AVX1-NEXT:    movzbl (%rdi), %eax
1805; AVX1-NEXT:    movq %rax, %rcx
1806; AVX1-NEXT:    shlq $62, %rcx
1807; AVX1-NEXT:    sarq $63, %rcx
1808; AVX1-NEXT:    movq %rax, %rdx
1809; AVX1-NEXT:    shlq $63, %rdx
1810; AVX1-NEXT:    sarq $63, %rdx
1811; AVX1-NEXT:    vmovd %edx, %xmm0
1812; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
1813; AVX1-NEXT:    movq %rax, %rcx
1814; AVX1-NEXT:    shlq $61, %rcx
1815; AVX1-NEXT:    sarq $63, %rcx
1816; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
1817; AVX1-NEXT:    shlq $60, %rax
1818; AVX1-NEXT:    sarq $63, %rax
1819; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
1820; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
1821; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1822; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1823; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1824; AVX1-NEXT:    retq
1825;
1826; AVX2-LABEL: load_sext_4i1_to_4i64:
1827; AVX2:       # %bb.0: # %entry
1828; AVX2-NEXT:    movzbl (%rdi), %eax
1829; AVX2-NEXT:    movq %rax, %rcx
1830; AVX2-NEXT:    shlq $60, %rcx
1831; AVX2-NEXT:    sarq $63, %rcx
1832; AVX2-NEXT:    vmovq %rcx, %xmm0
1833; AVX2-NEXT:    movq %rax, %rcx
1834; AVX2-NEXT:    shlq $61, %rcx
1835; AVX2-NEXT:    sarq $63, %rcx
1836; AVX2-NEXT:    vmovq %rcx, %xmm1
1837; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1838; AVX2-NEXT:    movq %rax, %rcx
1839; AVX2-NEXT:    shlq $62, %rcx
1840; AVX2-NEXT:    sarq $63, %rcx
1841; AVX2-NEXT:    vmovq %rcx, %xmm1
1842; AVX2-NEXT:    shlq $63, %rax
1843; AVX2-NEXT:    sarq $63, %rax
1844; AVX2-NEXT:    vmovq %rax, %xmm2
1845; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1846; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
1847; AVX2-NEXT:    retq
1848;
1849; AVX512F-LABEL: load_sext_4i1_to_4i64:
1850; AVX512F:       # %bb.0: # %entry
1851; AVX512F-NEXT:    movzbl (%rdi), %eax
1852; AVX512F-NEXT:    kmovw %eax, %k1
1853; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1854; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1855; AVX512F-NEXT:    retq
1856;
1857; AVX512BW-LABEL: load_sext_4i1_to_4i64:
1858; AVX512BW:       # %bb.0: # %entry
1859; AVX512BW-NEXT:    movzbl (%rdi), %eax
1860; AVX512BW-NEXT:    kmovd %eax, %k1
1861; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
1862; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1863; AVX512BW-NEXT:    retq
1864;
1865; X32-SSE2-LABEL: load_sext_4i1_to_4i64:
1866; X32-SSE2:       # %bb.0: # %entry
1867; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1868; X32-SSE2-NEXT:    movzbl (%eax), %eax
1869; X32-SSE2-NEXT:    movl %eax, %ecx
1870; X32-SSE2-NEXT:    shrl $3, %ecx
1871; X32-SSE2-NEXT:    movd %ecx, %xmm0
1872; X32-SSE2-NEXT:    movl %eax, %ecx
1873; X32-SSE2-NEXT:    shrl $2, %ecx
1874; X32-SSE2-NEXT:    movd %ecx, %xmm1
1875; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1876; X32-SSE2-NEXT:    movd %eax, %xmm2
1877; X32-SSE2-NEXT:    shrl %eax
1878; X32-SSE2-NEXT:    movd %eax, %xmm0
1879; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1880; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
1881; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
1882; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
1883; X32-SSE2-NEXT:    psllq $63, %xmm0
1884; X32-SSE2-NEXT:    psrad $31, %xmm0
1885; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1886; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
1887; X32-SSE2-NEXT:    psllq $63, %xmm1
1888; X32-SSE2-NEXT:    psrad $31, %xmm1
1889; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1890; X32-SSE2-NEXT:    retl
1891;
1892; X32-SSE41-LABEL: load_sext_4i1_to_4i64:
1893; X32-SSE41:       # %bb.0: # %entry
1894; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1895; X32-SSE41-NEXT:    movzbl (%eax), %eax
1896; X32-SSE41-NEXT:    movl %eax, %ecx
1897; X32-SSE41-NEXT:    shrl %ecx
1898; X32-SSE41-NEXT:    movd %eax, %xmm1
1899; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
1900; X32-SSE41-NEXT:    movl %eax, %ecx
1901; X32-SSE41-NEXT:    shrl $2, %ecx
1902; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
1903; X32-SSE41-NEXT:    shrl $3, %eax
1904; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm1
1905; X32-SSE41-NEXT:    pand {{\.LCPI.*}}, %xmm1
1906; X32-SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1907; X32-SSE41-NEXT:    psllq $63, %xmm0
1908; X32-SSE41-NEXT:    psrad $31, %xmm0
1909; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
1910; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1911; X32-SSE41-NEXT:    psllq $63, %xmm1
1912; X32-SSE41-NEXT:    psrad $31, %xmm1
1913; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
1914; X32-SSE41-NEXT:    retl
1915entry:
1916 %X = load <4 x i1>, <4 x i1>* %ptr
1917 %Y = sext <4 x i1> %X to <4 x i64>
1918 ret <4 x i64> %Y
1919}
1920
1921define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
1922; SSE2-LABEL: load_sext_4i8_to_4i64:
1923; SSE2:       # %bb.0: # %entry
1924; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1925; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1926; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1927; SSE2-NEXT:    psrad $24, %xmm1
1928; SSE2-NEXT:    pxor %xmm2, %xmm2
1929; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1930; SSE2-NEXT:    movdqa %xmm1, %xmm0
1931; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1932; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1933; SSE2-NEXT:    retq
1934;
1935; SSSE3-LABEL: load_sext_4i8_to_4i64:
1936; SSSE3:       # %bb.0: # %entry
1937; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1938; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1939; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1940; SSSE3-NEXT:    psrad $24, %xmm1
1941; SSSE3-NEXT:    pxor %xmm2, %xmm2
1942; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
1943; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1944; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1945; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1946; SSSE3-NEXT:    retq
1947;
1948; SSE41-LABEL: load_sext_4i8_to_4i64:
1949; SSE41:       # %bb.0: # %entry
1950; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
1951; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
1952; SSE41-NEXT:    retq
1953;
1954; AVX1-LABEL: load_sext_4i8_to_4i64:
1955; AVX1:       # %bb.0: # %entry
1956; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
1957; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm1
1958; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1959; AVX1-NEXT:    retq
1960;
1961; AVX2-LABEL: load_sext_4i8_to_4i64:
1962; AVX2:       # %bb.0: # %entry
1963; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
1964; AVX2-NEXT:    retq
1965;
1966; AVX512-LABEL: load_sext_4i8_to_4i64:
1967; AVX512:       # %bb.0: # %entry
1968; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
1969; AVX512-NEXT:    retq
1970;
1971; X32-SSE2-LABEL: load_sext_4i8_to_4i64:
1972; X32-SSE2:       # %bb.0: # %entry
1973; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
1974; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1975; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1976; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1977; X32-SSE2-NEXT:    psrad $24, %xmm1
1978; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
1979; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
1980; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
1981; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1982; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1983; X32-SSE2-NEXT:    retl
1984;
1985; X32-SSE41-LABEL: load_sext_4i8_to_4i64:
1986; X32-SSE41:       # %bb.0: # %entry
1987; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
1988; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
1989; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
1990; X32-SSE41-NEXT:    retl
1991entry:
1992 %X = load <4 x i8>, <4 x i8>* %ptr
1993 %Y = sext <4 x i8> %X to <4 x i64>
1994 ret <4 x i64> %Y
1995}
1996
1997define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
1998; SSE2-LABEL: load_sext_4i8_to_4i64_extract:
1999; SSE2:       # %bb.0:
2000; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2001; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2002; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2003; SSE2-NEXT:    psrad $24, %xmm0
2004; SSE2-NEXT:    pxor %xmm1, %xmm1
2005; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2006; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2007; SSE2-NEXT:    retq
2008;
2009; SSSE3-LABEL: load_sext_4i8_to_4i64_extract:
2010; SSSE3:       # %bb.0:
2011; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2012; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2013; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2014; SSSE3-NEXT:    psrad $24, %xmm0
2015; SSSE3-NEXT:    pxor %xmm1, %xmm1
2016; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
2017; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2018; SSSE3-NEXT:    retq
2019;
2020; SSE41-LABEL: load_sext_4i8_to_4i64_extract:
2021; SSE41:       # %bb.0:
2022; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm0
2023; SSE41-NEXT:    retq
2024;
2025; AVX1-LABEL: load_sext_4i8_to_4i64_extract:
2026; AVX1:       # %bb.0:
2027; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
2028; AVX1-NEXT:    retq
2029;
2030; AVX2-LABEL: load_sext_4i8_to_4i64_extract:
2031; AVX2:       # %bb.0:
2032; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2033; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
2034; AVX2-NEXT:    vzeroupper
2035; AVX2-NEXT:    retq
2036;
2037; AVX512-LABEL: load_sext_4i8_to_4i64_extract:
2038; AVX512:       # %bb.0:
2039; AVX512-NEXT:    vpmovsxbq (%rdi), %ymm0
2040; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
2041; AVX512-NEXT:    vzeroupper
2042; AVX512-NEXT:    retq
2043;
2044; X32-SSE2-LABEL: load_sext_4i8_to_4i64_extract:
2045; X32-SSE2:       # %bb.0:
2046; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2047; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2048; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2049; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2050; X32-SSE2-NEXT:    psrad $24, %xmm0
2051; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
2052; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
2053; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2054; X32-SSE2-NEXT:    retl
2055;
2056; X32-SSE41-LABEL: load_sext_4i8_to_4i64_extract:
2057; X32-SSE41:       # %bb.0:
2058; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2059; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm0
2060; X32-SSE41-NEXT:    retl
2061 %ld = load <4 x i8>, <4 x i8>* %ptr
2062 %sext = sext <4 x i8> %ld to <4 x i64>
2063 %extract = shufflevector <4 x i64> %sext, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
2064 ret <2 x i64> %extract
2065}
2066
2067define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
2068; SSE2-LABEL: load_sext_8i1_to_8i16:
2069; SSE2:       # %bb.0: # %entry
2070; SSE2-NEXT:    movsbq (%rdi), %rax
2071; SSE2-NEXT:    movq %rax, %rcx
2072; SSE2-NEXT:    shrq $7, %rcx
2073; SSE2-NEXT:    movd %ecx, %xmm0
2074; SSE2-NEXT:    movq %rax, %rcx
2075; SSE2-NEXT:    shlq $57, %rcx
2076; SSE2-NEXT:    sarq $63, %rcx
2077; SSE2-NEXT:    movd %ecx, %xmm2
2078; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2079; SSE2-NEXT:    movq %rax, %rcx
2080; SSE2-NEXT:    shlq $58, %rcx
2081; SSE2-NEXT:    sarq $63, %rcx
2082; SSE2-NEXT:    movd %ecx, %xmm0
2083; SSE2-NEXT:    movq %rax, %rcx
2084; SSE2-NEXT:    shlq $59, %rcx
2085; SSE2-NEXT:    sarq $63, %rcx
2086; SSE2-NEXT:    movd %ecx, %xmm1
2087; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2088; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2089; SSE2-NEXT:    movq %rax, %rcx
2090; SSE2-NEXT:    shlq $60, %rcx
2091; SSE2-NEXT:    sarq $63, %rcx
2092; SSE2-NEXT:    movd %ecx, %xmm0
2093; SSE2-NEXT:    movq %rax, %rcx
2094; SSE2-NEXT:    shlq $61, %rcx
2095; SSE2-NEXT:    sarq $63, %rcx
2096; SSE2-NEXT:    movd %ecx, %xmm2
2097; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2098; SSE2-NEXT:    movq %rax, %rcx
2099; SSE2-NEXT:    shlq $62, %rcx
2100; SSE2-NEXT:    sarq $63, %rcx
2101; SSE2-NEXT:    movd %ecx, %xmm3
2102; SSE2-NEXT:    shlq $63, %rax
2103; SSE2-NEXT:    sarq $63, %rax
2104; SSE2-NEXT:    movd %eax, %xmm0
2105; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2106; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2107; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2108; SSE2-NEXT:    retq
2109;
2110; SSSE3-LABEL: load_sext_8i1_to_8i16:
2111; SSSE3:       # %bb.0: # %entry
2112; SSSE3-NEXT:    movsbq (%rdi), %rax
2113; SSSE3-NEXT:    movq %rax, %rcx
2114; SSSE3-NEXT:    shrq $7, %rcx
2115; SSSE3-NEXT:    movd %ecx, %xmm0
2116; SSSE3-NEXT:    movq %rax, %rcx
2117; SSSE3-NEXT:    shlq $57, %rcx
2118; SSSE3-NEXT:    sarq $63, %rcx
2119; SSSE3-NEXT:    movd %ecx, %xmm2
2120; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2121; SSSE3-NEXT:    movq %rax, %rcx
2122; SSSE3-NEXT:    shlq $58, %rcx
2123; SSSE3-NEXT:    sarq $63, %rcx
2124; SSSE3-NEXT:    movd %ecx, %xmm0
2125; SSSE3-NEXT:    movq %rax, %rcx
2126; SSSE3-NEXT:    shlq $59, %rcx
2127; SSSE3-NEXT:    sarq $63, %rcx
2128; SSSE3-NEXT:    movd %ecx, %xmm1
2129; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2130; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2131; SSSE3-NEXT:    movq %rax, %rcx
2132; SSSE3-NEXT:    shlq $60, %rcx
2133; SSSE3-NEXT:    sarq $63, %rcx
2134; SSSE3-NEXT:    movd %ecx, %xmm0
2135; SSSE3-NEXT:    movq %rax, %rcx
2136; SSSE3-NEXT:    shlq $61, %rcx
2137; SSSE3-NEXT:    sarq $63, %rcx
2138; SSSE3-NEXT:    movd %ecx, %xmm2
2139; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2140; SSSE3-NEXT:    movq %rax, %rcx
2141; SSSE3-NEXT:    shlq $62, %rcx
2142; SSSE3-NEXT:    sarq $63, %rcx
2143; SSSE3-NEXT:    movd %ecx, %xmm3
2144; SSSE3-NEXT:    shlq $63, %rax
2145; SSSE3-NEXT:    sarq $63, %rax
2146; SSSE3-NEXT:    movd %eax, %xmm0
2147; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2148; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2149; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2150; SSSE3-NEXT:    retq
2151;
2152; SSE41-LABEL: load_sext_8i1_to_8i16:
2153; SSE41:       # %bb.0: # %entry
2154; SSE41-NEXT:    movsbq (%rdi), %rax
2155; SSE41-NEXT:    movq %rax, %rcx
2156; SSE41-NEXT:    shlq $62, %rcx
2157; SSE41-NEXT:    sarq $63, %rcx
2158; SSE41-NEXT:    movq %rax, %rdx
2159; SSE41-NEXT:    shlq $63, %rdx
2160; SSE41-NEXT:    sarq $63, %rdx
2161; SSE41-NEXT:    movd %edx, %xmm0
2162; SSE41-NEXT:    pinsrw $1, %ecx, %xmm0
2163; SSE41-NEXT:    movq %rax, %rcx
2164; SSE41-NEXT:    shlq $61, %rcx
2165; SSE41-NEXT:    sarq $63, %rcx
2166; SSE41-NEXT:    pinsrw $2, %ecx, %xmm0
2167; SSE41-NEXT:    movq %rax, %rcx
2168; SSE41-NEXT:    shlq $60, %rcx
2169; SSE41-NEXT:    sarq $63, %rcx
2170; SSE41-NEXT:    pinsrw $3, %ecx, %xmm0
2171; SSE41-NEXT:    movq %rax, %rcx
2172; SSE41-NEXT:    shlq $59, %rcx
2173; SSE41-NEXT:    sarq $63, %rcx
2174; SSE41-NEXT:    pinsrw $4, %ecx, %xmm0
2175; SSE41-NEXT:    movq %rax, %rcx
2176; SSE41-NEXT:    shlq $58, %rcx
2177; SSE41-NEXT:    sarq $63, %rcx
2178; SSE41-NEXT:    pinsrw $5, %ecx, %xmm0
2179; SSE41-NEXT:    movq %rax, %rcx
2180; SSE41-NEXT:    shlq $57, %rcx
2181; SSE41-NEXT:    sarq $63, %rcx
2182; SSE41-NEXT:    pinsrw $6, %ecx, %xmm0
2183; SSE41-NEXT:    shrq $7, %rax
2184; SSE41-NEXT:    pinsrw $7, %eax, %xmm0
2185; SSE41-NEXT:    retq
2186;
2187; AVX1-LABEL: load_sext_8i1_to_8i16:
2188; AVX1:       # %bb.0: # %entry
2189; AVX1-NEXT:    movsbq (%rdi), %rax
2190; AVX1-NEXT:    movq %rax, %rcx
2191; AVX1-NEXT:    shlq $62, %rcx
2192; AVX1-NEXT:    sarq $63, %rcx
2193; AVX1-NEXT:    movq %rax, %rdx
2194; AVX1-NEXT:    shlq $63, %rdx
2195; AVX1-NEXT:    sarq $63, %rdx
2196; AVX1-NEXT:    vmovd %edx, %xmm0
2197; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0
2198; AVX1-NEXT:    movq %rax, %rcx
2199; AVX1-NEXT:    shlq $61, %rcx
2200; AVX1-NEXT:    sarq $63, %rcx
2201; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0
2202; AVX1-NEXT:    movq %rax, %rcx
2203; AVX1-NEXT:    shlq $60, %rcx
2204; AVX1-NEXT:    sarq $63, %rcx
2205; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
2206; AVX1-NEXT:    movq %rax, %rcx
2207; AVX1-NEXT:    shlq $59, %rcx
2208; AVX1-NEXT:    sarq $63, %rcx
2209; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
2210; AVX1-NEXT:    movq %rax, %rcx
2211; AVX1-NEXT:    shlq $58, %rcx
2212; AVX1-NEXT:    sarq $63, %rcx
2213; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
2214; AVX1-NEXT:    movq %rax, %rcx
2215; AVX1-NEXT:    shlq $57, %rcx
2216; AVX1-NEXT:    sarq $63, %rcx
2217; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
2218; AVX1-NEXT:    shrq $7, %rax
2219; AVX1-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
2220; AVX1-NEXT:    retq
2221;
2222; AVX2-LABEL: load_sext_8i1_to_8i16:
2223; AVX2:       # %bb.0: # %entry
2224; AVX2-NEXT:    movsbq (%rdi), %rax
2225; AVX2-NEXT:    movq %rax, %rcx
2226; AVX2-NEXT:    shlq $62, %rcx
2227; AVX2-NEXT:    sarq $63, %rcx
2228; AVX2-NEXT:    movq %rax, %rdx
2229; AVX2-NEXT:    shlq $63, %rdx
2230; AVX2-NEXT:    sarq $63, %rdx
2231; AVX2-NEXT:    vmovd %edx, %xmm0
2232; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0
2233; AVX2-NEXT:    movq %rax, %rcx
2234; AVX2-NEXT:    shlq $61, %rcx
2235; AVX2-NEXT:    sarq $63, %rcx
2236; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0
2237; AVX2-NEXT:    movq %rax, %rcx
2238; AVX2-NEXT:    shlq $60, %rcx
2239; AVX2-NEXT:    sarq $63, %rcx
2240; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
2241; AVX2-NEXT:    movq %rax, %rcx
2242; AVX2-NEXT:    shlq $59, %rcx
2243; AVX2-NEXT:    sarq $63, %rcx
2244; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
2245; AVX2-NEXT:    movq %rax, %rcx
2246; AVX2-NEXT:    shlq $58, %rcx
2247; AVX2-NEXT:    sarq $63, %rcx
2248; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
2249; AVX2-NEXT:    movq %rax, %rcx
2250; AVX2-NEXT:    shlq $57, %rcx
2251; AVX2-NEXT:    sarq $63, %rcx
2252; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
2253; AVX2-NEXT:    shrq $7, %rax
2254; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
2255; AVX2-NEXT:    retq
2256;
2257; AVX512F-LABEL: load_sext_8i1_to_8i16:
2258; AVX512F:       # %bb.0: # %entry
2259; AVX512F-NEXT:    movzbl (%rdi), %eax
2260; AVX512F-NEXT:    kmovw %eax, %k1
2261; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2262; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
2263; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2264; AVX512F-NEXT:    vzeroupper
2265; AVX512F-NEXT:    retq
2266;
2267; AVX512BW-LABEL: load_sext_8i1_to_8i16:
2268; AVX512BW:       # %bb.0: # %entry
2269; AVX512BW-NEXT:    movzbl (%rdi), %eax
2270; AVX512BW-NEXT:    kmovd %eax, %k0
2271; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
2272; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
2273; AVX512BW-NEXT:    vzeroupper
2274; AVX512BW-NEXT:    retq
2275;
2276; X32-SSE2-LABEL: load_sext_8i1_to_8i16:
2277; X32-SSE2:       # %bb.0: # %entry
2278; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2279; X32-SSE2-NEXT:    movsbl (%eax), %eax
2280; X32-SSE2-NEXT:    movl %eax, %ecx
2281; X32-SSE2-NEXT:    shrl $7, %ecx
2282; X32-SSE2-NEXT:    movd %ecx, %xmm0
2283; X32-SSE2-NEXT:    movl %eax, %ecx
2284; X32-SSE2-NEXT:    shll $25, %ecx
2285; X32-SSE2-NEXT:    sarl $31, %ecx
2286; X32-SSE2-NEXT:    movd %ecx, %xmm1
2287; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2288; X32-SSE2-NEXT:    movl %eax, %ecx
2289; X32-SSE2-NEXT:    shll $26, %ecx
2290; X32-SSE2-NEXT:    sarl $31, %ecx
2291; X32-SSE2-NEXT:    movd %ecx, %xmm0
2292; X32-SSE2-NEXT:    movl %eax, %ecx
2293; X32-SSE2-NEXT:    shll $27, %ecx
2294; X32-SSE2-NEXT:    sarl $31, %ecx
2295; X32-SSE2-NEXT:    movd %ecx, %xmm2
2296; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2297; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2298; X32-SSE2-NEXT:    movl %eax, %ecx
2299; X32-SSE2-NEXT:    shll $28, %ecx
2300; X32-SSE2-NEXT:    sarl $31, %ecx
2301; X32-SSE2-NEXT:    movd %ecx, %xmm0
2302; X32-SSE2-NEXT:    movl %eax, %ecx
2303; X32-SSE2-NEXT:    shll $29, %ecx
2304; X32-SSE2-NEXT:    sarl $31, %ecx
2305; X32-SSE2-NEXT:    movd %ecx, %xmm1
2306; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2307; X32-SSE2-NEXT:    movl %eax, %ecx
2308; X32-SSE2-NEXT:    shll $30, %ecx
2309; X32-SSE2-NEXT:    sarl $31, %ecx
2310; X32-SSE2-NEXT:    movd %ecx, %xmm3
2311; X32-SSE2-NEXT:    shll $31, %eax
2312; X32-SSE2-NEXT:    sarl $31, %eax
2313; X32-SSE2-NEXT:    movd %eax, %xmm0
2314; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2315; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2316; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
2317; X32-SSE2-NEXT:    retl
2318;
2319; X32-SSE41-LABEL: load_sext_8i1_to_8i16:
2320; X32-SSE41:       # %bb.0: # %entry
2321; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2322; X32-SSE41-NEXT:    movsbl (%eax), %eax
2323; X32-SSE41-NEXT:    movl %eax, %ecx
2324; X32-SSE41-NEXT:    shll $30, %ecx
2325; X32-SSE41-NEXT:    sarl $31, %ecx
2326; X32-SSE41-NEXT:    movl %eax, %edx
2327; X32-SSE41-NEXT:    shll $31, %edx
2328; X32-SSE41-NEXT:    sarl $31, %edx
2329; X32-SSE41-NEXT:    movd %edx, %xmm0
2330; X32-SSE41-NEXT:    pinsrw $1, %ecx, %xmm0
2331; X32-SSE41-NEXT:    movl %eax, %ecx
2332; X32-SSE41-NEXT:    shll $29, %ecx
2333; X32-SSE41-NEXT:    sarl $31, %ecx
2334; X32-SSE41-NEXT:    pinsrw $2, %ecx, %xmm0
2335; X32-SSE41-NEXT:    movl %eax, %ecx
2336; X32-SSE41-NEXT:    shll $28, %ecx
2337; X32-SSE41-NEXT:    sarl $31, %ecx
2338; X32-SSE41-NEXT:    pinsrw $3, %ecx, %xmm0
2339; X32-SSE41-NEXT:    movl %eax, %ecx
2340; X32-SSE41-NEXT:    shll $27, %ecx
2341; X32-SSE41-NEXT:    sarl $31, %ecx
2342; X32-SSE41-NEXT:    pinsrw $4, %ecx, %xmm0
2343; X32-SSE41-NEXT:    movl %eax, %ecx
2344; X32-SSE41-NEXT:    shll $26, %ecx
2345; X32-SSE41-NEXT:    sarl $31, %ecx
2346; X32-SSE41-NEXT:    pinsrw $5, %ecx, %xmm0
2347; X32-SSE41-NEXT:    movl %eax, %ecx
2348; X32-SSE41-NEXT:    shll $25, %ecx
2349; X32-SSE41-NEXT:    sarl $31, %ecx
2350; X32-SSE41-NEXT:    pinsrw $6, %ecx, %xmm0
2351; X32-SSE41-NEXT:    shrl $7, %eax
2352; X32-SSE41-NEXT:    pinsrw $7, %eax, %xmm0
2353; X32-SSE41-NEXT:    retl
2354entry:
2355 %X = load <8 x i1>, <8 x i1>* %ptr
2356 %Y = sext <8 x i1> %X to <8 x i16>
2357 ret <8 x i16> %Y
2358}
2359
2360define <8 x i16> @load_sext_8i8_to_8i16(<8 x i8> *%ptr) {
2361; SSE2-LABEL: load_sext_8i8_to_8i16:
2362; SSE2:       # %bb.0: # %entry
2363; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2364; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2365; SSE2-NEXT:    psraw $8, %xmm0
2366; SSE2-NEXT:    retq
2367;
2368; SSSE3-LABEL: load_sext_8i8_to_8i16:
2369; SSSE3:       # %bb.0: # %entry
2370; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2371; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2372; SSSE3-NEXT:    psraw $8, %xmm0
2373; SSSE3-NEXT:    retq
2374;
2375; SSE41-LABEL: load_sext_8i8_to_8i16:
2376; SSE41:       # %bb.0: # %entry
2377; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
2378; SSE41-NEXT:    retq
2379;
2380; AVX-LABEL: load_sext_8i8_to_8i16:
2381; AVX:       # %bb.0: # %entry
2382; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
2383; AVX-NEXT:    retq
2384;
2385; X32-SSE2-LABEL: load_sext_8i8_to_8i16:
2386; X32-SSE2:       # %bb.0: # %entry
2387; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2388; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
2389; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2390; X32-SSE2-NEXT:    psraw $8, %xmm0
2391; X32-SSE2-NEXT:    retl
2392;
2393; X32-SSE41-LABEL: load_sext_8i8_to_8i16:
2394; X32-SSE41:       # %bb.0: # %entry
2395; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2396; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
2397; X32-SSE41-NEXT:    retl
2398entry:
2399 %X = load <8 x i8>, <8 x i8>* %ptr
2400 %Y = sext <8 x i8> %X to <8 x i16>
2401 ret <8 x i16> %Y
2402}
2403
2404define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
2405; SSE2-LABEL: load_sext_8i8_to_8i64:
2406; SSE2:       # %bb.0: # %entry
2407; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
2408; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2409; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2410; SSE2-NEXT:    psrad $24, %xmm1
2411; SSE2-NEXT:    pxor %xmm4, %xmm4
2412; SSE2-NEXT:    pxor %xmm3, %xmm3
2413; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
2414; SSE2-NEXT:    movdqa %xmm1, %xmm0
2415; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2416; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2417; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2418; SSE2-NEXT:    psrad $24, %xmm3
2419; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
2420; SSE2-NEXT:    movdqa %xmm3, %xmm2
2421; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2422; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2423; SSE2-NEXT:    retq
2424;
2425; SSSE3-LABEL: load_sext_8i8_to_8i64:
2426; SSSE3:       # %bb.0: # %entry
2427; SSSE3-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
2428; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2429; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2430; SSSE3-NEXT:    psrad $24, %xmm1
2431; SSSE3-NEXT:    pxor %xmm4, %xmm4
2432; SSSE3-NEXT:    pxor %xmm3, %xmm3
2433; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
2434; SSSE3-NEXT:    movdqa %xmm1, %xmm0
2435; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2436; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2437; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2438; SSSE3-NEXT:    psrad $24, %xmm3
2439; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
2440; SSSE3-NEXT:    movdqa %xmm3, %xmm2
2441; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2442; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2443; SSSE3-NEXT:    retq
2444;
2445; SSE41-LABEL: load_sext_8i8_to_8i64:
2446; SSE41:       # %bb.0: # %entry
2447; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
2448; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
2449; SSE41-NEXT:    pmovsxbq 4(%rdi), %xmm2
2450; SSE41-NEXT:    pmovsxbq 6(%rdi), %xmm3
2451; SSE41-NEXT:    retq
2452;
2453; AVX1-LABEL: load_sext_8i8_to_8i64:
2454; AVX1:       # %bb.0: # %entry
2455; AVX1-NEXT:    vpmovsxbq 6(%rdi), %xmm1
2456; AVX1-NEXT:    vpmovsxbq 4(%rdi), %xmm2
2457; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
2458; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm3
2459; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
2460; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
2461; AVX1-NEXT:    retq
2462;
2463; AVX2-LABEL: load_sext_8i8_to_8i64:
2464; AVX2:       # %bb.0: # %entry
2465; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
2466; AVX2-NEXT:    vpmovsxbq 4(%rdi), %ymm1
2467; AVX2-NEXT:    retq
2468;
2469; AVX512-LABEL: load_sext_8i8_to_8i64:
2470; AVX512:       # %bb.0: # %entry
2471; AVX512-NEXT:    vpmovsxbq (%rdi), %zmm0
2472; AVX512-NEXT:    retq
2473;
2474; X32-SSE2-LABEL: load_sext_8i8_to_8i64:
2475; X32-SSE2:       # %bb.0: # %entry
2476; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2477; X32-SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
2478; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2479; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2480; X32-SSE2-NEXT:    psrad $24, %xmm1
2481; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
2482; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
2483; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
2484; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
2485; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
2486; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
2487; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2488; X32-SSE2-NEXT:    psrad $24, %xmm3
2489; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
2490; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
2491; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2492; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2493; X32-SSE2-NEXT:    retl
2494;
2495; X32-SSE41-LABEL: load_sext_8i8_to_8i64:
2496; X32-SSE41:       # %bb.0: # %entry
2497; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2498; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
2499; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
2500; X32-SSE41-NEXT:    pmovsxbq 4(%eax), %xmm2
2501; X32-SSE41-NEXT:    pmovsxbq 6(%eax), %xmm3
2502; X32-SSE41-NEXT:    retl
2503entry:
2504 %X = load <8 x i8>, <8 x i8>* %ptr
2505 %Y = sext <8 x i8> %X to <8 x i64>
2506 ret <8 x i64> %Y
2507}
2508
2509define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
2510; SSE2-LABEL: load_sext_8i1_to_8i32:
2511; SSE2:       # %bb.0: # %entry
2512; SSE2-NEXT:    movzbl (%rdi), %eax
2513; SSE2-NEXT:    movl %eax, %ecx
2514; SSE2-NEXT:    shrl $7, %ecx
2515; SSE2-NEXT:    movd %ecx, %xmm0
2516; SSE2-NEXT:    movl %eax, %ecx
2517; SSE2-NEXT:    shrl $6, %ecx
2518; SSE2-NEXT:    andl $1, %ecx
2519; SSE2-NEXT:    movd %ecx, %xmm2
2520; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2521; SSE2-NEXT:    movl %eax, %ecx
2522; SSE2-NEXT:    shrl $5, %ecx
2523; SSE2-NEXT:    andl $1, %ecx
2524; SSE2-NEXT:    movd %ecx, %xmm0
2525; SSE2-NEXT:    movl %eax, %ecx
2526; SSE2-NEXT:    shrl $4, %ecx
2527; SSE2-NEXT:    andl $1, %ecx
2528; SSE2-NEXT:    movd %ecx, %xmm1
2529; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2530; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2531; SSE2-NEXT:    movl %eax, %ecx
2532; SSE2-NEXT:    shrl $3, %ecx
2533; SSE2-NEXT:    andl $1, %ecx
2534; SSE2-NEXT:    movd %ecx, %xmm0
2535; SSE2-NEXT:    movl %eax, %ecx
2536; SSE2-NEXT:    shrl $2, %ecx
2537; SSE2-NEXT:    andl $1, %ecx
2538; SSE2-NEXT:    movd %ecx, %xmm2
2539; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2540; SSE2-NEXT:    movl %eax, %ecx
2541; SSE2-NEXT:    andl $1, %ecx
2542; SSE2-NEXT:    movd %ecx, %xmm0
2543; SSE2-NEXT:    shrl %eax
2544; SSE2-NEXT:    andl $1, %eax
2545; SSE2-NEXT:    movd %eax, %xmm3
2546; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2547; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2548; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2549; SSE2-NEXT:    pslld $31, %xmm0
2550; SSE2-NEXT:    psrad $31, %xmm0
2551; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2552; SSE2-NEXT:    pslld $31, %xmm1
2553; SSE2-NEXT:    psrad $31, %xmm1
2554; SSE2-NEXT:    retq
2555;
2556; SSSE3-LABEL: load_sext_8i1_to_8i32:
2557; SSSE3:       # %bb.0: # %entry
2558; SSSE3-NEXT:    movzbl (%rdi), %eax
2559; SSSE3-NEXT:    movl %eax, %ecx
2560; SSSE3-NEXT:    shrl $7, %ecx
2561; SSSE3-NEXT:    movd %ecx, %xmm0
2562; SSSE3-NEXT:    movl %eax, %ecx
2563; SSSE3-NEXT:    shrl $6, %ecx
2564; SSSE3-NEXT:    andl $1, %ecx
2565; SSSE3-NEXT:    movd %ecx, %xmm2
2566; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2567; SSSE3-NEXT:    movl %eax, %ecx
2568; SSSE3-NEXT:    shrl $5, %ecx
2569; SSSE3-NEXT:    andl $1, %ecx
2570; SSSE3-NEXT:    movd %ecx, %xmm0
2571; SSSE3-NEXT:    movl %eax, %ecx
2572; SSSE3-NEXT:    shrl $4, %ecx
2573; SSSE3-NEXT:    andl $1, %ecx
2574; SSSE3-NEXT:    movd %ecx, %xmm1
2575; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2576; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2577; SSSE3-NEXT:    movl %eax, %ecx
2578; SSSE3-NEXT:    shrl $3, %ecx
2579; SSSE3-NEXT:    andl $1, %ecx
2580; SSSE3-NEXT:    movd %ecx, %xmm0
2581; SSSE3-NEXT:    movl %eax, %ecx
2582; SSSE3-NEXT:    shrl $2, %ecx
2583; SSSE3-NEXT:    andl $1, %ecx
2584; SSSE3-NEXT:    movd %ecx, %xmm2
2585; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2586; SSSE3-NEXT:    movl %eax, %ecx
2587; SSSE3-NEXT:    andl $1, %ecx
2588; SSSE3-NEXT:    movd %ecx, %xmm0
2589; SSSE3-NEXT:    shrl %eax
2590; SSSE3-NEXT:    andl $1, %eax
2591; SSSE3-NEXT:    movd %eax, %xmm3
2592; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2593; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2594; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2595; SSSE3-NEXT:    pslld $31, %xmm0
2596; SSSE3-NEXT:    psrad $31, %xmm0
2597; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2598; SSSE3-NEXT:    pslld $31, %xmm1
2599; SSSE3-NEXT:    psrad $31, %xmm1
2600; SSSE3-NEXT:    retq
2601;
2602; SSE41-LABEL: load_sext_8i1_to_8i32:
2603; SSE41:       # %bb.0: # %entry
2604; SSE41-NEXT:    movzbl (%rdi), %eax
2605; SSE41-NEXT:    movl %eax, %ecx
2606; SSE41-NEXT:    shrl %ecx
2607; SSE41-NEXT:    andl $1, %ecx
2608; SSE41-NEXT:    movl %eax, %edx
2609; SSE41-NEXT:    andl $1, %edx
2610; SSE41-NEXT:    movd %edx, %xmm1
2611; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
2612; SSE41-NEXT:    movl %eax, %ecx
2613; SSE41-NEXT:    shrl $2, %ecx
2614; SSE41-NEXT:    andl $1, %ecx
2615; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
2616; SSE41-NEXT:    movl %eax, %ecx
2617; SSE41-NEXT:    shrl $3, %ecx
2618; SSE41-NEXT:    andl $1, %ecx
2619; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
2620; SSE41-NEXT:    movl %eax, %ecx
2621; SSE41-NEXT:    shrl $4, %ecx
2622; SSE41-NEXT:    andl $1, %ecx
2623; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
2624; SSE41-NEXT:    movl %eax, %ecx
2625; SSE41-NEXT:    shrl $5, %ecx
2626; SSE41-NEXT:    andl $1, %ecx
2627; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
2628; SSE41-NEXT:    movl %eax, %ecx
2629; SSE41-NEXT:    shrl $6, %ecx
2630; SSE41-NEXT:    andl $1, %ecx
2631; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
2632; SSE41-NEXT:    shrl $7, %eax
2633; SSE41-NEXT:    pinsrw $7, %eax, %xmm1
2634; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2635; SSE41-NEXT:    pslld $31, %xmm0
2636; SSE41-NEXT:    psrad $31, %xmm0
2637; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2638; SSE41-NEXT:    pslld $31, %xmm1
2639; SSE41-NEXT:    psrad $31, %xmm1
2640; SSE41-NEXT:    retq
2641;
2642; AVX1-LABEL: load_sext_8i1_to_8i32:
2643; AVX1:       # %bb.0: # %entry
2644; AVX1-NEXT:    movsbq (%rdi), %rax
2645; AVX1-NEXT:    movq %rax, %rcx
2646; AVX1-NEXT:    shlq $58, %rcx
2647; AVX1-NEXT:    sarq $63, %rcx
2648; AVX1-NEXT:    movq %rax, %rdx
2649; AVX1-NEXT:    shlq $59, %rdx
2650; AVX1-NEXT:    sarq $63, %rdx
2651; AVX1-NEXT:    vmovd %edx, %xmm0
2652; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
2653; AVX1-NEXT:    movq %rax, %rcx
2654; AVX1-NEXT:    shlq $57, %rcx
2655; AVX1-NEXT:    sarq $63, %rcx
2656; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
2657; AVX1-NEXT:    movq %rax, %rcx
2658; AVX1-NEXT:    shrq $7, %rcx
2659; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
2660; AVX1-NEXT:    movq %rax, %rcx
2661; AVX1-NEXT:    shlq $62, %rcx
2662; AVX1-NEXT:    sarq $63, %rcx
2663; AVX1-NEXT:    movq %rax, %rdx
2664; AVX1-NEXT:    shlq $63, %rdx
2665; AVX1-NEXT:    sarq $63, %rdx
2666; AVX1-NEXT:    vmovd %edx, %xmm1
2667; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
2668; AVX1-NEXT:    movq %rax, %rcx
2669; AVX1-NEXT:    shlq $61, %rcx
2670; AVX1-NEXT:    sarq $63, %rcx
2671; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
2672; AVX1-NEXT:    shlq $60, %rax
2673; AVX1-NEXT:    sarq $63, %rax
2674; AVX1-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
2675; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2676; AVX1-NEXT:    retq
2677;
2678; AVX2-LABEL: load_sext_8i1_to_8i32:
2679; AVX2:       # %bb.0: # %entry
2680; AVX2-NEXT:    movsbq (%rdi), %rax
2681; AVX2-NEXT:    movq %rax, %rcx
2682; AVX2-NEXT:    shlq $58, %rcx
2683; AVX2-NEXT:    sarq $63, %rcx
2684; AVX2-NEXT:    movq %rax, %rdx
2685; AVX2-NEXT:    shlq $59, %rdx
2686; AVX2-NEXT:    sarq $63, %rdx
2687; AVX2-NEXT:    vmovd %edx, %xmm0
2688; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
2689; AVX2-NEXT:    movq %rax, %rcx
2690; AVX2-NEXT:    shlq $57, %rcx
2691; AVX2-NEXT:    sarq $63, %rcx
2692; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
2693; AVX2-NEXT:    movq %rax, %rcx
2694; AVX2-NEXT:    shrq $7, %rcx
2695; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
2696; AVX2-NEXT:    movq %rax, %rcx
2697; AVX2-NEXT:    shlq $62, %rcx
2698; AVX2-NEXT:    sarq $63, %rcx
2699; AVX2-NEXT:    movq %rax, %rdx
2700; AVX2-NEXT:    shlq $63, %rdx
2701; AVX2-NEXT:    sarq $63, %rdx
2702; AVX2-NEXT:    vmovd %edx, %xmm1
2703; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
2704; AVX2-NEXT:    movq %rax, %rcx
2705; AVX2-NEXT:    shlq $61, %rcx
2706; AVX2-NEXT:    sarq $63, %rcx
2707; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
2708; AVX2-NEXT:    shlq $60, %rax
2709; AVX2-NEXT:    sarq $63, %rax
2710; AVX2-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
2711; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
2712; AVX2-NEXT:    retq
2713;
2714; AVX512F-LABEL: load_sext_8i1_to_8i32:
2715; AVX512F:       # %bb.0: # %entry
2716; AVX512F-NEXT:    movzbl (%rdi), %eax
2717; AVX512F-NEXT:    kmovw %eax, %k1
2718; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2719; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2720; AVX512F-NEXT:    retq
2721;
2722; AVX512BW-LABEL: load_sext_8i1_to_8i32:
2723; AVX512BW:       # %bb.0: # %entry
2724; AVX512BW-NEXT:    movzbl (%rdi), %eax
2725; AVX512BW-NEXT:    kmovd %eax, %k1
2726; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
2727; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
2728; AVX512BW-NEXT:    retq
2729;
2730; X32-SSE2-LABEL: load_sext_8i1_to_8i32:
2731; X32-SSE2:       # %bb.0: # %entry
2732; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2733; X32-SSE2-NEXT:    movzbl (%eax), %eax
2734; X32-SSE2-NEXT:    movl %eax, %ecx
2735; X32-SSE2-NEXT:    shrl $7, %ecx
2736; X32-SSE2-NEXT:    movd %ecx, %xmm0
2737; X32-SSE2-NEXT:    movl %eax, %ecx
2738; X32-SSE2-NEXT:    shrl $6, %ecx
2739; X32-SSE2-NEXT:    andl $1, %ecx
2740; X32-SSE2-NEXT:    movd %ecx, %xmm2
2741; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2742; X32-SSE2-NEXT:    movl %eax, %ecx
2743; X32-SSE2-NEXT:    shrl $5, %ecx
2744; X32-SSE2-NEXT:    andl $1, %ecx
2745; X32-SSE2-NEXT:    movd %ecx, %xmm0
2746; X32-SSE2-NEXT:    movl %eax, %ecx
2747; X32-SSE2-NEXT:    shrl $4, %ecx
2748; X32-SSE2-NEXT:    andl $1, %ecx
2749; X32-SSE2-NEXT:    movd %ecx, %xmm1
2750; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2751; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2752; X32-SSE2-NEXT:    movl %eax, %ecx
2753; X32-SSE2-NEXT:    shrl $3, %ecx
2754; X32-SSE2-NEXT:    andl $1, %ecx
2755; X32-SSE2-NEXT:    movd %ecx, %xmm0
2756; X32-SSE2-NEXT:    movl %eax, %ecx
2757; X32-SSE2-NEXT:    shrl $2, %ecx
2758; X32-SSE2-NEXT:    andl $1, %ecx
2759; X32-SSE2-NEXT:    movd %ecx, %xmm2
2760; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2761; X32-SSE2-NEXT:    movl %eax, %ecx
2762; X32-SSE2-NEXT:    andl $1, %ecx
2763; X32-SSE2-NEXT:    movd %ecx, %xmm0
2764; X32-SSE2-NEXT:    shrl %eax
2765; X32-SSE2-NEXT:    andl $1, %eax
2766; X32-SSE2-NEXT:    movd %eax, %xmm3
2767; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2768; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2769; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
2770; X32-SSE2-NEXT:    pslld $31, %xmm0
2771; X32-SSE2-NEXT:    psrad $31, %xmm0
2772; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2773; X32-SSE2-NEXT:    pslld $31, %xmm1
2774; X32-SSE2-NEXT:    psrad $31, %xmm1
2775; X32-SSE2-NEXT:    retl
2776;
2777; X32-SSE41-LABEL: load_sext_8i1_to_8i32:
2778; X32-SSE41:       # %bb.0: # %entry
2779; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2780; X32-SSE41-NEXT:    movzbl (%eax), %eax
2781; X32-SSE41-NEXT:    movl %eax, %ecx
2782; X32-SSE41-NEXT:    shrl %ecx
2783; X32-SSE41-NEXT:    andl $1, %ecx
2784; X32-SSE41-NEXT:    movl %eax, %edx
2785; X32-SSE41-NEXT:    andl $1, %edx
2786; X32-SSE41-NEXT:    movd %edx, %xmm1
2787; X32-SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
2788; X32-SSE41-NEXT:    movl %eax, %ecx
2789; X32-SSE41-NEXT:    shrl $2, %ecx
2790; X32-SSE41-NEXT:    andl $1, %ecx
2791; X32-SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
2792; X32-SSE41-NEXT:    movl %eax, %ecx
2793; X32-SSE41-NEXT:    shrl $3, %ecx
2794; X32-SSE41-NEXT:    andl $1, %ecx
2795; X32-SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
2796; X32-SSE41-NEXT:    movl %eax, %ecx
2797; X32-SSE41-NEXT:    shrl $4, %ecx
2798; X32-SSE41-NEXT:    andl $1, %ecx
2799; X32-SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
2800; X32-SSE41-NEXT:    movl %eax, %ecx
2801; X32-SSE41-NEXT:    shrl $5, %ecx
2802; X32-SSE41-NEXT:    andl $1, %ecx
2803; X32-SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
2804; X32-SSE41-NEXT:    movl %eax, %ecx
2805; X32-SSE41-NEXT:    shrl $6, %ecx
2806; X32-SSE41-NEXT:    andl $1, %ecx
2807; X32-SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
2808; X32-SSE41-NEXT:    shrl $7, %eax
2809; X32-SSE41-NEXT:    pinsrw $7, %eax, %xmm1
2810; X32-SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2811; X32-SSE41-NEXT:    pslld $31, %xmm0
2812; X32-SSE41-NEXT:    psrad $31, %xmm0
2813; X32-SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2814; X32-SSE41-NEXT:    pslld $31, %xmm1
2815; X32-SSE41-NEXT:    psrad $31, %xmm1
2816; X32-SSE41-NEXT:    retl
2817entry:
2818 %X = load <8 x i1>, <8 x i1>* %ptr
2819 %Y = sext <8 x i1> %X to <8 x i32>
2820 ret <8 x i32> %Y
2821}
2822
2823define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
2824; SSE2-LABEL: load_sext_8i8_to_8i32:
2825; SSE2:       # %bb.0: # %entry
2826; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
2827; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2828; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2829; SSE2-NEXT:    psrad $24, %xmm0
2830; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2831; SSE2-NEXT:    psrad $24, %xmm1
2832; SSE2-NEXT:    retq
2833;
2834; SSSE3-LABEL: load_sext_8i8_to_8i32:
2835; SSSE3:       # %bb.0: # %entry
2836; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
2837; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2838; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2839; SSSE3-NEXT:    psrad $24, %xmm0
2840; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2841; SSSE3-NEXT:    psrad $24, %xmm1
2842; SSSE3-NEXT:    retq
2843;
2844; SSE41-LABEL: load_sext_8i8_to_8i32:
2845; SSE41:       # %bb.0: # %entry
2846; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
2847; SSE41-NEXT:    pmovsxbd 4(%rdi), %xmm1
2848; SSE41-NEXT:    retq
2849;
2850; AVX1-LABEL: load_sext_8i8_to_8i32:
2851; AVX1:       # %bb.0: # %entry
2852; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm0
2853; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm1
2854; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
2855; AVX1-NEXT:    retq
2856;
2857; AVX2-LABEL: load_sext_8i8_to_8i32:
2858; AVX2:       # %bb.0: # %entry
2859; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm0
2860; AVX2-NEXT:    retq
2861;
2862; AVX512-LABEL: load_sext_8i8_to_8i32:
2863; AVX512:       # %bb.0: # %entry
2864; AVX512-NEXT:    vpmovsxbd (%rdi), %ymm0
2865; AVX512-NEXT:    retq
2866;
2867; X32-SSE2-LABEL: load_sext_8i8_to_8i32:
2868; X32-SSE2:       # %bb.0: # %entry
2869; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
2870; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
2871; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2872; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2873; X32-SSE2-NEXT:    psrad $24, %xmm0
2874; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
2875; X32-SSE2-NEXT:    psrad $24, %xmm1
2876; X32-SSE2-NEXT:    retl
2877;
2878; X32-SSE41-LABEL: load_sext_8i8_to_8i32:
2879; X32-SSE41:       # %bb.0: # %entry
2880; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
2881; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
2882; X32-SSE41-NEXT:    pmovsxbd 4(%eax), %xmm1
2883; X32-SSE41-NEXT:    retl
2884entry:
2885 %X = load <8 x i8>, <8 x i8>* %ptr
2886 %Y = sext <8 x i8> %X to <8 x i32>
2887 ret <8 x i32> %Y
2888}
2889
2890define <16 x i8> @load_sext_16i1_to_16i8(<16 x i1> *%ptr) nounwind readnone {
2891; SSE2-LABEL: load_sext_16i1_to_16i8:
2892; SSE2:       # %bb.0: # %entry
2893; SSE2-NEXT:    pushq %rbp
2894; SSE2-NEXT:    pushq %r15
2895; SSE2-NEXT:    pushq %r14
2896; SSE2-NEXT:    pushq %r13
2897; SSE2-NEXT:    pushq %r12
2898; SSE2-NEXT:    pushq %rbx
2899; SSE2-NEXT:    movswq (%rdi), %rax
2900; SSE2-NEXT:    movq %rax, %r8
2901; SSE2-NEXT:    movq %rax, %r9
2902; SSE2-NEXT:    movq %rax, %r10
2903; SSE2-NEXT:    movq %rax, %r11
2904; SSE2-NEXT:    movq %rax, %r14
2905; SSE2-NEXT:    movq %rax, %r15
2906; SSE2-NEXT:    movq %rax, %r12
2907; SSE2-NEXT:    movq %rax, %r13
2908; SSE2-NEXT:    movq %rax, %rbx
2909; SSE2-NEXT:    movq %rax, %rcx
2910; SSE2-NEXT:    movq %rax, %rdx
2911; SSE2-NEXT:    movq %rax, %rsi
2912; SSE2-NEXT:    movq %rax, %rdi
2913; SSE2-NEXT:    movq %rax, %rbp
2914; SSE2-NEXT:    shrq $15, %rbp
2915; SSE2-NEXT:    movd %ebp, %xmm0
2916; SSE2-NEXT:    movq %rax, %rbp
2917; SSE2-NEXT:    movsbq %al, %rax
2918; SSE2-NEXT:    shlq $49, %r8
2919; SSE2-NEXT:    sarq $63, %r8
2920; SSE2-NEXT:    movd %r8d, %xmm1
2921; SSE2-NEXT:    shlq $50, %r9
2922; SSE2-NEXT:    sarq $63, %r9
2923; SSE2-NEXT:    movd %r9d, %xmm2
2924; SSE2-NEXT:    shlq $51, %r10
2925; SSE2-NEXT:    sarq $63, %r10
2926; SSE2-NEXT:    movd %r10d, %xmm3
2927; SSE2-NEXT:    shlq $52, %r11
2928; SSE2-NEXT:    sarq $63, %r11
2929; SSE2-NEXT:    movd %r11d, %xmm4
2930; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2931; SSE2-NEXT:    shlq $53, %r14
2932; SSE2-NEXT:    sarq $63, %r14
2933; SSE2-NEXT:    movd %r14d, %xmm0
2934; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
2935; SSE2-NEXT:    shlq $54, %r15
2936; SSE2-NEXT:    sarq $63, %r15
2937; SSE2-NEXT:    movd %r15d, %xmm2
2938; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
2939; SSE2-NEXT:    shlq $55, %r12
2940; SSE2-NEXT:    sarq $63, %r12
2941; SSE2-NEXT:    movd %r12d, %xmm1
2942; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
2943; SSE2-NEXT:    shlq $60, %r13
2944; SSE2-NEXT:    sarq $63, %r13
2945; SSE2-NEXT:    movd %r13d, %xmm4
2946; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2947; SSE2-NEXT:    shlq $61, %rbx
2948; SSE2-NEXT:    sarq $63, %rbx
2949; SSE2-NEXT:    movd %ebx, %xmm2
2950; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2951; SSE2-NEXT:    shlq $62, %rcx
2952; SSE2-NEXT:    sarq $63, %rcx
2953; SSE2-NEXT:    movd %ecx, %xmm5
2954; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
2955; SSE2-NEXT:    shlq $63, %rdx
2956; SSE2-NEXT:    sarq $63, %rdx
2957; SSE2-NEXT:    movd %edx, %xmm0
2958; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
2959; SSE2-NEXT:    shlq $58, %rsi
2960; SSE2-NEXT:    sarq $63, %rsi
2961; SSE2-NEXT:    movd %esi, %xmm3
2962; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
2963; SSE2-NEXT:    shlq $59, %rdi
2964; SSE2-NEXT:    sarq $63, %rdi
2965; SSE2-NEXT:    movd %edi, %xmm4
2966; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2967; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
2968; SSE2-NEXT:    shlq $57, %rbp
2969; SSE2-NEXT:    sarq $63, %rbp
2970; SSE2-NEXT:    movd %ebp, %xmm2
2971; SSE2-NEXT:    shrq $7, %rax
2972; SSE2-NEXT:    movd %eax, %xmm3
2973; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
2974; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
2975; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
2976; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2977; SSE2-NEXT:    popq %rbx
2978; SSE2-NEXT:    popq %r12
2979; SSE2-NEXT:    popq %r13
2980; SSE2-NEXT:    popq %r14
2981; SSE2-NEXT:    popq %r15
2982; SSE2-NEXT:    popq %rbp
2983; SSE2-NEXT:    retq
2984;
2985; SSSE3-LABEL: load_sext_16i1_to_16i8:
2986; SSSE3:       # %bb.0: # %entry
2987; SSSE3-NEXT:    pushq %rbp
2988; SSSE3-NEXT:    pushq %r15
2989; SSSE3-NEXT:    pushq %r14
2990; SSSE3-NEXT:    pushq %r13
2991; SSSE3-NEXT:    pushq %r12
2992; SSSE3-NEXT:    pushq %rbx
2993; SSSE3-NEXT:    movswq (%rdi), %rax
2994; SSSE3-NEXT:    movq %rax, %r8
2995; SSSE3-NEXT:    movq %rax, %r9
2996; SSSE3-NEXT:    movq %rax, %r10
2997; SSSE3-NEXT:    movq %rax, %r11
2998; SSSE3-NEXT:    movq %rax, %r14
2999; SSSE3-NEXT:    movq %rax, %r15
3000; SSSE3-NEXT:    movq %rax, %r12
3001; SSSE3-NEXT:    movq %rax, %r13
3002; SSSE3-NEXT:    movq %rax, %rbx
3003; SSSE3-NEXT:    movq %rax, %rcx
3004; SSSE3-NEXT:    movq %rax, %rdx
3005; SSSE3-NEXT:    movq %rax, %rsi
3006; SSSE3-NEXT:    movq %rax, %rdi
3007; SSSE3-NEXT:    movq %rax, %rbp
3008; SSSE3-NEXT:    shrq $15, %rbp
3009; SSSE3-NEXT:    movd %ebp, %xmm0
3010; SSSE3-NEXT:    movq %rax, %rbp
3011; SSSE3-NEXT:    movsbq %al, %rax
3012; SSSE3-NEXT:    shlq $49, %r8
3013; SSSE3-NEXT:    sarq $63, %r8
3014; SSSE3-NEXT:    movd %r8d, %xmm1
3015; SSSE3-NEXT:    shlq $50, %r9
3016; SSSE3-NEXT:    sarq $63, %r9
3017; SSSE3-NEXT:    movd %r9d, %xmm2
3018; SSSE3-NEXT:    shlq $51, %r10
3019; SSSE3-NEXT:    sarq $63, %r10
3020; SSSE3-NEXT:    movd %r10d, %xmm3
3021; SSSE3-NEXT:    shlq $52, %r11
3022; SSSE3-NEXT:    sarq $63, %r11
3023; SSSE3-NEXT:    movd %r11d, %xmm4
3024; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3025; SSSE3-NEXT:    shlq $53, %r14
3026; SSSE3-NEXT:    sarq $63, %r14
3027; SSSE3-NEXT:    movd %r14d, %xmm0
3028; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
3029; SSSE3-NEXT:    shlq $54, %r15
3030; SSSE3-NEXT:    sarq $63, %r15
3031; SSSE3-NEXT:    movd %r15d, %xmm2
3032; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3033; SSSE3-NEXT:    shlq $55, %r12
3034; SSSE3-NEXT:    sarq $63, %r12
3035; SSSE3-NEXT:    movd %r12d, %xmm1
3036; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3037; SSSE3-NEXT:    shlq $60, %r13
3038; SSSE3-NEXT:    sarq $63, %r13
3039; SSSE3-NEXT:    movd %r13d, %xmm4
3040; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3041; SSSE3-NEXT:    shlq $61, %rbx
3042; SSSE3-NEXT:    sarq $63, %rbx
3043; SSSE3-NEXT:    movd %ebx, %xmm2
3044; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3045; SSSE3-NEXT:    shlq $62, %rcx
3046; SSSE3-NEXT:    sarq $63, %rcx
3047; SSSE3-NEXT:    movd %ecx, %xmm5
3048; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
3049; SSSE3-NEXT:    shlq $63, %rdx
3050; SSSE3-NEXT:    sarq $63, %rdx
3051; SSSE3-NEXT:    movd %edx, %xmm0
3052; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
3053; SSSE3-NEXT:    shlq $58, %rsi
3054; SSSE3-NEXT:    sarq $63, %rsi
3055; SSSE3-NEXT:    movd %esi, %xmm3
3056; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
3057; SSSE3-NEXT:    shlq $59, %rdi
3058; SSSE3-NEXT:    sarq $63, %rdi
3059; SSSE3-NEXT:    movd %edi, %xmm4
3060; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3061; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
3062; SSSE3-NEXT:    shlq $57, %rbp
3063; SSSE3-NEXT:    sarq $63, %rbp
3064; SSSE3-NEXT:    movd %ebp, %xmm2
3065; SSSE3-NEXT:    shrq $7, %rax
3066; SSSE3-NEXT:    movd %eax, %xmm3
3067; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
3068; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
3069; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
3070; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
3071; SSSE3-NEXT:    popq %rbx
3072; SSSE3-NEXT:    popq %r12
3073; SSSE3-NEXT:    popq %r13
3074; SSSE3-NEXT:    popq %r14
3075; SSSE3-NEXT:    popq %r15
3076; SSSE3-NEXT:    popq %rbp
3077; SSSE3-NEXT:    retq
3078;
3079; SSE41-LABEL: load_sext_16i1_to_16i8:
3080; SSE41:       # %bb.0: # %entry
3081; SSE41-NEXT:    movswq (%rdi), %rax
3082; SSE41-NEXT:    movq %rax, %rcx
3083; SSE41-NEXT:    shlq $62, %rcx
3084; SSE41-NEXT:    sarq $63, %rcx
3085; SSE41-NEXT:    movq %rax, %rdx
3086; SSE41-NEXT:    shlq $63, %rdx
3087; SSE41-NEXT:    sarq $63, %rdx
3088; SSE41-NEXT:    movd %edx, %xmm0
3089; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
3090; SSE41-NEXT:    movq %rax, %rcx
3091; SSE41-NEXT:    shlq $61, %rcx
3092; SSE41-NEXT:    sarq $63, %rcx
3093; SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
3094; SSE41-NEXT:    movq %rax, %rcx
3095; SSE41-NEXT:    shlq $60, %rcx
3096; SSE41-NEXT:    sarq $63, %rcx
3097; SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
3098; SSE41-NEXT:    movq %rax, %rcx
3099; SSE41-NEXT:    shlq $59, %rcx
3100; SSE41-NEXT:    sarq $63, %rcx
3101; SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
3102; SSE41-NEXT:    movq %rax, %rcx
3103; SSE41-NEXT:    shlq $58, %rcx
3104; SSE41-NEXT:    sarq $63, %rcx
3105; SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
3106; SSE41-NEXT:    movq %rax, %rcx
3107; SSE41-NEXT:    shlq $57, %rcx
3108; SSE41-NEXT:    sarq $63, %rcx
3109; SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
3110; SSE41-NEXT:    movsbq %al, %rcx
3111; SSE41-NEXT:    shrq $7, %rcx
3112; SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
3113; SSE41-NEXT:    movq %rax, %rcx
3114; SSE41-NEXT:    shlq $55, %rcx
3115; SSE41-NEXT:    sarq $63, %rcx
3116; SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
3117; SSE41-NEXT:    movq %rax, %rcx
3118; SSE41-NEXT:    shlq $54, %rcx
3119; SSE41-NEXT:    sarq $63, %rcx
3120; SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
3121; SSE41-NEXT:    movq %rax, %rcx
3122; SSE41-NEXT:    shlq $53, %rcx
3123; SSE41-NEXT:    sarq $63, %rcx
3124; SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
3125; SSE41-NEXT:    movq %rax, %rcx
3126; SSE41-NEXT:    shlq $52, %rcx
3127; SSE41-NEXT:    sarq $63, %rcx
3128; SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
3129; SSE41-NEXT:    movq %rax, %rcx
3130; SSE41-NEXT:    shlq $51, %rcx
3131; SSE41-NEXT:    sarq $63, %rcx
3132; SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
3133; SSE41-NEXT:    movq %rax, %rcx
3134; SSE41-NEXT:    shlq $50, %rcx
3135; SSE41-NEXT:    sarq $63, %rcx
3136; SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
3137; SSE41-NEXT:    movq %rax, %rcx
3138; SSE41-NEXT:    shlq $49, %rcx
3139; SSE41-NEXT:    sarq $63, %rcx
3140; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
3141; SSE41-NEXT:    shrq $15, %rax
3142; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
3143; SSE41-NEXT:    retq
3144;
3145; AVX1-LABEL: load_sext_16i1_to_16i8:
3146; AVX1:       # %bb.0: # %entry
3147; AVX1-NEXT:    movswq (%rdi), %rax
3148; AVX1-NEXT:    movq %rax, %rcx
3149; AVX1-NEXT:    shlq $62, %rcx
3150; AVX1-NEXT:    sarq $63, %rcx
3151; AVX1-NEXT:    movq %rax, %rdx
3152; AVX1-NEXT:    shlq $63, %rdx
3153; AVX1-NEXT:    sarq $63, %rdx
3154; AVX1-NEXT:    vmovd %edx, %xmm0
3155; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
3156; AVX1-NEXT:    movq %rax, %rcx
3157; AVX1-NEXT:    shlq $61, %rcx
3158; AVX1-NEXT:    sarq $63, %rcx
3159; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0
3160; AVX1-NEXT:    movq %rax, %rcx
3161; AVX1-NEXT:    shlq $60, %rcx
3162; AVX1-NEXT:    sarq $63, %rcx
3163; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
3164; AVX1-NEXT:    movq %rax, %rcx
3165; AVX1-NEXT:    shlq $59, %rcx
3166; AVX1-NEXT:    sarq $63, %rcx
3167; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
3168; AVX1-NEXT:    movq %rax, %rcx
3169; AVX1-NEXT:    shlq $58, %rcx
3170; AVX1-NEXT:    sarq $63, %rcx
3171; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0
3172; AVX1-NEXT:    movq %rax, %rcx
3173; AVX1-NEXT:    shlq $57, %rcx
3174; AVX1-NEXT:    sarq $63, %rcx
3175; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0
3176; AVX1-NEXT:    movsbq %al, %rcx
3177; AVX1-NEXT:    shrq $7, %rcx
3178; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm0, %xmm0
3179; AVX1-NEXT:    movq %rax, %rcx
3180; AVX1-NEXT:    shlq $55, %rcx
3181; AVX1-NEXT:    sarq $63, %rcx
3182; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
3183; AVX1-NEXT:    movq %rax, %rcx
3184; AVX1-NEXT:    shlq $54, %rcx
3185; AVX1-NEXT:    sarq $63, %rcx
3186; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
3187; AVX1-NEXT:    movq %rax, %rcx
3188; AVX1-NEXT:    shlq $53, %rcx
3189; AVX1-NEXT:    sarq $63, %rcx
3190; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
3191; AVX1-NEXT:    movq %rax, %rcx
3192; AVX1-NEXT:    shlq $52, %rcx
3193; AVX1-NEXT:    sarq $63, %rcx
3194; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
3195; AVX1-NEXT:    movq %rax, %rcx
3196; AVX1-NEXT:    shlq $51, %rcx
3197; AVX1-NEXT:    sarq $63, %rcx
3198; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
3199; AVX1-NEXT:    movq %rax, %rcx
3200; AVX1-NEXT:    shlq $50, %rcx
3201; AVX1-NEXT:    sarq $63, %rcx
3202; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
3203; AVX1-NEXT:    movq %rax, %rcx
3204; AVX1-NEXT:    shlq $49, %rcx
3205; AVX1-NEXT:    sarq $63, %rcx
3206; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
3207; AVX1-NEXT:    shrq $15, %rax
3208; AVX1-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
3209; AVX1-NEXT:    retq
3210;
3211; AVX2-LABEL: load_sext_16i1_to_16i8:
3212; AVX2:       # %bb.0: # %entry
3213; AVX2-NEXT:    movswq (%rdi), %rax
3214; AVX2-NEXT:    movq %rax, %rcx
3215; AVX2-NEXT:    shlq $62, %rcx
3216; AVX2-NEXT:    sarq $63, %rcx
3217; AVX2-NEXT:    movq %rax, %rdx
3218; AVX2-NEXT:    shlq $63, %rdx
3219; AVX2-NEXT:    sarq $63, %rdx
3220; AVX2-NEXT:    vmovd %edx, %xmm0
3221; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
3222; AVX2-NEXT:    movq %rax, %rcx
3223; AVX2-NEXT:    shlq $61, %rcx
3224; AVX2-NEXT:    sarq $63, %rcx
3225; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0
3226; AVX2-NEXT:    movq %rax, %rcx
3227; AVX2-NEXT:    shlq $60, %rcx
3228; AVX2-NEXT:    sarq $63, %rcx
3229; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
3230; AVX2-NEXT:    movq %rax, %rcx
3231; AVX2-NEXT:    shlq $59, %rcx
3232; AVX2-NEXT:    sarq $63, %rcx
3233; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
3234; AVX2-NEXT:    movq %rax, %rcx
3235; AVX2-NEXT:    shlq $58, %rcx
3236; AVX2-NEXT:    sarq $63, %rcx
3237; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0
3238; AVX2-NEXT:    movq %rax, %rcx
3239; AVX2-NEXT:    shlq $57, %rcx
3240; AVX2-NEXT:    sarq $63, %rcx
3241; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0
3242; AVX2-NEXT:    movsbq %al, %rcx
3243; AVX2-NEXT:    shrq $7, %rcx
3244; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm0, %xmm0
3245; AVX2-NEXT:    movq %rax, %rcx
3246; AVX2-NEXT:    shlq $55, %rcx
3247; AVX2-NEXT:    sarq $63, %rcx
3248; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
3249; AVX2-NEXT:    movq %rax, %rcx
3250; AVX2-NEXT:    shlq $54, %rcx
3251; AVX2-NEXT:    sarq $63, %rcx
3252; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
3253; AVX2-NEXT:    movq %rax, %rcx
3254; AVX2-NEXT:    shlq $53, %rcx
3255; AVX2-NEXT:    sarq $63, %rcx
3256; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
3257; AVX2-NEXT:    movq %rax, %rcx
3258; AVX2-NEXT:    shlq $52, %rcx
3259; AVX2-NEXT:    sarq $63, %rcx
3260; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
3261; AVX2-NEXT:    movq %rax, %rcx
3262; AVX2-NEXT:    shlq $51, %rcx
3263; AVX2-NEXT:    sarq $63, %rcx
3264; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
3265; AVX2-NEXT:    movq %rax, %rcx
3266; AVX2-NEXT:    shlq $50, %rcx
3267; AVX2-NEXT:    sarq $63, %rcx
3268; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
3269; AVX2-NEXT:    movq %rax, %rcx
3270; AVX2-NEXT:    shlq $49, %rcx
3271; AVX2-NEXT:    sarq $63, %rcx
3272; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
3273; AVX2-NEXT:    shrq $15, %rax
3274; AVX2-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
3275; AVX2-NEXT:    retq
3276;
3277; AVX512F-LABEL: load_sext_16i1_to_16i8:
3278; AVX512F:       # %bb.0: # %entry
3279; AVX512F-NEXT:    kmovw (%rdi), %k1
3280; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
3281; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
3282; AVX512F-NEXT:    vzeroupper
3283; AVX512F-NEXT:    retq
3284;
3285; AVX512BW-LABEL: load_sext_16i1_to_16i8:
3286; AVX512BW:       # %bb.0: # %entry
3287; AVX512BW-NEXT:    kmovw (%rdi), %k0
3288; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
3289; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
3290; AVX512BW-NEXT:    vzeroupper
3291; AVX512BW-NEXT:    retq
3292;
3293; X32-SSE2-LABEL: load_sext_16i1_to_16i8:
3294; X32-SSE2:       # %bb.0: # %entry
3295; X32-SSE2-NEXT:    pushl %ebp
3296; X32-SSE2-NEXT:    pushl %ebx
3297; X32-SSE2-NEXT:    pushl %edi
3298; X32-SSE2-NEXT:    pushl %esi
3299; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3300; X32-SSE2-NEXT:    movswl (%eax), %eax
3301; X32-SSE2-NEXT:    movl %eax, %edx
3302; X32-SSE2-NEXT:    movl %eax, %ebp
3303; X32-SSE2-NEXT:    movl %eax, %ecx
3304; X32-SSE2-NEXT:    movl %eax, %esi
3305; X32-SSE2-NEXT:    movl %eax, %edi
3306; X32-SSE2-NEXT:    movl %eax, %ebx
3307; X32-SSE2-NEXT:    shrl $15, %ebx
3308; X32-SSE2-NEXT:    movd %ebx, %xmm2
3309; X32-SSE2-NEXT:    movl %eax, %ebx
3310; X32-SSE2-NEXT:    shll $17, %edx
3311; X32-SSE2-NEXT:    sarl $31, %edx
3312; X32-SSE2-NEXT:    movd %edx, %xmm0
3313; X32-SSE2-NEXT:    movl %eax, %edx
3314; X32-SSE2-NEXT:    shll $18, %ebp
3315; X32-SSE2-NEXT:    sarl $31, %ebp
3316; X32-SSE2-NEXT:    movd %ebp, %xmm1
3317; X32-SSE2-NEXT:    movl %eax, %ebp
3318; X32-SSE2-NEXT:    shll $19, %ecx
3319; X32-SSE2-NEXT:    sarl $31, %ecx
3320; X32-SSE2-NEXT:    movd %ecx, %xmm3
3321; X32-SSE2-NEXT:    movl %eax, %ecx
3322; X32-SSE2-NEXT:    shll $20, %esi
3323; X32-SSE2-NEXT:    sarl $31, %esi
3324; X32-SSE2-NEXT:    movd %esi, %xmm4
3325; X32-SSE2-NEXT:    movl %eax, %esi
3326; X32-SSE2-NEXT:    shll $21, %edi
3327; X32-SSE2-NEXT:    sarl $31, %edi
3328; X32-SSE2-NEXT:    movd %edi, %xmm6
3329; X32-SSE2-NEXT:    movl %eax, %edi
3330; X32-SSE2-NEXT:    shll $22, %ebx
3331; X32-SSE2-NEXT:    sarl $31, %ebx
3332; X32-SSE2-NEXT:    movd %ebx, %xmm7
3333; X32-SSE2-NEXT:    movl %eax, %ebx
3334; X32-SSE2-NEXT:    shll $23, %edx
3335; X32-SSE2-NEXT:    sarl $31, %edx
3336; X32-SSE2-NEXT:    movd %edx, %xmm5
3337; X32-SSE2-NEXT:    movl %eax, %edx
3338; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
3339; X32-SSE2-NEXT:    shll $28, %ebp
3340; X32-SSE2-NEXT:    sarl $31, %ebp
3341; X32-SSE2-NEXT:    movd %ebp, %xmm2
3342; X32-SSE2-NEXT:    movl %eax, %ebp
3343; X32-SSE2-NEXT:    movsbl %al, %eax
3344; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
3345; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
3346; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
3347; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
3348; X32-SSE2-NEXT:    shll $29, %ecx
3349; X32-SSE2-NEXT:    sarl $31, %ecx
3350; X32-SSE2-NEXT:    movd %ecx, %xmm1
3351; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3352; X32-SSE2-NEXT:    shll $30, %esi
3353; X32-SSE2-NEXT:    sarl $31, %esi
3354; X32-SSE2-NEXT:    movd %esi, %xmm4
3355; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
3356; X32-SSE2-NEXT:    shll $31, %edi
3357; X32-SSE2-NEXT:    sarl $31, %edi
3358; X32-SSE2-NEXT:    movd %edi, %xmm0
3359; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3360; X32-SSE2-NEXT:    shll $26, %ebx
3361; X32-SSE2-NEXT:    sarl $31, %ebx
3362; X32-SSE2-NEXT:    movd %ebx, %xmm2
3363; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3364; X32-SSE2-NEXT:    shll $27, %edx
3365; X32-SSE2-NEXT:    sarl $31, %edx
3366; X32-SSE2-NEXT:    movd %edx, %xmm3
3367; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3368; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
3369; X32-SSE2-NEXT:    shll $25, %ebp
3370; X32-SSE2-NEXT:    sarl $31, %ebp
3371; X32-SSE2-NEXT:    movd %ebp, %xmm1
3372; X32-SSE2-NEXT:    shrl $7, %eax
3373; X32-SSE2-NEXT:    movd %eax, %xmm2
3374; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
3375; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3376; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3377; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
3378; X32-SSE2-NEXT:    popl %esi
3379; X32-SSE2-NEXT:    popl %edi
3380; X32-SSE2-NEXT:    popl %ebx
3381; X32-SSE2-NEXT:    popl %ebp
3382; X32-SSE2-NEXT:    retl
3383;
3384; X32-SSE41-LABEL: load_sext_16i1_to_16i8:
3385; X32-SSE41:       # %bb.0: # %entry
3386; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3387; X32-SSE41-NEXT:    movswl (%eax), %eax
3388; X32-SSE41-NEXT:    movl %eax, %ecx
3389; X32-SSE41-NEXT:    shll $30, %ecx
3390; X32-SSE41-NEXT:    sarl $31, %ecx
3391; X32-SSE41-NEXT:    movl %eax, %edx
3392; X32-SSE41-NEXT:    shll $31, %edx
3393; X32-SSE41-NEXT:    sarl $31, %edx
3394; X32-SSE41-NEXT:    movd %edx, %xmm0
3395; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
3396; X32-SSE41-NEXT:    movl %eax, %ecx
3397; X32-SSE41-NEXT:    shll $29, %ecx
3398; X32-SSE41-NEXT:    sarl $31, %ecx
3399; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
3400; X32-SSE41-NEXT:    movl %eax, %ecx
3401; X32-SSE41-NEXT:    shll $28, %ecx
3402; X32-SSE41-NEXT:    sarl $31, %ecx
3403; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
3404; X32-SSE41-NEXT:    movl %eax, %ecx
3405; X32-SSE41-NEXT:    shll $27, %ecx
3406; X32-SSE41-NEXT:    sarl $31, %ecx
3407; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
3408; X32-SSE41-NEXT:    movl %eax, %ecx
3409; X32-SSE41-NEXT:    shll $26, %ecx
3410; X32-SSE41-NEXT:    sarl $31, %ecx
3411; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
3412; X32-SSE41-NEXT:    movl %eax, %ecx
3413; X32-SSE41-NEXT:    shll $25, %ecx
3414; X32-SSE41-NEXT:    sarl $31, %ecx
3415; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
3416; X32-SSE41-NEXT:    movsbl %al, %ecx
3417; X32-SSE41-NEXT:    shrl $7, %ecx
3418; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
3419; X32-SSE41-NEXT:    movl %eax, %ecx
3420; X32-SSE41-NEXT:    shll $23, %ecx
3421; X32-SSE41-NEXT:    sarl $31, %ecx
3422; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
3423; X32-SSE41-NEXT:    movl %eax, %ecx
3424; X32-SSE41-NEXT:    shll $22, %ecx
3425; X32-SSE41-NEXT:    sarl $31, %ecx
3426; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
3427; X32-SSE41-NEXT:    movl %eax, %ecx
3428; X32-SSE41-NEXT:    shll $21, %ecx
3429; X32-SSE41-NEXT:    sarl $31, %ecx
3430; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
3431; X32-SSE41-NEXT:    movl %eax, %ecx
3432; X32-SSE41-NEXT:    shll $20, %ecx
3433; X32-SSE41-NEXT:    sarl $31, %ecx
3434; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
3435; X32-SSE41-NEXT:    movl %eax, %ecx
3436; X32-SSE41-NEXT:    shll $19, %ecx
3437; X32-SSE41-NEXT:    sarl $31, %ecx
3438; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
3439; X32-SSE41-NEXT:    movl %eax, %ecx
3440; X32-SSE41-NEXT:    shll $18, %ecx
3441; X32-SSE41-NEXT:    sarl $31, %ecx
3442; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
3443; X32-SSE41-NEXT:    movl %eax, %ecx
3444; X32-SSE41-NEXT:    shll $17, %ecx
3445; X32-SSE41-NEXT:    sarl $31, %ecx
3446; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
3447; X32-SSE41-NEXT:    shrl $15, %eax
3448; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm0
3449; X32-SSE41-NEXT:    retl
3450entry:
3451 %X = load <16 x i1>, <16 x i1>* %ptr
3452 %Y = sext <16 x i1> %X to <16 x i8>
3453 ret <16 x i8> %Y
3454}
3455
3456define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
3457; SSE2-LABEL: load_sext_16i1_to_16i16:
3458; SSE2:       # %bb.0: # %entry
3459; SSE2-NEXT:    movzwl (%rdi), %eax
3460; SSE2-NEXT:    movl %eax, %ecx
3461; SSE2-NEXT:    shrl $15, %ecx
3462; SSE2-NEXT:    movd %ecx, %xmm0
3463; SSE2-NEXT:    movl %eax, %ecx
3464; SSE2-NEXT:    shrl $14, %ecx
3465; SSE2-NEXT:    andl $1, %ecx
3466; SSE2-NEXT:    movd %ecx, %xmm1
3467; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3468; SSE2-NEXT:    movl %eax, %ecx
3469; SSE2-NEXT:    shrl $13, %ecx
3470; SSE2-NEXT:    andl $1, %ecx
3471; SSE2-NEXT:    movd %ecx, %xmm0
3472; SSE2-NEXT:    movl %eax, %ecx
3473; SSE2-NEXT:    shrl $12, %ecx
3474; SSE2-NEXT:    andl $1, %ecx
3475; SSE2-NEXT:    movd %ecx, %xmm2
3476; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3477; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3478; SSE2-NEXT:    movl %eax, %ecx
3479; SSE2-NEXT:    shrl $11, %ecx
3480; SSE2-NEXT:    andl $1, %ecx
3481; SSE2-NEXT:    movd %ecx, %xmm0
3482; SSE2-NEXT:    movl %eax, %ecx
3483; SSE2-NEXT:    shrl $10, %ecx
3484; SSE2-NEXT:    andl $1, %ecx
3485; SSE2-NEXT:    movd %ecx, %xmm3
3486; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3487; SSE2-NEXT:    movl %eax, %ecx
3488; SSE2-NEXT:    shrl $9, %ecx
3489; SSE2-NEXT:    andl $1, %ecx
3490; SSE2-NEXT:    movd %ecx, %xmm0
3491; SSE2-NEXT:    movl %eax, %ecx
3492; SSE2-NEXT:    shrl $8, %ecx
3493; SSE2-NEXT:    andl $1, %ecx
3494; SSE2-NEXT:    movd %ecx, %xmm1
3495; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3496; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
3497; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3498; SSE2-NEXT:    movl %eax, %ecx
3499; SSE2-NEXT:    shrl $7, %ecx
3500; SSE2-NEXT:    andl $1, %ecx
3501; SSE2-NEXT:    movd %ecx, %xmm0
3502; SSE2-NEXT:    movl %eax, %ecx
3503; SSE2-NEXT:    shrl $6, %ecx
3504; SSE2-NEXT:    andl $1, %ecx
3505; SSE2-NEXT:    movd %ecx, %xmm2
3506; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3507; SSE2-NEXT:    movl %eax, %ecx
3508; SSE2-NEXT:    shrl $5, %ecx
3509; SSE2-NEXT:    andl $1, %ecx
3510; SSE2-NEXT:    movd %ecx, %xmm0
3511; SSE2-NEXT:    movl %eax, %ecx
3512; SSE2-NEXT:    shrl $4, %ecx
3513; SSE2-NEXT:    andl $1, %ecx
3514; SSE2-NEXT:    movd %ecx, %xmm3
3515; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3516; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
3517; SSE2-NEXT:    movl %eax, %ecx
3518; SSE2-NEXT:    shrl $3, %ecx
3519; SSE2-NEXT:    andl $1, %ecx
3520; SSE2-NEXT:    movd %ecx, %xmm0
3521; SSE2-NEXT:    movl %eax, %ecx
3522; SSE2-NEXT:    shrl $2, %ecx
3523; SSE2-NEXT:    andl $1, %ecx
3524; SSE2-NEXT:    movd %ecx, %xmm2
3525; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3526; SSE2-NEXT:    movl %eax, %ecx
3527; SSE2-NEXT:    andl $1, %ecx
3528; SSE2-NEXT:    movd %ecx, %xmm0
3529; SSE2-NEXT:    shrl %eax
3530; SSE2-NEXT:    andl $1, %eax
3531; SSE2-NEXT:    movd %eax, %xmm4
3532; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3533; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3534; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3535; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3536; SSE2-NEXT:    psllw $15, %xmm0
3537; SSE2-NEXT:    psraw $15, %xmm0
3538; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3539; SSE2-NEXT:    psllw $15, %xmm1
3540; SSE2-NEXT:    psraw $15, %xmm1
3541; SSE2-NEXT:    retq
3542;
3543; SSSE3-LABEL: load_sext_16i1_to_16i16:
3544; SSSE3:       # %bb.0: # %entry
3545; SSSE3-NEXT:    movzwl (%rdi), %eax
3546; SSSE3-NEXT:    movl %eax, %ecx
3547; SSSE3-NEXT:    shrl $15, %ecx
3548; SSSE3-NEXT:    movd %ecx, %xmm0
3549; SSSE3-NEXT:    movl %eax, %ecx
3550; SSSE3-NEXT:    shrl $14, %ecx
3551; SSSE3-NEXT:    andl $1, %ecx
3552; SSSE3-NEXT:    movd %ecx, %xmm1
3553; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3554; SSSE3-NEXT:    movl %eax, %ecx
3555; SSSE3-NEXT:    shrl $13, %ecx
3556; SSSE3-NEXT:    andl $1, %ecx
3557; SSSE3-NEXT:    movd %ecx, %xmm0
3558; SSSE3-NEXT:    movl %eax, %ecx
3559; SSSE3-NEXT:    shrl $12, %ecx
3560; SSSE3-NEXT:    andl $1, %ecx
3561; SSSE3-NEXT:    movd %ecx, %xmm2
3562; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3563; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3564; SSSE3-NEXT:    movl %eax, %ecx
3565; SSSE3-NEXT:    shrl $11, %ecx
3566; SSSE3-NEXT:    andl $1, %ecx
3567; SSSE3-NEXT:    movd %ecx, %xmm0
3568; SSSE3-NEXT:    movl %eax, %ecx
3569; SSSE3-NEXT:    shrl $10, %ecx
3570; SSSE3-NEXT:    andl $1, %ecx
3571; SSSE3-NEXT:    movd %ecx, %xmm3
3572; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3573; SSSE3-NEXT:    movl %eax, %ecx
3574; SSSE3-NEXT:    shrl $9, %ecx
3575; SSSE3-NEXT:    andl $1, %ecx
3576; SSSE3-NEXT:    movd %ecx, %xmm0
3577; SSSE3-NEXT:    movl %eax, %ecx
3578; SSSE3-NEXT:    shrl $8, %ecx
3579; SSSE3-NEXT:    andl $1, %ecx
3580; SSSE3-NEXT:    movd %ecx, %xmm1
3581; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3582; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
3583; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3584; SSSE3-NEXT:    movl %eax, %ecx
3585; SSSE3-NEXT:    shrl $7, %ecx
3586; SSSE3-NEXT:    andl $1, %ecx
3587; SSSE3-NEXT:    movd %ecx, %xmm0
3588; SSSE3-NEXT:    movl %eax, %ecx
3589; SSSE3-NEXT:    shrl $6, %ecx
3590; SSSE3-NEXT:    andl $1, %ecx
3591; SSSE3-NEXT:    movd %ecx, %xmm2
3592; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3593; SSSE3-NEXT:    movl %eax, %ecx
3594; SSSE3-NEXT:    shrl $5, %ecx
3595; SSSE3-NEXT:    andl $1, %ecx
3596; SSSE3-NEXT:    movd %ecx, %xmm0
3597; SSSE3-NEXT:    movl %eax, %ecx
3598; SSSE3-NEXT:    shrl $4, %ecx
3599; SSSE3-NEXT:    andl $1, %ecx
3600; SSSE3-NEXT:    movd %ecx, %xmm3
3601; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3602; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
3603; SSSE3-NEXT:    movl %eax, %ecx
3604; SSSE3-NEXT:    shrl $3, %ecx
3605; SSSE3-NEXT:    andl $1, %ecx
3606; SSSE3-NEXT:    movd %ecx, %xmm0
3607; SSSE3-NEXT:    movl %eax, %ecx
3608; SSSE3-NEXT:    shrl $2, %ecx
3609; SSSE3-NEXT:    andl $1, %ecx
3610; SSSE3-NEXT:    movd %ecx, %xmm2
3611; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3612; SSSE3-NEXT:    movl %eax, %ecx
3613; SSSE3-NEXT:    andl $1, %ecx
3614; SSSE3-NEXT:    movd %ecx, %xmm0
3615; SSSE3-NEXT:    shrl %eax
3616; SSSE3-NEXT:    andl $1, %eax
3617; SSSE3-NEXT:    movd %eax, %xmm4
3618; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3619; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3620; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3621; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3622; SSSE3-NEXT:    psllw $15, %xmm0
3623; SSSE3-NEXT:    psraw $15, %xmm0
3624; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3625; SSSE3-NEXT:    psllw $15, %xmm1
3626; SSSE3-NEXT:    psraw $15, %xmm1
3627; SSSE3-NEXT:    retq
3628;
3629; SSE41-LABEL: load_sext_16i1_to_16i16:
3630; SSE41:       # %bb.0: # %entry
3631; SSE41-NEXT:    movzwl (%rdi), %eax
3632; SSE41-NEXT:    movl %eax, %ecx
3633; SSE41-NEXT:    shrl %ecx
3634; SSE41-NEXT:    andl $1, %ecx
3635; SSE41-NEXT:    movl %eax, %edx
3636; SSE41-NEXT:    andl $1, %edx
3637; SSE41-NEXT:    movd %edx, %xmm1
3638; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
3639; SSE41-NEXT:    movl %eax, %ecx
3640; SSE41-NEXT:    shrl $2, %ecx
3641; SSE41-NEXT:    andl $1, %ecx
3642; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
3643; SSE41-NEXT:    movl %eax, %ecx
3644; SSE41-NEXT:    shrl $3, %ecx
3645; SSE41-NEXT:    andl $1, %ecx
3646; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
3647; SSE41-NEXT:    movl %eax, %ecx
3648; SSE41-NEXT:    shrl $4, %ecx
3649; SSE41-NEXT:    andl $1, %ecx
3650; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
3651; SSE41-NEXT:    movl %eax, %ecx
3652; SSE41-NEXT:    shrl $5, %ecx
3653; SSE41-NEXT:    andl $1, %ecx
3654; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
3655; SSE41-NEXT:    movl %eax, %ecx
3656; SSE41-NEXT:    shrl $6, %ecx
3657; SSE41-NEXT:    andl $1, %ecx
3658; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
3659; SSE41-NEXT:    movl %eax, %ecx
3660; SSE41-NEXT:    shrl $7, %ecx
3661; SSE41-NEXT:    andl $1, %ecx
3662; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
3663; SSE41-NEXT:    movl %eax, %ecx
3664; SSE41-NEXT:    shrl $8, %ecx
3665; SSE41-NEXT:    andl $1, %ecx
3666; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
3667; SSE41-NEXT:    movl %eax, %ecx
3668; SSE41-NEXT:    shrl $9, %ecx
3669; SSE41-NEXT:    andl $1, %ecx
3670; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
3671; SSE41-NEXT:    movl %eax, %ecx
3672; SSE41-NEXT:    shrl $10, %ecx
3673; SSE41-NEXT:    andl $1, %ecx
3674; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
3675; SSE41-NEXT:    movl %eax, %ecx
3676; SSE41-NEXT:    shrl $11, %ecx
3677; SSE41-NEXT:    andl $1, %ecx
3678; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
3679; SSE41-NEXT:    movl %eax, %ecx
3680; SSE41-NEXT:    shrl $12, %ecx
3681; SSE41-NEXT:    andl $1, %ecx
3682; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
3683; SSE41-NEXT:    movl %eax, %ecx
3684; SSE41-NEXT:    shrl $13, %ecx
3685; SSE41-NEXT:    andl $1, %ecx
3686; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
3687; SSE41-NEXT:    movl %eax, %ecx
3688; SSE41-NEXT:    shrl $14, %ecx
3689; SSE41-NEXT:    andl $1, %ecx
3690; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
3691; SSE41-NEXT:    shrl $15, %eax
3692; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
3693; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3694; SSE41-NEXT:    psllw $15, %xmm0
3695; SSE41-NEXT:    psraw $15, %xmm0
3696; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
3697; SSE41-NEXT:    psllw $15, %xmm1
3698; SSE41-NEXT:    psraw $15, %xmm1
3699; SSE41-NEXT:    retq
3700;
3701; AVX1-LABEL: load_sext_16i1_to_16i16:
3702; AVX1:       # %bb.0: # %entry
3703; AVX1-NEXT:    pushq %rbp
3704; AVX1-NEXT:    .cfi_def_cfa_offset 16
3705; AVX1-NEXT:    pushq %r15
3706; AVX1-NEXT:    .cfi_def_cfa_offset 24
3707; AVX1-NEXT:    pushq %r14
3708; AVX1-NEXT:    .cfi_def_cfa_offset 32
3709; AVX1-NEXT:    pushq %r13
3710; AVX1-NEXT:    .cfi_def_cfa_offset 40
3711; AVX1-NEXT:    pushq %r12
3712; AVX1-NEXT:    .cfi_def_cfa_offset 48
3713; AVX1-NEXT:    pushq %rbx
3714; AVX1-NEXT:    .cfi_def_cfa_offset 56
3715; AVX1-NEXT:    .cfi_offset %rbx, -56
3716; AVX1-NEXT:    .cfi_offset %r12, -48
3717; AVX1-NEXT:    .cfi_offset %r13, -40
3718; AVX1-NEXT:    .cfi_offset %r14, -32
3719; AVX1-NEXT:    .cfi_offset %r15, -24
3720; AVX1-NEXT:    .cfi_offset %rbp, -16
3721; AVX1-NEXT:    movswq (%rdi), %rax
3722; AVX1-NEXT:    movq %rax, %rcx
3723; AVX1-NEXT:    shlq $55, %rcx
3724; AVX1-NEXT:    sarq $63, %rcx
3725; AVX1-NEXT:    vmovd %ecx, %xmm0
3726; AVX1-NEXT:    movq %rax, %r8
3727; AVX1-NEXT:    movq %rax, %r10
3728; AVX1-NEXT:    movq %rax, %r11
3729; AVX1-NEXT:    movq %rax, %r14
3730; AVX1-NEXT:    movq %rax, %r15
3731; AVX1-NEXT:    movq %rax, %r9
3732; AVX1-NEXT:    movq %rax, %r12
3733; AVX1-NEXT:    movq %rax, %r13
3734; AVX1-NEXT:    movq %rax, %rbx
3735; AVX1-NEXT:    movq %rax, %rdi
3736; AVX1-NEXT:    movq %rax, %rcx
3737; AVX1-NEXT:    movq %rax, %rdx
3738; AVX1-NEXT:    movq %rax, %rsi
3739; AVX1-NEXT:    movsbq %al, %rbp
3740; AVX1-NEXT:    shlq $54, %rax
3741; AVX1-NEXT:    sarq $63, %rax
3742; AVX1-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
3743; AVX1-NEXT:    shlq $53, %r8
3744; AVX1-NEXT:    sarq $63, %r8
3745; AVX1-NEXT:    vpinsrw $2, %r8d, %xmm0, %xmm0
3746; AVX1-NEXT:    shlq $52, %r10
3747; AVX1-NEXT:    sarq $63, %r10
3748; AVX1-NEXT:    vpinsrw $3, %r10d, %xmm0, %xmm0
3749; AVX1-NEXT:    shlq $51, %r11
3750; AVX1-NEXT:    sarq $63, %r11
3751; AVX1-NEXT:    vpinsrw $4, %r11d, %xmm0, %xmm0
3752; AVX1-NEXT:    shlq $50, %r14
3753; AVX1-NEXT:    sarq $63, %r14
3754; AVX1-NEXT:    vpinsrw $5, %r14d, %xmm0, %xmm0
3755; AVX1-NEXT:    shlq $49, %r15
3756; AVX1-NEXT:    sarq $63, %r15
3757; AVX1-NEXT:    vpinsrw $6, %r15d, %xmm0, %xmm0
3758; AVX1-NEXT:    shrq $15, %r9
3759; AVX1-NEXT:    vpinsrw $7, %r9d, %xmm0, %xmm0
3760; AVX1-NEXT:    shlq $63, %r13
3761; AVX1-NEXT:    sarq $63, %r13
3762; AVX1-NEXT:    vmovd %r13d, %xmm1
3763; AVX1-NEXT:    shlq $62, %r12
3764; AVX1-NEXT:    sarq $63, %r12
3765; AVX1-NEXT:    vpinsrw $1, %r12d, %xmm1, %xmm1
3766; AVX1-NEXT:    shlq $61, %rbx
3767; AVX1-NEXT:    sarq $63, %rbx
3768; AVX1-NEXT:    vpinsrw $2, %ebx, %xmm1, %xmm1
3769; AVX1-NEXT:    shlq $60, %rdi
3770; AVX1-NEXT:    sarq $63, %rdi
3771; AVX1-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
3772; AVX1-NEXT:    shlq $59, %rcx
3773; AVX1-NEXT:    sarq $63, %rcx
3774; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
3775; AVX1-NEXT:    shlq $58, %rdx
3776; AVX1-NEXT:    sarq $63, %rdx
3777; AVX1-NEXT:    vpinsrw $5, %edx, %xmm1, %xmm1
3778; AVX1-NEXT:    shlq $57, %rsi
3779; AVX1-NEXT:    sarq $63, %rsi
3780; AVX1-NEXT:    vpinsrw $6, %esi, %xmm1, %xmm1
3781; AVX1-NEXT:    shrq $7, %rbp
3782; AVX1-NEXT:    vpinsrw $7, %ebp, %xmm1, %xmm1
3783; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
3784; AVX1-NEXT:    popq %rbx
3785; AVX1-NEXT:    .cfi_def_cfa_offset 48
3786; AVX1-NEXT:    popq %r12
3787; AVX1-NEXT:    .cfi_def_cfa_offset 40
3788; AVX1-NEXT:    popq %r13
3789; AVX1-NEXT:    .cfi_def_cfa_offset 32
3790; AVX1-NEXT:    popq %r14
3791; AVX1-NEXT:    .cfi_def_cfa_offset 24
3792; AVX1-NEXT:    popq %r15
3793; AVX1-NEXT:    .cfi_def_cfa_offset 16
3794; AVX1-NEXT:    popq %rbp
3795; AVX1-NEXT:    .cfi_def_cfa_offset 8
3796; AVX1-NEXT:    retq
3797;
3798; AVX2-LABEL: load_sext_16i1_to_16i16:
3799; AVX2:       # %bb.0: # %entry
3800; AVX2-NEXT:    pushq %rbp
3801; AVX2-NEXT:    .cfi_def_cfa_offset 16
3802; AVX2-NEXT:    pushq %r15
3803; AVX2-NEXT:    .cfi_def_cfa_offset 24
3804; AVX2-NEXT:    pushq %r14
3805; AVX2-NEXT:    .cfi_def_cfa_offset 32
3806; AVX2-NEXT:    pushq %r13
3807; AVX2-NEXT:    .cfi_def_cfa_offset 40
3808; AVX2-NEXT:    pushq %r12
3809; AVX2-NEXT:    .cfi_def_cfa_offset 48
3810; AVX2-NEXT:    pushq %rbx
3811; AVX2-NEXT:    .cfi_def_cfa_offset 56
3812; AVX2-NEXT:    .cfi_offset %rbx, -56
3813; AVX2-NEXT:    .cfi_offset %r12, -48
3814; AVX2-NEXT:    .cfi_offset %r13, -40
3815; AVX2-NEXT:    .cfi_offset %r14, -32
3816; AVX2-NEXT:    .cfi_offset %r15, -24
3817; AVX2-NEXT:    .cfi_offset %rbp, -16
3818; AVX2-NEXT:    movswq (%rdi), %rax
3819; AVX2-NEXT:    movq %rax, %rcx
3820; AVX2-NEXT:    shlq $55, %rcx
3821; AVX2-NEXT:    sarq $63, %rcx
3822; AVX2-NEXT:    vmovd %ecx, %xmm0
3823; AVX2-NEXT:    movq %rax, %r8
3824; AVX2-NEXT:    movq %rax, %r10
3825; AVX2-NEXT:    movq %rax, %r11
3826; AVX2-NEXT:    movq %rax, %r14
3827; AVX2-NEXT:    movq %rax, %r15
3828; AVX2-NEXT:    movq %rax, %r9
3829; AVX2-NEXT:    movq %rax, %r12
3830; AVX2-NEXT:    movq %rax, %r13
3831; AVX2-NEXT:    movq %rax, %rbx
3832; AVX2-NEXT:    movq %rax, %rdi
3833; AVX2-NEXT:    movq %rax, %rcx
3834; AVX2-NEXT:    movq %rax, %rdx
3835; AVX2-NEXT:    movq %rax, %rsi
3836; AVX2-NEXT:    movsbq %al, %rbp
3837; AVX2-NEXT:    shlq $54, %rax
3838; AVX2-NEXT:    sarq $63, %rax
3839; AVX2-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
3840; AVX2-NEXT:    shlq $53, %r8
3841; AVX2-NEXT:    sarq $63, %r8
3842; AVX2-NEXT:    vpinsrw $2, %r8d, %xmm0, %xmm0
3843; AVX2-NEXT:    shlq $52, %r10
3844; AVX2-NEXT:    sarq $63, %r10
3845; AVX2-NEXT:    vpinsrw $3, %r10d, %xmm0, %xmm0
3846; AVX2-NEXT:    shlq $51, %r11
3847; AVX2-NEXT:    sarq $63, %r11
3848; AVX2-NEXT:    vpinsrw $4, %r11d, %xmm0, %xmm0
3849; AVX2-NEXT:    shlq $50, %r14
3850; AVX2-NEXT:    sarq $63, %r14
3851; AVX2-NEXT:    vpinsrw $5, %r14d, %xmm0, %xmm0
3852; AVX2-NEXT:    shlq $49, %r15
3853; AVX2-NEXT:    sarq $63, %r15
3854; AVX2-NEXT:    vpinsrw $6, %r15d, %xmm0, %xmm0
3855; AVX2-NEXT:    shrq $15, %r9
3856; AVX2-NEXT:    vpinsrw $7, %r9d, %xmm0, %xmm0
3857; AVX2-NEXT:    shlq $63, %r13
3858; AVX2-NEXT:    sarq $63, %r13
3859; AVX2-NEXT:    vmovd %r13d, %xmm1
3860; AVX2-NEXT:    shlq $62, %r12
3861; AVX2-NEXT:    sarq $63, %r12
3862; AVX2-NEXT:    vpinsrw $1, %r12d, %xmm1, %xmm1
3863; AVX2-NEXT:    shlq $61, %rbx
3864; AVX2-NEXT:    sarq $63, %rbx
3865; AVX2-NEXT:    vpinsrw $2, %ebx, %xmm1, %xmm1
3866; AVX2-NEXT:    shlq $60, %rdi
3867; AVX2-NEXT:    sarq $63, %rdi
3868; AVX2-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
3869; AVX2-NEXT:    shlq $59, %rcx
3870; AVX2-NEXT:    sarq $63, %rcx
3871; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
3872; AVX2-NEXT:    shlq $58, %rdx
3873; AVX2-NEXT:    sarq $63, %rdx
3874; AVX2-NEXT:    vpinsrw $5, %edx, %xmm1, %xmm1
3875; AVX2-NEXT:    shlq $57, %rsi
3876; AVX2-NEXT:    sarq $63, %rsi
3877; AVX2-NEXT:    vpinsrw $6, %esi, %xmm1, %xmm1
3878; AVX2-NEXT:    shrq $7, %rbp
3879; AVX2-NEXT:    vpinsrw $7, %ebp, %xmm1, %xmm1
3880; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
3881; AVX2-NEXT:    popq %rbx
3882; AVX2-NEXT:    .cfi_def_cfa_offset 48
3883; AVX2-NEXT:    popq %r12
3884; AVX2-NEXT:    .cfi_def_cfa_offset 40
3885; AVX2-NEXT:    popq %r13
3886; AVX2-NEXT:    .cfi_def_cfa_offset 32
3887; AVX2-NEXT:    popq %r14
3888; AVX2-NEXT:    .cfi_def_cfa_offset 24
3889; AVX2-NEXT:    popq %r15
3890; AVX2-NEXT:    .cfi_def_cfa_offset 16
3891; AVX2-NEXT:    popq %rbp
3892; AVX2-NEXT:    .cfi_def_cfa_offset 8
3893; AVX2-NEXT:    retq
3894;
3895; AVX512F-LABEL: load_sext_16i1_to_16i16:
3896; AVX512F:       # %bb.0: # %entry
3897; AVX512F-NEXT:    kmovw (%rdi), %k1
3898; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
3899; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
3900; AVX512F-NEXT:    retq
3901;
3902; AVX512BW-LABEL: load_sext_16i1_to_16i16:
3903; AVX512BW:       # %bb.0: # %entry
3904; AVX512BW-NEXT:    kmovw (%rdi), %k0
3905; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
3906; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
3907; AVX512BW-NEXT:    retq
3908;
3909; X32-SSE2-LABEL: load_sext_16i1_to_16i16:
3910; X32-SSE2:       # %bb.0: # %entry
3911; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
3912; X32-SSE2-NEXT:    movzwl (%eax), %eax
3913; X32-SSE2-NEXT:    movl %eax, %ecx
3914; X32-SSE2-NEXT:    shrl $15, %ecx
3915; X32-SSE2-NEXT:    movd %ecx, %xmm0
3916; X32-SSE2-NEXT:    movl %eax, %ecx
3917; X32-SSE2-NEXT:    shrl $14, %ecx
3918; X32-SSE2-NEXT:    andl $1, %ecx
3919; X32-SSE2-NEXT:    movd %ecx, %xmm1
3920; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3921; X32-SSE2-NEXT:    movl %eax, %ecx
3922; X32-SSE2-NEXT:    shrl $13, %ecx
3923; X32-SSE2-NEXT:    andl $1, %ecx
3924; X32-SSE2-NEXT:    movd %ecx, %xmm0
3925; X32-SSE2-NEXT:    movl %eax, %ecx
3926; X32-SSE2-NEXT:    shrl $12, %ecx
3927; X32-SSE2-NEXT:    andl $1, %ecx
3928; X32-SSE2-NEXT:    movd %ecx, %xmm2
3929; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3930; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3931; X32-SSE2-NEXT:    movl %eax, %ecx
3932; X32-SSE2-NEXT:    shrl $11, %ecx
3933; X32-SSE2-NEXT:    andl $1, %ecx
3934; X32-SSE2-NEXT:    movd %ecx, %xmm0
3935; X32-SSE2-NEXT:    movl %eax, %ecx
3936; X32-SSE2-NEXT:    shrl $10, %ecx
3937; X32-SSE2-NEXT:    andl $1, %ecx
3938; X32-SSE2-NEXT:    movd %ecx, %xmm3
3939; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3940; X32-SSE2-NEXT:    movl %eax, %ecx
3941; X32-SSE2-NEXT:    shrl $9, %ecx
3942; X32-SSE2-NEXT:    andl $1, %ecx
3943; X32-SSE2-NEXT:    movd %ecx, %xmm0
3944; X32-SSE2-NEXT:    movl %eax, %ecx
3945; X32-SSE2-NEXT:    shrl $8, %ecx
3946; X32-SSE2-NEXT:    andl $1, %ecx
3947; X32-SSE2-NEXT:    movd %ecx, %xmm1
3948; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3949; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
3950; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
3951; X32-SSE2-NEXT:    movl %eax, %ecx
3952; X32-SSE2-NEXT:    shrl $7, %ecx
3953; X32-SSE2-NEXT:    andl $1, %ecx
3954; X32-SSE2-NEXT:    movd %ecx, %xmm0
3955; X32-SSE2-NEXT:    movl %eax, %ecx
3956; X32-SSE2-NEXT:    shrl $6, %ecx
3957; X32-SSE2-NEXT:    andl $1, %ecx
3958; X32-SSE2-NEXT:    movd %ecx, %xmm2
3959; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3960; X32-SSE2-NEXT:    movl %eax, %ecx
3961; X32-SSE2-NEXT:    shrl $5, %ecx
3962; X32-SSE2-NEXT:    andl $1, %ecx
3963; X32-SSE2-NEXT:    movd %ecx, %xmm0
3964; X32-SSE2-NEXT:    movl %eax, %ecx
3965; X32-SSE2-NEXT:    shrl $4, %ecx
3966; X32-SSE2-NEXT:    andl $1, %ecx
3967; X32-SSE2-NEXT:    movd %ecx, %xmm3
3968; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
3969; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
3970; X32-SSE2-NEXT:    movl %eax, %ecx
3971; X32-SSE2-NEXT:    shrl $3, %ecx
3972; X32-SSE2-NEXT:    andl $1, %ecx
3973; X32-SSE2-NEXT:    movd %ecx, %xmm0
3974; X32-SSE2-NEXT:    movl %eax, %ecx
3975; X32-SSE2-NEXT:    shrl $2, %ecx
3976; X32-SSE2-NEXT:    andl $1, %ecx
3977; X32-SSE2-NEXT:    movd %ecx, %xmm2
3978; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3979; X32-SSE2-NEXT:    movl %eax, %ecx
3980; X32-SSE2-NEXT:    andl $1, %ecx
3981; X32-SSE2-NEXT:    movd %ecx, %xmm0
3982; X32-SSE2-NEXT:    shrl %eax
3983; X32-SSE2-NEXT:    andl $1, %eax
3984; X32-SSE2-NEXT:    movd %eax, %xmm4
3985; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
3986; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
3987; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
3988; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3989; X32-SSE2-NEXT:    psllw $15, %xmm0
3990; X32-SSE2-NEXT:    psraw $15, %xmm0
3991; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3992; X32-SSE2-NEXT:    psllw $15, %xmm1
3993; X32-SSE2-NEXT:    psraw $15, %xmm1
3994; X32-SSE2-NEXT:    retl
3995;
3996; X32-SSE41-LABEL: load_sext_16i1_to_16i16:
3997; X32-SSE41:       # %bb.0: # %entry
3998; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
3999; X32-SSE41-NEXT:    movzwl (%eax), %eax
4000; X32-SSE41-NEXT:    movl %eax, %ecx
4001; X32-SSE41-NEXT:    shrl %ecx
4002; X32-SSE41-NEXT:    andl $1, %ecx
4003; X32-SSE41-NEXT:    movl %eax, %edx
4004; X32-SSE41-NEXT:    andl $1, %edx
4005; X32-SSE41-NEXT:    movd %edx, %xmm1
4006; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
4007; X32-SSE41-NEXT:    movl %eax, %ecx
4008; X32-SSE41-NEXT:    shrl $2, %ecx
4009; X32-SSE41-NEXT:    andl $1, %ecx
4010; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
4011; X32-SSE41-NEXT:    movl %eax, %ecx
4012; X32-SSE41-NEXT:    shrl $3, %ecx
4013; X32-SSE41-NEXT:    andl $1, %ecx
4014; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
4015; X32-SSE41-NEXT:    movl %eax, %ecx
4016; X32-SSE41-NEXT:    shrl $4, %ecx
4017; X32-SSE41-NEXT:    andl $1, %ecx
4018; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
4019; X32-SSE41-NEXT:    movl %eax, %ecx
4020; X32-SSE41-NEXT:    shrl $5, %ecx
4021; X32-SSE41-NEXT:    andl $1, %ecx
4022; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
4023; X32-SSE41-NEXT:    movl %eax, %ecx
4024; X32-SSE41-NEXT:    shrl $6, %ecx
4025; X32-SSE41-NEXT:    andl $1, %ecx
4026; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
4027; X32-SSE41-NEXT:    movl %eax, %ecx
4028; X32-SSE41-NEXT:    shrl $7, %ecx
4029; X32-SSE41-NEXT:    andl $1, %ecx
4030; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
4031; X32-SSE41-NEXT:    movl %eax, %ecx
4032; X32-SSE41-NEXT:    shrl $8, %ecx
4033; X32-SSE41-NEXT:    andl $1, %ecx
4034; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
4035; X32-SSE41-NEXT:    movl %eax, %ecx
4036; X32-SSE41-NEXT:    shrl $9, %ecx
4037; X32-SSE41-NEXT:    andl $1, %ecx
4038; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
4039; X32-SSE41-NEXT:    movl %eax, %ecx
4040; X32-SSE41-NEXT:    shrl $10, %ecx
4041; X32-SSE41-NEXT:    andl $1, %ecx
4042; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
4043; X32-SSE41-NEXT:    movl %eax, %ecx
4044; X32-SSE41-NEXT:    shrl $11, %ecx
4045; X32-SSE41-NEXT:    andl $1, %ecx
4046; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
4047; X32-SSE41-NEXT:    movl %eax, %ecx
4048; X32-SSE41-NEXT:    shrl $12, %ecx
4049; X32-SSE41-NEXT:    andl $1, %ecx
4050; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
4051; X32-SSE41-NEXT:    movl %eax, %ecx
4052; X32-SSE41-NEXT:    shrl $13, %ecx
4053; X32-SSE41-NEXT:    andl $1, %ecx
4054; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
4055; X32-SSE41-NEXT:    movl %eax, %ecx
4056; X32-SSE41-NEXT:    shrl $14, %ecx
4057; X32-SSE41-NEXT:    andl $1, %ecx
4058; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
4059; X32-SSE41-NEXT:    shrl $15, %eax
4060; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm1
4061; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
4062; X32-SSE41-NEXT:    psllw $15, %xmm0
4063; X32-SSE41-NEXT:    psraw $15, %xmm0
4064; X32-SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
4065; X32-SSE41-NEXT:    psllw $15, %xmm1
4066; X32-SSE41-NEXT:    psraw $15, %xmm1
4067; X32-SSE41-NEXT:    retl
4068entry:
4069 %X = load <16 x i1>, <16 x i1>* %ptr
4070 %Y = sext <16 x i1> %X to <16 x i16>
4071 ret <16 x i16> %Y
4072}
4073
4074define <32 x i8> @load_sext_32i1_to_32i8(<32 x i1> *%ptr) nounwind readnone {
4075; SSE2-LABEL: load_sext_32i1_to_32i8:
4076; SSE2:       # %bb.0: # %entry
4077; SSE2-NEXT:    pushq %rbp
4078; SSE2-NEXT:    pushq %r15
4079; SSE2-NEXT:    pushq %r14
4080; SSE2-NEXT:    pushq %r13
4081; SSE2-NEXT:    pushq %r12
4082; SSE2-NEXT:    pushq %rbx
4083; SSE2-NEXT:    movswq (%rdi), %rax
4084; SSE2-NEXT:    movq %rax, %r10
4085; SSE2-NEXT:    movq %rax, %r8
4086; SSE2-NEXT:    movq %rax, %r9
4087; SSE2-NEXT:    movq %rax, %r11
4088; SSE2-NEXT:    movq %rax, %r14
4089; SSE2-NEXT:    movq %rax, %r15
4090; SSE2-NEXT:    movq %rax, %r12
4091; SSE2-NEXT:    movq %rax, %r13
4092; SSE2-NEXT:    movq %rax, %rdx
4093; SSE2-NEXT:    movq %rax, %rsi
4094; SSE2-NEXT:    movq %rax, %rcx
4095; SSE2-NEXT:    movq %rax, %rbp
4096; SSE2-NEXT:    movq %rax, %rbx
4097; SSE2-NEXT:    shrq $15, %rbx
4098; SSE2-NEXT:    movd %ebx, %xmm0
4099; SSE2-NEXT:    movq %rax, %rbx
4100; SSE2-NEXT:    shlq $49, %r10
4101; SSE2-NEXT:    sarq $63, %r10
4102; SSE2-NEXT:    movd %r10d, %xmm15
4103; SSE2-NEXT:    movq %rax, %r10
4104; SSE2-NEXT:    movsbq %al, %rax
4105; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
4106; SSE2-NEXT:    shlq $50, %r8
4107; SSE2-NEXT:    sarq $63, %r8
4108; SSE2-NEXT:    movd %r8d, %xmm8
4109; SSE2-NEXT:    shlq $51, %r9
4110; SSE2-NEXT:    sarq $63, %r9
4111; SSE2-NEXT:    movd %r9d, %xmm3
4112; SSE2-NEXT:    shlq $52, %r11
4113; SSE2-NEXT:    sarq $63, %r11
4114; SSE2-NEXT:    movd %r11d, %xmm9
4115; SSE2-NEXT:    shlq $53, %r14
4116; SSE2-NEXT:    sarq $63, %r14
4117; SSE2-NEXT:    movd %r14d, %xmm6
4118; SSE2-NEXT:    shlq $54, %r15
4119; SSE2-NEXT:    sarq $63, %r15
4120; SSE2-NEXT:    movd %r15d, %xmm10
4121; SSE2-NEXT:    shlq $55, %r12
4122; SSE2-NEXT:    sarq $63, %r12
4123; SSE2-NEXT:    movd %r12d, %xmm2
4124; SSE2-NEXT:    shlq $60, %r13
4125; SSE2-NEXT:    sarq $63, %r13
4126; SSE2-NEXT:    movd %r13d, %xmm11
4127; SSE2-NEXT:    shlq $61, %rdx
4128; SSE2-NEXT:    sarq $63, %rdx
4129; SSE2-NEXT:    movd %edx, %xmm5
4130; SSE2-NEXT:    shlq $62, %rsi
4131; SSE2-NEXT:    sarq $63, %rsi
4132; SSE2-NEXT:    movd %esi, %xmm12
4133; SSE2-NEXT:    shlq $63, %rcx
4134; SSE2-NEXT:    sarq $63, %rcx
4135; SSE2-NEXT:    movd %ecx, %xmm0
4136; SSE2-NEXT:    shlq $58, %rbp
4137; SSE2-NEXT:    sarq $63, %rbp
4138; SSE2-NEXT:    movd %ebp, %xmm13
4139; SSE2-NEXT:    shlq $59, %rbx
4140; SSE2-NEXT:    sarq $63, %rbx
4141; SSE2-NEXT:    movd %ebx, %xmm7
4142; SSE2-NEXT:    shlq $57, %r10
4143; SSE2-NEXT:    sarq $63, %r10
4144; SSE2-NEXT:    movd %r10d, %xmm4
4145; SSE2-NEXT:    shrq $7, %rax
4146; SSE2-NEXT:    movd %eax, %xmm14
4147; SSE2-NEXT:    movswq 2(%rdi), %rsi
4148; SSE2-NEXT:    movq %rsi, %r8
4149; SSE2-NEXT:    movq %rsi, %r9
4150; SSE2-NEXT:    movq %rsi, %r10
4151; SSE2-NEXT:    movq %rsi, %r11
4152; SSE2-NEXT:    movq %rsi, %r14
4153; SSE2-NEXT:    movq %rsi, %r15
4154; SSE2-NEXT:    movq %rsi, %r12
4155; SSE2-NEXT:    movq %rsi, %r13
4156; SSE2-NEXT:    movq %rsi, %rbx
4157; SSE2-NEXT:    movq %rsi, %rax
4158; SSE2-NEXT:    movq %rsi, %rcx
4159; SSE2-NEXT:    movq %rsi, %rdx
4160; SSE2-NEXT:    movq %rsi, %rdi
4161; SSE2-NEXT:    movq %rsi, %rbp
4162; SSE2-NEXT:    shrq $15, %rbp
4163; SSE2-NEXT:    movd %ebp, %xmm1
4164; SSE2-NEXT:    movq %rsi, %rbp
4165; SSE2-NEXT:    movsbq %sil, %rsi
4166; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
4167; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
4168; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7]
4169; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]
4170; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
4171; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4172; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3],xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]
4173; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
4174; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
4175; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
4176; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
4177; SSE2-NEXT:    shlq $49, %r8
4178; SSE2-NEXT:    sarq $63, %r8
4179; SSE2-NEXT:    movd %r8d, %xmm3
4180; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
4181; SSE2-NEXT:    shlq $50, %r9
4182; SSE2-NEXT:    sarq $63, %r9
4183; SSE2-NEXT:    movd %r9d, %xmm4
4184; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
4185; SSE2-NEXT:    shlq $51, %r10
4186; SSE2-NEXT:    sarq $63, %r10
4187; SSE2-NEXT:    movd %r10d, %xmm5
4188; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
4189; SSE2-NEXT:    shlq $52, %r11
4190; SSE2-NEXT:    sarq $63, %r11
4191; SSE2-NEXT:    movd %r11d, %xmm2
4192; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
4193; SSE2-NEXT:    shlq $53, %r14
4194; SSE2-NEXT:    sarq $63, %r14
4195; SSE2-NEXT:    movd %r14d, %xmm1
4196; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
4197; SSE2-NEXT:    shlq $54, %r15
4198; SSE2-NEXT:    sarq $63, %r15
4199; SSE2-NEXT:    movd %r15d, %xmm4
4200; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
4201; SSE2-NEXT:    shlq $55, %r12
4202; SSE2-NEXT:    sarq $63, %r12
4203; SSE2-NEXT:    movd %r12d, %xmm3
4204; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
4205; SSE2-NEXT:    shlq $60, %r13
4206; SSE2-NEXT:    sarq $63, %r13
4207; SSE2-NEXT:    movd %r13d, %xmm2
4208; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
4209; SSE2-NEXT:    shlq $61, %rbx
4210; SSE2-NEXT:    sarq $63, %rbx
4211; SSE2-NEXT:    movd %ebx, %xmm4
4212; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
4213; SSE2-NEXT:    shlq $62, %rax
4214; SSE2-NEXT:    sarq $63, %rax
4215; SSE2-NEXT:    movd %eax, %xmm6
4216; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
4217; SSE2-NEXT:    shlq $63, %rcx
4218; SSE2-NEXT:    sarq $63, %rcx
4219; SSE2-NEXT:    movd %ecx, %xmm1
4220; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4221; SSE2-NEXT:    shlq $58, %rdx
4222; SSE2-NEXT:    sarq $63, %rdx
4223; SSE2-NEXT:    movd %edx, %xmm2
4224; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
4225; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
4226; SSE2-NEXT:    shlq $59, %rdi
4227; SSE2-NEXT:    sarq $63, %rdi
4228; SSE2-NEXT:    movd %edi, %xmm4
4229; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4230; SSE2-NEXT:    shlq $57, %rbp
4231; SSE2-NEXT:    sarq $63, %rbp
4232; SSE2-NEXT:    movd %ebp, %xmm2
4233; SSE2-NEXT:    shrq $7, %rsi
4234; SSE2-NEXT:    movd %esi, %xmm5
4235; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4236; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
4237; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
4238; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
4239; SSE2-NEXT:    popq %rbx
4240; SSE2-NEXT:    popq %r12
4241; SSE2-NEXT:    popq %r13
4242; SSE2-NEXT:    popq %r14
4243; SSE2-NEXT:    popq %r15
4244; SSE2-NEXT:    popq %rbp
4245; SSE2-NEXT:    retq
4246;
4247; SSSE3-LABEL: load_sext_32i1_to_32i8:
4248; SSSE3:       # %bb.0: # %entry
4249; SSSE3-NEXT:    pushq %rbp
4250; SSSE3-NEXT:    pushq %r15
4251; SSSE3-NEXT:    pushq %r14
4252; SSSE3-NEXT:    pushq %r13
4253; SSSE3-NEXT:    pushq %r12
4254; SSSE3-NEXT:    pushq %rbx
4255; SSSE3-NEXT:    movswq (%rdi), %rax
4256; SSSE3-NEXT:    movq %rax, %r10
4257; SSSE3-NEXT:    movq %rax, %r8
4258; SSSE3-NEXT:    movq %rax, %r9
4259; SSSE3-NEXT:    movq %rax, %r11
4260; SSSE3-NEXT:    movq %rax, %r14
4261; SSSE3-NEXT:    movq %rax, %r15
4262; SSSE3-NEXT:    movq %rax, %r12
4263; SSSE3-NEXT:    movq %rax, %r13
4264; SSSE3-NEXT:    movq %rax, %rdx
4265; SSSE3-NEXT:    movq %rax, %rsi
4266; SSSE3-NEXT:    movq %rax, %rcx
4267; SSSE3-NEXT:    movq %rax, %rbp
4268; SSSE3-NEXT:    movq %rax, %rbx
4269; SSSE3-NEXT:    shrq $15, %rbx
4270; SSSE3-NEXT:    movd %ebx, %xmm0
4271; SSSE3-NEXT:    movq %rax, %rbx
4272; SSSE3-NEXT:    shlq $49, %r10
4273; SSSE3-NEXT:    sarq $63, %r10
4274; SSSE3-NEXT:    movd %r10d, %xmm15
4275; SSSE3-NEXT:    movq %rax, %r10
4276; SSSE3-NEXT:    movsbq %al, %rax
4277; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
4278; SSSE3-NEXT:    shlq $50, %r8
4279; SSSE3-NEXT:    sarq $63, %r8
4280; SSSE3-NEXT:    movd %r8d, %xmm8
4281; SSSE3-NEXT:    shlq $51, %r9
4282; SSSE3-NEXT:    sarq $63, %r9
4283; SSSE3-NEXT:    movd %r9d, %xmm3
4284; SSSE3-NEXT:    shlq $52, %r11
4285; SSSE3-NEXT:    sarq $63, %r11
4286; SSSE3-NEXT:    movd %r11d, %xmm9
4287; SSSE3-NEXT:    shlq $53, %r14
4288; SSSE3-NEXT:    sarq $63, %r14
4289; SSSE3-NEXT:    movd %r14d, %xmm6
4290; SSSE3-NEXT:    shlq $54, %r15
4291; SSSE3-NEXT:    sarq $63, %r15
4292; SSSE3-NEXT:    movd %r15d, %xmm10
4293; SSSE3-NEXT:    shlq $55, %r12
4294; SSSE3-NEXT:    sarq $63, %r12
4295; SSSE3-NEXT:    movd %r12d, %xmm2
4296; SSSE3-NEXT:    shlq $60, %r13
4297; SSSE3-NEXT:    sarq $63, %r13
4298; SSSE3-NEXT:    movd %r13d, %xmm11
4299; SSSE3-NEXT:    shlq $61, %rdx
4300; SSSE3-NEXT:    sarq $63, %rdx
4301; SSSE3-NEXT:    movd %edx, %xmm5
4302; SSSE3-NEXT:    shlq $62, %rsi
4303; SSSE3-NEXT:    sarq $63, %rsi
4304; SSSE3-NEXT:    movd %esi, %xmm12
4305; SSSE3-NEXT:    shlq $63, %rcx
4306; SSSE3-NEXT:    sarq $63, %rcx
4307; SSSE3-NEXT:    movd %ecx, %xmm0
4308; SSSE3-NEXT:    shlq $58, %rbp
4309; SSSE3-NEXT:    sarq $63, %rbp
4310; SSSE3-NEXT:    movd %ebp, %xmm13
4311; SSSE3-NEXT:    shlq $59, %rbx
4312; SSSE3-NEXT:    sarq $63, %rbx
4313; SSSE3-NEXT:    movd %ebx, %xmm7
4314; SSSE3-NEXT:    shlq $57, %r10
4315; SSSE3-NEXT:    sarq $63, %r10
4316; SSSE3-NEXT:    movd %r10d, %xmm4
4317; SSSE3-NEXT:    shrq $7, %rax
4318; SSSE3-NEXT:    movd %eax, %xmm14
4319; SSSE3-NEXT:    movswq 2(%rdi), %rsi
4320; SSSE3-NEXT:    movq %rsi, %r8
4321; SSSE3-NEXT:    movq %rsi, %r9
4322; SSSE3-NEXT:    movq %rsi, %r10
4323; SSSE3-NEXT:    movq %rsi, %r11
4324; SSSE3-NEXT:    movq %rsi, %r14
4325; SSSE3-NEXT:    movq %rsi, %r15
4326; SSSE3-NEXT:    movq %rsi, %r12
4327; SSSE3-NEXT:    movq %rsi, %r13
4328; SSSE3-NEXT:    movq %rsi, %rbx
4329; SSSE3-NEXT:    movq %rsi, %rax
4330; SSSE3-NEXT:    movq %rsi, %rcx
4331; SSSE3-NEXT:    movq %rsi, %rdx
4332; SSSE3-NEXT:    movq %rsi, %rdi
4333; SSSE3-NEXT:    movq %rsi, %rbp
4334; SSSE3-NEXT:    shrq $15, %rbp
4335; SSSE3-NEXT:    movd %ebp, %xmm1
4336; SSSE3-NEXT:    movq %rsi, %rbp
4337; SSSE3-NEXT:    movsbq %sil, %rsi
4338; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
4339; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3]
4340; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7]
4341; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]
4342; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
4343; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4344; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3],xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]
4345; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
4346; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
4347; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
4348; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
4349; SSSE3-NEXT:    shlq $49, %r8
4350; SSSE3-NEXT:    sarq $63, %r8
4351; SSSE3-NEXT:    movd %r8d, %xmm3
4352; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
4353; SSSE3-NEXT:    shlq $50, %r9
4354; SSSE3-NEXT:    sarq $63, %r9
4355; SSSE3-NEXT:    movd %r9d, %xmm4
4356; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
4357; SSSE3-NEXT:    shlq $51, %r10
4358; SSSE3-NEXT:    sarq $63, %r10
4359; SSSE3-NEXT:    movd %r10d, %xmm5
4360; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
4361; SSSE3-NEXT:    shlq $52, %r11
4362; SSSE3-NEXT:    sarq $63, %r11
4363; SSSE3-NEXT:    movd %r11d, %xmm2
4364; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
4365; SSSE3-NEXT:    shlq $53, %r14
4366; SSSE3-NEXT:    sarq $63, %r14
4367; SSSE3-NEXT:    movd %r14d, %xmm1
4368; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
4369; SSSE3-NEXT:    shlq $54, %r15
4370; SSSE3-NEXT:    sarq $63, %r15
4371; SSSE3-NEXT:    movd %r15d, %xmm4
4372; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
4373; SSSE3-NEXT:    shlq $55, %r12
4374; SSSE3-NEXT:    sarq $63, %r12
4375; SSSE3-NEXT:    movd %r12d, %xmm3
4376; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
4377; SSSE3-NEXT:    shlq $60, %r13
4378; SSSE3-NEXT:    sarq $63, %r13
4379; SSSE3-NEXT:    movd %r13d, %xmm2
4380; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
4381; SSSE3-NEXT:    shlq $61, %rbx
4382; SSSE3-NEXT:    sarq $63, %rbx
4383; SSSE3-NEXT:    movd %ebx, %xmm4
4384; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
4385; SSSE3-NEXT:    shlq $62, %rax
4386; SSSE3-NEXT:    sarq $63, %rax
4387; SSSE3-NEXT:    movd %eax, %xmm6
4388; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
4389; SSSE3-NEXT:    shlq $63, %rcx
4390; SSSE3-NEXT:    sarq $63, %rcx
4391; SSSE3-NEXT:    movd %ecx, %xmm1
4392; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4393; SSSE3-NEXT:    shlq $58, %rdx
4394; SSSE3-NEXT:    sarq $63, %rdx
4395; SSSE3-NEXT:    movd %edx, %xmm2
4396; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
4397; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
4398; SSSE3-NEXT:    shlq $59, %rdi
4399; SSSE3-NEXT:    sarq $63, %rdi
4400; SSSE3-NEXT:    movd %edi, %xmm4
4401; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4402; SSSE3-NEXT:    shlq $57, %rbp
4403; SSSE3-NEXT:    sarq $63, %rbp
4404; SSSE3-NEXT:    movd %ebp, %xmm2
4405; SSSE3-NEXT:    shrq $7, %rsi
4406; SSSE3-NEXT:    movd %esi, %xmm5
4407; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4408; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
4409; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
4410; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
4411; SSSE3-NEXT:    popq %rbx
4412; SSSE3-NEXT:    popq %r12
4413; SSSE3-NEXT:    popq %r13
4414; SSSE3-NEXT:    popq %r14
4415; SSSE3-NEXT:    popq %r15
4416; SSSE3-NEXT:    popq %rbp
4417; SSSE3-NEXT:    retq
4418;
4419; SSE41-LABEL: load_sext_32i1_to_32i8:
4420; SSE41:       # %bb.0: # %entry
4421; SSE41-NEXT:    movswq (%rdi), %rax
4422; SSE41-NEXT:    movq %rax, %rcx
4423; SSE41-NEXT:    shlq $62, %rcx
4424; SSE41-NEXT:    sarq $63, %rcx
4425; SSE41-NEXT:    movq %rax, %rdx
4426; SSE41-NEXT:    shlq $63, %rdx
4427; SSE41-NEXT:    sarq $63, %rdx
4428; SSE41-NEXT:    movd %edx, %xmm0
4429; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
4430; SSE41-NEXT:    movq %rax, %rcx
4431; SSE41-NEXT:    shlq $61, %rcx
4432; SSE41-NEXT:    sarq $63, %rcx
4433; SSE41-NEXT:    pinsrb $2, %ecx, %xmm0
4434; SSE41-NEXT:    movq %rax, %rcx
4435; SSE41-NEXT:    shlq $60, %rcx
4436; SSE41-NEXT:    sarq $63, %rcx
4437; SSE41-NEXT:    pinsrb $3, %ecx, %xmm0
4438; SSE41-NEXT:    movq %rax, %rcx
4439; SSE41-NEXT:    shlq $59, %rcx
4440; SSE41-NEXT:    sarq $63, %rcx
4441; SSE41-NEXT:    pinsrb $4, %ecx, %xmm0
4442; SSE41-NEXT:    movq %rax, %rcx
4443; SSE41-NEXT:    shlq $58, %rcx
4444; SSE41-NEXT:    sarq $63, %rcx
4445; SSE41-NEXT:    pinsrb $5, %ecx, %xmm0
4446; SSE41-NEXT:    movq %rax, %rcx
4447; SSE41-NEXT:    shlq $57, %rcx
4448; SSE41-NEXT:    sarq $63, %rcx
4449; SSE41-NEXT:    pinsrb $6, %ecx, %xmm0
4450; SSE41-NEXT:    movsbq %al, %rcx
4451; SSE41-NEXT:    shrq $7, %rcx
4452; SSE41-NEXT:    pinsrb $7, %ecx, %xmm0
4453; SSE41-NEXT:    movq %rax, %rcx
4454; SSE41-NEXT:    shlq $55, %rcx
4455; SSE41-NEXT:    sarq $63, %rcx
4456; SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
4457; SSE41-NEXT:    movq %rax, %rcx
4458; SSE41-NEXT:    shlq $54, %rcx
4459; SSE41-NEXT:    sarq $63, %rcx
4460; SSE41-NEXT:    pinsrb $9, %ecx, %xmm0
4461; SSE41-NEXT:    movq %rax, %rcx
4462; SSE41-NEXT:    shlq $53, %rcx
4463; SSE41-NEXT:    sarq $63, %rcx
4464; SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
4465; SSE41-NEXT:    movq %rax, %rcx
4466; SSE41-NEXT:    shlq $52, %rcx
4467; SSE41-NEXT:    sarq $63, %rcx
4468; SSE41-NEXT:    pinsrb $11, %ecx, %xmm0
4469; SSE41-NEXT:    movq %rax, %rcx
4470; SSE41-NEXT:    shlq $51, %rcx
4471; SSE41-NEXT:    sarq $63, %rcx
4472; SSE41-NEXT:    pinsrb $12, %ecx, %xmm0
4473; SSE41-NEXT:    movq %rax, %rcx
4474; SSE41-NEXT:    shlq $50, %rcx
4475; SSE41-NEXT:    sarq $63, %rcx
4476; SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
4477; SSE41-NEXT:    movq %rax, %rcx
4478; SSE41-NEXT:    shlq $49, %rcx
4479; SSE41-NEXT:    sarq $63, %rcx
4480; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
4481; SSE41-NEXT:    shrq $15, %rax
4482; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
4483; SSE41-NEXT:    movswq 2(%rdi), %rax
4484; SSE41-NEXT:    movq %rax, %rcx
4485; SSE41-NEXT:    shlq $62, %rcx
4486; SSE41-NEXT:    sarq $63, %rcx
4487; SSE41-NEXT:    movq %rax, %rdx
4488; SSE41-NEXT:    shlq $63, %rdx
4489; SSE41-NEXT:    sarq $63, %rdx
4490; SSE41-NEXT:    movd %edx, %xmm1
4491; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
4492; SSE41-NEXT:    movq %rax, %rcx
4493; SSE41-NEXT:    shlq $61, %rcx
4494; SSE41-NEXT:    sarq $63, %rcx
4495; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
4496; SSE41-NEXT:    movq %rax, %rcx
4497; SSE41-NEXT:    shlq $60, %rcx
4498; SSE41-NEXT:    sarq $63, %rcx
4499; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
4500; SSE41-NEXT:    movq %rax, %rcx
4501; SSE41-NEXT:    shlq $59, %rcx
4502; SSE41-NEXT:    sarq $63, %rcx
4503; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
4504; SSE41-NEXT:    movq %rax, %rcx
4505; SSE41-NEXT:    shlq $58, %rcx
4506; SSE41-NEXT:    sarq $63, %rcx
4507; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
4508; SSE41-NEXT:    movq %rax, %rcx
4509; SSE41-NEXT:    shlq $57, %rcx
4510; SSE41-NEXT:    sarq $63, %rcx
4511; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
4512; SSE41-NEXT:    movsbq %al, %rcx
4513; SSE41-NEXT:    shrq $7, %rcx
4514; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
4515; SSE41-NEXT:    movq %rax, %rcx
4516; SSE41-NEXT:    shlq $55, %rcx
4517; SSE41-NEXT:    sarq $63, %rcx
4518; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
4519; SSE41-NEXT:    movq %rax, %rcx
4520; SSE41-NEXT:    shlq $54, %rcx
4521; SSE41-NEXT:    sarq $63, %rcx
4522; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
4523; SSE41-NEXT:    movq %rax, %rcx
4524; SSE41-NEXT:    shlq $53, %rcx
4525; SSE41-NEXT:    sarq $63, %rcx
4526; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
4527; SSE41-NEXT:    movq %rax, %rcx
4528; SSE41-NEXT:    shlq $52, %rcx
4529; SSE41-NEXT:    sarq $63, %rcx
4530; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
4531; SSE41-NEXT:    movq %rax, %rcx
4532; SSE41-NEXT:    shlq $51, %rcx
4533; SSE41-NEXT:    sarq $63, %rcx
4534; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
4535; SSE41-NEXT:    movq %rax, %rcx
4536; SSE41-NEXT:    shlq $50, %rcx
4537; SSE41-NEXT:    sarq $63, %rcx
4538; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
4539; SSE41-NEXT:    movq %rax, %rcx
4540; SSE41-NEXT:    shlq $49, %rcx
4541; SSE41-NEXT:    sarq $63, %rcx
4542; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
4543; SSE41-NEXT:    shrq $15, %rax
4544; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
4545; SSE41-NEXT:    retq
4546;
4547; AVX1-LABEL: load_sext_32i1_to_32i8:
4548; AVX1:       # %bb.0: # %entry
4549; AVX1-NEXT:    pushq %rbp
4550; AVX1-NEXT:    pushq %r15
4551; AVX1-NEXT:    pushq %r14
4552; AVX1-NEXT:    pushq %r13
4553; AVX1-NEXT:    pushq %r12
4554; AVX1-NEXT:    pushq %rbx
4555; AVX1-NEXT:    movslq (%rdi), %rax
4556; AVX1-NEXT:    movq %rax, %rcx
4557; AVX1-NEXT:    shlq $47, %rcx
4558; AVX1-NEXT:    sarq $63, %rcx
4559; AVX1-NEXT:    vmovd %ecx, %xmm0
4560; AVX1-NEXT:    movq %rax, %r8
4561; AVX1-NEXT:    movq %rax, %rdx
4562; AVX1-NEXT:    movq %rax, %rcx
4563; AVX1-NEXT:    movq %rax, %rdi
4564; AVX1-NEXT:    movq %rax, %r13
4565; AVX1-NEXT:    movq %rax, %rsi
4566; AVX1-NEXT:    movq %rax, %r10
4567; AVX1-NEXT:    movq %rax, %r11
4568; AVX1-NEXT:    movq %rax, %r9
4569; AVX1-NEXT:    movq %rax, %rbx
4570; AVX1-NEXT:    movq %rax, %r14
4571; AVX1-NEXT:    movq %rax, %r15
4572; AVX1-NEXT:    movq %rax, %r12
4573; AVX1-NEXT:    movq %rax, %rbp
4574; AVX1-NEXT:    shlq $46, %rbp
4575; AVX1-NEXT:    sarq $63, %rbp
4576; AVX1-NEXT:    vpinsrb $1, %ebp, %xmm0, %xmm0
4577; AVX1-NEXT:    movq %rax, %rbp
4578; AVX1-NEXT:    shlq $45, %r8
4579; AVX1-NEXT:    sarq $63, %r8
4580; AVX1-NEXT:    vpinsrb $2, %r8d, %xmm0, %xmm0
4581; AVX1-NEXT:    movq %rax, %r8
4582; AVX1-NEXT:    shlq $44, %rdx
4583; AVX1-NEXT:    sarq $63, %rdx
4584; AVX1-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
4585; AVX1-NEXT:    movq %rax, %rdx
4586; AVX1-NEXT:    shlq $43, %rcx
4587; AVX1-NEXT:    sarq $63, %rcx
4588; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
4589; AVX1-NEXT:    movq %rax, %rcx
4590; AVX1-NEXT:    shlq $42, %rdi
4591; AVX1-NEXT:    sarq $63, %rdi
4592; AVX1-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
4593; AVX1-NEXT:    movq %rax, %rdi
4594; AVX1-NEXT:    shlq $41, %r13
4595; AVX1-NEXT:    sarq $63, %r13
4596; AVX1-NEXT:    vpinsrb $6, %r13d, %xmm0, %xmm0
4597; AVX1-NEXT:    movq %rax, %r13
4598; AVX1-NEXT:    shlq $40, %rsi
4599; AVX1-NEXT:    sarq $63, %rsi
4600; AVX1-NEXT:    vpinsrb $7, %esi, %xmm0, %xmm0
4601; AVX1-NEXT:    movq %rax, %rsi
4602; AVX1-NEXT:    shlq $39, %r10
4603; AVX1-NEXT:    sarq $63, %r10
4604; AVX1-NEXT:    vpinsrb $8, %r10d, %xmm0, %xmm0
4605; AVX1-NEXT:    movq %rax, %r10
4606; AVX1-NEXT:    shlq $38, %r11
4607; AVX1-NEXT:    sarq $63, %r11
4608; AVX1-NEXT:    vpinsrb $9, %r11d, %xmm0, %xmm0
4609; AVX1-NEXT:    movsbq %al, %r11
4610; AVX1-NEXT:    shlq $37, %r9
4611; AVX1-NEXT:    sarq $63, %r9
4612; AVX1-NEXT:    vpinsrb $10, %r9d, %xmm0, %xmm0
4613; AVX1-NEXT:    movq %rax, %r9
4614; AVX1-NEXT:    shlq $36, %rbx
4615; AVX1-NEXT:    sarq $63, %rbx
4616; AVX1-NEXT:    vpinsrb $11, %ebx, %xmm0, %xmm0
4617; AVX1-NEXT:    movq %rax, %rbx
4618; AVX1-NEXT:    shlq $35, %r14
4619; AVX1-NEXT:    sarq $63, %r14
4620; AVX1-NEXT:    vpinsrb $12, %r14d, %xmm0, %xmm0
4621; AVX1-NEXT:    movq %rax, %r14
4622; AVX1-NEXT:    shlq $34, %r15
4623; AVX1-NEXT:    sarq $63, %r15
4624; AVX1-NEXT:    vpinsrb $13, %r15d, %xmm0, %xmm0
4625; AVX1-NEXT:    movq %rax, %r15
4626; AVX1-NEXT:    shlq $33, %r12
4627; AVX1-NEXT:    sarq $63, %r12
4628; AVX1-NEXT:    vpinsrb $14, %r12d, %xmm0, %xmm0
4629; AVX1-NEXT:    movq %rax, %r12
4630; AVX1-NEXT:    shrq $31, %rbp
4631; AVX1-NEXT:    vpinsrb $15, %ebp, %xmm0, %xmm0
4632; AVX1-NEXT:    movq %rax, %rbp
4633; AVX1-NEXT:    shlq $63, %rdx
4634; AVX1-NEXT:    sarq $63, %rdx
4635; AVX1-NEXT:    vmovd %edx, %xmm1
4636; AVX1-NEXT:    movq %rax, %rdx
4637; AVX1-NEXT:    movswq %ax, %rax
4638; AVX1-NEXT:    shlq $62, %r8
4639; AVX1-NEXT:    sarq $63, %r8
4640; AVX1-NEXT:    vpinsrb $1, %r8d, %xmm1, %xmm1
4641; AVX1-NEXT:    shlq $61, %rcx
4642; AVX1-NEXT:    sarq $63, %rcx
4643; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
4644; AVX1-NEXT:    shlq $60, %rdi
4645; AVX1-NEXT:    sarq $63, %rdi
4646; AVX1-NEXT:    vpinsrb $3, %edi, %xmm1, %xmm1
4647; AVX1-NEXT:    shlq $59, %r13
4648; AVX1-NEXT:    sarq $63, %r13
4649; AVX1-NEXT:    vpinsrb $4, %r13d, %xmm1, %xmm1
4650; AVX1-NEXT:    shlq $58, %rsi
4651; AVX1-NEXT:    sarq $63, %rsi
4652; AVX1-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
4653; AVX1-NEXT:    shlq $57, %r10
4654; AVX1-NEXT:    sarq $63, %r10
4655; AVX1-NEXT:    vpinsrb $6, %r10d, %xmm1, %xmm1
4656; AVX1-NEXT:    shrq $7, %r11
4657; AVX1-NEXT:    vpinsrb $7, %r11d, %xmm1, %xmm1
4658; AVX1-NEXT:    shlq $55, %r9
4659; AVX1-NEXT:    sarq $63, %r9
4660; AVX1-NEXT:    vpinsrb $8, %r9d, %xmm1, %xmm1
4661; AVX1-NEXT:    shlq $54, %rbx
4662; AVX1-NEXT:    sarq $63, %rbx
4663; AVX1-NEXT:    vpinsrb $9, %ebx, %xmm1, %xmm1
4664; AVX1-NEXT:    shlq $53, %r14
4665; AVX1-NEXT:    sarq $63, %r14
4666; AVX1-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
4667; AVX1-NEXT:    shlq $52, %r15
4668; AVX1-NEXT:    sarq $63, %r15
4669; AVX1-NEXT:    vpinsrb $11, %r15d, %xmm1, %xmm1
4670; AVX1-NEXT:    shlq $51, %r12
4671; AVX1-NEXT:    sarq $63, %r12
4672; AVX1-NEXT:    vpinsrb $12, %r12d, %xmm1, %xmm1
4673; AVX1-NEXT:    shlq $50, %rbp
4674; AVX1-NEXT:    sarq $63, %rbp
4675; AVX1-NEXT:    vpinsrb $13, %ebp, %xmm1, %xmm1
4676; AVX1-NEXT:    shlq $49, %rdx
4677; AVX1-NEXT:    sarq $63, %rdx
4678; AVX1-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
4679; AVX1-NEXT:    shrq $15, %rax
4680; AVX1-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
4681; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
4682; AVX1-NEXT:    popq %rbx
4683; AVX1-NEXT:    popq %r12
4684; AVX1-NEXT:    popq %r13
4685; AVX1-NEXT:    popq %r14
4686; AVX1-NEXT:    popq %r15
4687; AVX1-NEXT:    popq %rbp
4688; AVX1-NEXT:    retq
4689;
4690; AVX2-LABEL: load_sext_32i1_to_32i8:
4691; AVX2:       # %bb.0: # %entry
4692; AVX2-NEXT:    pushq %rbp
4693; AVX2-NEXT:    pushq %r15
4694; AVX2-NEXT:    pushq %r14
4695; AVX2-NEXT:    pushq %r13
4696; AVX2-NEXT:    pushq %r12
4697; AVX2-NEXT:    pushq %rbx
4698; AVX2-NEXT:    movslq (%rdi), %rax
4699; AVX2-NEXT:    movq %rax, %rcx
4700; AVX2-NEXT:    shlq $47, %rcx
4701; AVX2-NEXT:    sarq $63, %rcx
4702; AVX2-NEXT:    vmovd %ecx, %xmm0
4703; AVX2-NEXT:    movq %rax, %r8
4704; AVX2-NEXT:    movq %rax, %rdx
4705; AVX2-NEXT:    movq %rax, %rcx
4706; AVX2-NEXT:    movq %rax, %rdi
4707; AVX2-NEXT:    movq %rax, %r13
4708; AVX2-NEXT:    movq %rax, %rsi
4709; AVX2-NEXT:    movq %rax, %r10
4710; AVX2-NEXT:    movq %rax, %r11
4711; AVX2-NEXT:    movq %rax, %r9
4712; AVX2-NEXT:    movq %rax, %rbx
4713; AVX2-NEXT:    movq %rax, %r14
4714; AVX2-NEXT:    movq %rax, %r15
4715; AVX2-NEXT:    movq %rax, %r12
4716; AVX2-NEXT:    movq %rax, %rbp
4717; AVX2-NEXT:    shlq $46, %rbp
4718; AVX2-NEXT:    sarq $63, %rbp
4719; AVX2-NEXT:    vpinsrb $1, %ebp, %xmm0, %xmm0
4720; AVX2-NEXT:    movq %rax, %rbp
4721; AVX2-NEXT:    shlq $45, %r8
4722; AVX2-NEXT:    sarq $63, %r8
4723; AVX2-NEXT:    vpinsrb $2, %r8d, %xmm0, %xmm0
4724; AVX2-NEXT:    movq %rax, %r8
4725; AVX2-NEXT:    shlq $44, %rdx
4726; AVX2-NEXT:    sarq $63, %rdx
4727; AVX2-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
4728; AVX2-NEXT:    movq %rax, %rdx
4729; AVX2-NEXT:    shlq $43, %rcx
4730; AVX2-NEXT:    sarq $63, %rcx
4731; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
4732; AVX2-NEXT:    movq %rax, %rcx
4733; AVX2-NEXT:    shlq $42, %rdi
4734; AVX2-NEXT:    sarq $63, %rdi
4735; AVX2-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
4736; AVX2-NEXT:    movq %rax, %rdi
4737; AVX2-NEXT:    shlq $41, %r13
4738; AVX2-NEXT:    sarq $63, %r13
4739; AVX2-NEXT:    vpinsrb $6, %r13d, %xmm0, %xmm0
4740; AVX2-NEXT:    movq %rax, %r13
4741; AVX2-NEXT:    shlq $40, %rsi
4742; AVX2-NEXT:    sarq $63, %rsi
4743; AVX2-NEXT:    vpinsrb $7, %esi, %xmm0, %xmm0
4744; AVX2-NEXT:    movq %rax, %rsi
4745; AVX2-NEXT:    shlq $39, %r10
4746; AVX2-NEXT:    sarq $63, %r10
4747; AVX2-NEXT:    vpinsrb $8, %r10d, %xmm0, %xmm0
4748; AVX2-NEXT:    movq %rax, %r10
4749; AVX2-NEXT:    shlq $38, %r11
4750; AVX2-NEXT:    sarq $63, %r11
4751; AVX2-NEXT:    vpinsrb $9, %r11d, %xmm0, %xmm0
4752; AVX2-NEXT:    movsbq %al, %r11
4753; AVX2-NEXT:    shlq $37, %r9
4754; AVX2-NEXT:    sarq $63, %r9
4755; AVX2-NEXT:    vpinsrb $10, %r9d, %xmm0, %xmm0
4756; AVX2-NEXT:    movq %rax, %r9
4757; AVX2-NEXT:    shlq $36, %rbx
4758; AVX2-NEXT:    sarq $63, %rbx
4759; AVX2-NEXT:    vpinsrb $11, %ebx, %xmm0, %xmm0
4760; AVX2-NEXT:    movq %rax, %rbx
4761; AVX2-NEXT:    shlq $35, %r14
4762; AVX2-NEXT:    sarq $63, %r14
4763; AVX2-NEXT:    vpinsrb $12, %r14d, %xmm0, %xmm0
4764; AVX2-NEXT:    movq %rax, %r14
4765; AVX2-NEXT:    shlq $34, %r15
4766; AVX2-NEXT:    sarq $63, %r15
4767; AVX2-NEXT:    vpinsrb $13, %r15d, %xmm0, %xmm0
4768; AVX2-NEXT:    movq %rax, %r15
4769; AVX2-NEXT:    shlq $33, %r12
4770; AVX2-NEXT:    sarq $63, %r12
4771; AVX2-NEXT:    vpinsrb $14, %r12d, %xmm0, %xmm0
4772; AVX2-NEXT:    movq %rax, %r12
4773; AVX2-NEXT:    shrq $31, %rbp
4774; AVX2-NEXT:    vpinsrb $15, %ebp, %xmm0, %xmm0
4775; AVX2-NEXT:    movq %rax, %rbp
4776; AVX2-NEXT:    shlq $63, %rdx
4777; AVX2-NEXT:    sarq $63, %rdx
4778; AVX2-NEXT:    vmovd %edx, %xmm1
4779; AVX2-NEXT:    movq %rax, %rdx
4780; AVX2-NEXT:    movswq %ax, %rax
4781; AVX2-NEXT:    shlq $62, %r8
4782; AVX2-NEXT:    sarq $63, %r8
4783; AVX2-NEXT:    vpinsrb $1, %r8d, %xmm1, %xmm1
4784; AVX2-NEXT:    shlq $61, %rcx
4785; AVX2-NEXT:    sarq $63, %rcx
4786; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
4787; AVX2-NEXT:    shlq $60, %rdi
4788; AVX2-NEXT:    sarq $63, %rdi
4789; AVX2-NEXT:    vpinsrb $3, %edi, %xmm1, %xmm1
4790; AVX2-NEXT:    shlq $59, %r13
4791; AVX2-NEXT:    sarq $63, %r13
4792; AVX2-NEXT:    vpinsrb $4, %r13d, %xmm1, %xmm1
4793; AVX2-NEXT:    shlq $58, %rsi
4794; AVX2-NEXT:    sarq $63, %rsi
4795; AVX2-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
4796; AVX2-NEXT:    shlq $57, %r10
4797; AVX2-NEXT:    sarq $63, %r10
4798; AVX2-NEXT:    vpinsrb $6, %r10d, %xmm1, %xmm1
4799; AVX2-NEXT:    shrq $7, %r11
4800; AVX2-NEXT:    vpinsrb $7, %r11d, %xmm1, %xmm1
4801; AVX2-NEXT:    shlq $55, %r9
4802; AVX2-NEXT:    sarq $63, %r9
4803; AVX2-NEXT:    vpinsrb $8, %r9d, %xmm1, %xmm1
4804; AVX2-NEXT:    shlq $54, %rbx
4805; AVX2-NEXT:    sarq $63, %rbx
4806; AVX2-NEXT:    vpinsrb $9, %ebx, %xmm1, %xmm1
4807; AVX2-NEXT:    shlq $53, %r14
4808; AVX2-NEXT:    sarq $63, %r14
4809; AVX2-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
4810; AVX2-NEXT:    shlq $52, %r15
4811; AVX2-NEXT:    sarq $63, %r15
4812; AVX2-NEXT:    vpinsrb $11, %r15d, %xmm1, %xmm1
4813; AVX2-NEXT:    shlq $51, %r12
4814; AVX2-NEXT:    sarq $63, %r12
4815; AVX2-NEXT:    vpinsrb $12, %r12d, %xmm1, %xmm1
4816; AVX2-NEXT:    shlq $50, %rbp
4817; AVX2-NEXT:    sarq $63, %rbp
4818; AVX2-NEXT:    vpinsrb $13, %ebp, %xmm1, %xmm1
4819; AVX2-NEXT:    shlq $49, %rdx
4820; AVX2-NEXT:    sarq $63, %rdx
4821; AVX2-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
4822; AVX2-NEXT:    shrq $15, %rax
4823; AVX2-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
4824; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
4825; AVX2-NEXT:    popq %rbx
4826; AVX2-NEXT:    popq %r12
4827; AVX2-NEXT:    popq %r13
4828; AVX2-NEXT:    popq %r14
4829; AVX2-NEXT:    popq %r15
4830; AVX2-NEXT:    popq %rbp
4831; AVX2-NEXT:    retq
4832;
4833; AVX512F-LABEL: load_sext_32i1_to_32i8:
4834; AVX512F:       # %bb.0: # %entry
4835; AVX512F-NEXT:    kmovw (%rdi), %k1
4836; AVX512F-NEXT:    kmovw 2(%rdi), %k2
4837; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
4838; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
4839; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
4840; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
4841; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
4842; AVX512F-NEXT:    retq
4843;
4844; AVX512BW-LABEL: load_sext_32i1_to_32i8:
4845; AVX512BW:       # %bb.0: # %entry
4846; AVX512BW-NEXT:    kmovd (%rdi), %k0
4847; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
4848; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
4849; AVX512BW-NEXT:    retq
4850;
4851; X32-SSE2-LABEL: load_sext_32i1_to_32i8:
4852; X32-SSE2:       # %bb.0: # %entry
4853; X32-SSE2-NEXT:    pushl %ebp
4854; X32-SSE2-NEXT:    pushl %ebx
4855; X32-SSE2-NEXT:    pushl %edi
4856; X32-SSE2-NEXT:    pushl %esi
4857; X32-SSE2-NEXT:    subl $28, %esp
4858; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
4859; X32-SSE2-NEXT:    movswl (%eax), %edx
4860; X32-SSE2-NEXT:    movl %edx, %ebp
4861; X32-SSE2-NEXT:    movl %edx, %esi
4862; X32-SSE2-NEXT:    movl %edx, %edi
4863; X32-SSE2-NEXT:    movl %edx, %ebx
4864; X32-SSE2-NEXT:    movl %edx, %ecx
4865; X32-SSE2-NEXT:    shrl $15, %ecx
4866; X32-SSE2-NEXT:    movd %ecx, %xmm0
4867; X32-SSE2-NEXT:    movl %edx, %ecx
4868; X32-SSE2-NEXT:    shll $17, %ebp
4869; X32-SSE2-NEXT:    sarl $31, %ebp
4870; X32-SSE2-NEXT:    movd %ebp, %xmm4
4871; X32-SSE2-NEXT:    movl %edx, %ebp
4872; X32-SSE2-NEXT:    shll $18, %esi
4873; X32-SSE2-NEXT:    sarl $31, %esi
4874; X32-SSE2-NEXT:    movd %esi, %xmm1
4875; X32-SSE2-NEXT:    movl %edx, %esi
4876; X32-SSE2-NEXT:    shll $19, %edi
4877; X32-SSE2-NEXT:    sarl $31, %edi
4878; X32-SSE2-NEXT:    movd %edi, %xmm2
4879; X32-SSE2-NEXT:    movl %edx, %edi
4880; X32-SSE2-NEXT:    shll $20, %ebx
4881; X32-SSE2-NEXT:    sarl $31, %ebx
4882; X32-SSE2-NEXT:    movd %ebx, %xmm5
4883; X32-SSE2-NEXT:    movl %edx, %ebx
4884; X32-SSE2-NEXT:    shll $21, %ecx
4885; X32-SSE2-NEXT:    sarl $31, %ecx
4886; X32-SSE2-NEXT:    movd %ecx, %xmm6
4887; X32-SSE2-NEXT:    movl %edx, %ecx
4888; X32-SSE2-NEXT:    shll $22, %ebp
4889; X32-SSE2-NEXT:    sarl $31, %ebp
4890; X32-SSE2-NEXT:    movd %ebp, %xmm7
4891; X32-SSE2-NEXT:    movl %edx, %ebp
4892; X32-SSE2-NEXT:    shll $23, %esi
4893; X32-SSE2-NEXT:    sarl $31, %esi
4894; X32-SSE2-NEXT:    movd %esi, %xmm3
4895; X32-SSE2-NEXT:    movl %edx, %esi
4896; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
4897; X32-SSE2-NEXT:    shll $28, %edi
4898; X32-SSE2-NEXT:    sarl $31, %edi
4899; X32-SSE2-NEXT:    movd %edi, %xmm0
4900; X32-SSE2-NEXT:    movl %edx, %edi
4901; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
4902; X32-SSE2-NEXT:    shll $29, %ebx
4903; X32-SSE2-NEXT:    sarl $31, %ebx
4904; X32-SSE2-NEXT:    movd %ebx, %xmm1
4905; X32-SSE2-NEXT:    movl %edx, %ebx
4906; X32-SSE2-NEXT:    movsbl %dl, %edx
4907; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
4908; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
4909; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
4910; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
4911; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4912; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
4913; X32-SSE2-NEXT:    shll $30, %ecx
4914; X32-SSE2-NEXT:    sarl $31, %ecx
4915; X32-SSE2-NEXT:    movd %ecx, %xmm2
4916; X32-SSE2-NEXT:    shll $31, %ebp
4917; X32-SSE2-NEXT:    sarl $31, %ebp
4918; X32-SSE2-NEXT:    movd %ebp, %xmm0
4919; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
4920; X32-SSE2-NEXT:    shll $26, %esi
4921; X32-SSE2-NEXT:    sarl $31, %esi
4922; X32-SSE2-NEXT:    movd %esi, %xmm7
4923; X32-SSE2-NEXT:    shll $27, %edi
4924; X32-SSE2-NEXT:    sarl $31, %edi
4925; X32-SSE2-NEXT:    movd %edi, %xmm2
4926; X32-SSE2-NEXT:    shll $25, %ebx
4927; X32-SSE2-NEXT:    sarl $31, %ebx
4928; X32-SSE2-NEXT:    movd %ebx, %xmm6
4929; X32-SSE2-NEXT:    shrl $7, %edx
4930; X32-SSE2-NEXT:    movd %edx, %xmm5
4931; X32-SSE2-NEXT:    movswl 2(%eax), %eax
4932; X32-SSE2-NEXT:    movl %eax, %edx
4933; X32-SSE2-NEXT:    movl %eax, %ebp
4934; X32-SSE2-NEXT:    movl %eax, %ecx
4935; X32-SSE2-NEXT:    movl %eax, %esi
4936; X32-SSE2-NEXT:    movl %eax, %edi
4937; X32-SSE2-NEXT:    movl %eax, %ebx
4938; X32-SSE2-NEXT:    shrl $15, %ebx
4939; X32-SSE2-NEXT:    movd %ebx, %xmm4
4940; X32-SSE2-NEXT:    movdqu %xmm4, (%esp) # 16-byte Spill
4941; X32-SSE2-NEXT:    movl %eax, %ebx
4942; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
4943; X32-SSE2-NEXT:    shll $17, %edx
4944; X32-SSE2-NEXT:    sarl $31, %edx
4945; X32-SSE2-NEXT:    movd %edx, %xmm4
4946; X32-SSE2-NEXT:    movl %eax, %edx
4947; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3],xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
4948; X32-SSE2-NEXT:    shll $18, %ebp
4949; X32-SSE2-NEXT:    sarl $31, %ebp
4950; X32-SSE2-NEXT:    movd %ebp, %xmm7
4951; X32-SSE2-NEXT:    movl %eax, %ebp
4952; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
4953; X32-SSE2-NEXT:    shll $19, %ecx
4954; X32-SSE2-NEXT:    sarl $31, %ecx
4955; X32-SSE2-NEXT:    movd %ecx, %xmm5
4956; X32-SSE2-NEXT:    movl %eax, %ecx
4957; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
4958; X32-SSE2-NEXT:    shll $20, %esi
4959; X32-SSE2-NEXT:    sarl $31, %esi
4960; X32-SSE2-NEXT:    movd %esi, %xmm6
4961; X32-SSE2-NEXT:    movl %eax, %esi
4962; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
4963; X32-SSE2-NEXT:    shll $21, %edi
4964; X32-SSE2-NEXT:    sarl $31, %edi
4965; X32-SSE2-NEXT:    movd %edi, %xmm1
4966; X32-SSE2-NEXT:    movl %eax, %edi
4967; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
4968; X32-SSE2-NEXT:    shll $22, %ebx
4969; X32-SSE2-NEXT:    sarl $31, %ebx
4970; X32-SSE2-NEXT:    movd %ebx, %xmm3
4971; X32-SSE2-NEXT:    movl %eax, %ebx
4972; X32-SSE2-NEXT:    movdqu (%esp), %xmm2 # 16-byte Reload
4973; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
4974; X32-SSE2-NEXT:    shll $23, %edx
4975; X32-SSE2-NEXT:    sarl $31, %edx
4976; X32-SSE2-NEXT:    movd %edx, %xmm2
4977; X32-SSE2-NEXT:    movl %eax, %edx
4978; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
4979; X32-SSE2-NEXT:    shll $28, %ebp
4980; X32-SSE2-NEXT:    sarl $31, %ebp
4981; X32-SSE2-NEXT:    movd %ebp, %xmm7
4982; X32-SSE2-NEXT:    movl %eax, %ebp
4983; X32-SSE2-NEXT:    movsbl %al, %eax
4984; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
4985; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
4986; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
4987; X32-SSE2-NEXT:    shll $29, %ecx
4988; X32-SSE2-NEXT:    sarl $31, %ecx
4989; X32-SSE2-NEXT:    movd %ecx, %xmm3
4990; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4991; X32-SSE2-NEXT:    shll $30, %esi
4992; X32-SSE2-NEXT:    sarl $31, %esi
4993; X32-SSE2-NEXT:    movd %esi, %xmm4
4994; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
4995; X32-SSE2-NEXT:    shll $31, %edi
4996; X32-SSE2-NEXT:    sarl $31, %edi
4997; X32-SSE2-NEXT:    movd %edi, %xmm1
4998; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
4999; X32-SSE2-NEXT:    shll $26, %ebx
5000; X32-SSE2-NEXT:    sarl $31, %ebx
5001; X32-SSE2-NEXT:    movd %ebx, %xmm5
5002; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
5003; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
5004; X32-SSE2-NEXT:    shll $27, %edx
5005; X32-SSE2-NEXT:    sarl $31, %edx
5006; X32-SSE2-NEXT:    movd %edx, %xmm3
5007; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
5008; X32-SSE2-NEXT:    shll $25, %ebp
5009; X32-SSE2-NEXT:    sarl $31, %ebp
5010; X32-SSE2-NEXT:    movd %ebp, %xmm4
5011; X32-SSE2-NEXT:    shrl $7, %eax
5012; X32-SSE2-NEXT:    movd %eax, %xmm5
5013; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
5014; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
5015; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
5016; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5017; X32-SSE2-NEXT:    addl $28, %esp
5018; X32-SSE2-NEXT:    popl %esi
5019; X32-SSE2-NEXT:    popl %edi
5020; X32-SSE2-NEXT:    popl %ebx
5021; X32-SSE2-NEXT:    popl %ebp
5022; X32-SSE2-NEXT:    retl
5023;
5024; X32-SSE41-LABEL: load_sext_32i1_to_32i8:
5025; X32-SSE41:       # %bb.0: # %entry
5026; X32-SSE41-NEXT:    pushl %esi
5027; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5028; X32-SSE41-NEXT:    movswl (%eax), %ecx
5029; X32-SSE41-NEXT:    movl %ecx, %edx
5030; X32-SSE41-NEXT:    shll $30, %edx
5031; X32-SSE41-NEXT:    sarl $31, %edx
5032; X32-SSE41-NEXT:    movl %ecx, %esi
5033; X32-SSE41-NEXT:    shll $31, %esi
5034; X32-SSE41-NEXT:    sarl $31, %esi
5035; X32-SSE41-NEXT:    movd %esi, %xmm0
5036; X32-SSE41-NEXT:    pinsrb $1, %edx, %xmm0
5037; X32-SSE41-NEXT:    movl %ecx, %edx
5038; X32-SSE41-NEXT:    shll $29, %edx
5039; X32-SSE41-NEXT:    sarl $31, %edx
5040; X32-SSE41-NEXT:    pinsrb $2, %edx, %xmm0
5041; X32-SSE41-NEXT:    movl %ecx, %edx
5042; X32-SSE41-NEXT:    shll $28, %edx
5043; X32-SSE41-NEXT:    sarl $31, %edx
5044; X32-SSE41-NEXT:    pinsrb $3, %edx, %xmm0
5045; X32-SSE41-NEXT:    movl %ecx, %edx
5046; X32-SSE41-NEXT:    shll $27, %edx
5047; X32-SSE41-NEXT:    sarl $31, %edx
5048; X32-SSE41-NEXT:    pinsrb $4, %edx, %xmm0
5049; X32-SSE41-NEXT:    movl %ecx, %edx
5050; X32-SSE41-NEXT:    shll $26, %edx
5051; X32-SSE41-NEXT:    sarl $31, %edx
5052; X32-SSE41-NEXT:    pinsrb $5, %edx, %xmm0
5053; X32-SSE41-NEXT:    movl %ecx, %edx
5054; X32-SSE41-NEXT:    shll $25, %edx
5055; X32-SSE41-NEXT:    sarl $31, %edx
5056; X32-SSE41-NEXT:    pinsrb $6, %edx, %xmm0
5057; X32-SSE41-NEXT:    movsbl %cl, %edx
5058; X32-SSE41-NEXT:    shrl $7, %edx
5059; X32-SSE41-NEXT:    pinsrb $7, %edx, %xmm0
5060; X32-SSE41-NEXT:    movl %ecx, %edx
5061; X32-SSE41-NEXT:    shll $23, %edx
5062; X32-SSE41-NEXT:    sarl $31, %edx
5063; X32-SSE41-NEXT:    pinsrb $8, %edx, %xmm0
5064; X32-SSE41-NEXT:    movl %ecx, %edx
5065; X32-SSE41-NEXT:    shll $22, %edx
5066; X32-SSE41-NEXT:    sarl $31, %edx
5067; X32-SSE41-NEXT:    pinsrb $9, %edx, %xmm0
5068; X32-SSE41-NEXT:    movl %ecx, %edx
5069; X32-SSE41-NEXT:    shll $21, %edx
5070; X32-SSE41-NEXT:    sarl $31, %edx
5071; X32-SSE41-NEXT:    pinsrb $10, %edx, %xmm0
5072; X32-SSE41-NEXT:    movl %ecx, %edx
5073; X32-SSE41-NEXT:    shll $20, %edx
5074; X32-SSE41-NEXT:    sarl $31, %edx
5075; X32-SSE41-NEXT:    pinsrb $11, %edx, %xmm0
5076; X32-SSE41-NEXT:    movl %ecx, %edx
5077; X32-SSE41-NEXT:    shll $19, %edx
5078; X32-SSE41-NEXT:    sarl $31, %edx
5079; X32-SSE41-NEXT:    pinsrb $12, %edx, %xmm0
5080; X32-SSE41-NEXT:    movl %ecx, %edx
5081; X32-SSE41-NEXT:    shll $18, %edx
5082; X32-SSE41-NEXT:    sarl $31, %edx
5083; X32-SSE41-NEXT:    pinsrb $13, %edx, %xmm0
5084; X32-SSE41-NEXT:    movl %ecx, %edx
5085; X32-SSE41-NEXT:    shll $17, %edx
5086; X32-SSE41-NEXT:    sarl $31, %edx
5087; X32-SSE41-NEXT:    pinsrb $14, %edx, %xmm0
5088; X32-SSE41-NEXT:    shrl $15, %ecx
5089; X32-SSE41-NEXT:    pinsrb $15, %ecx, %xmm0
5090; X32-SSE41-NEXT:    movswl 2(%eax), %eax
5091; X32-SSE41-NEXT:    movl %eax, %ecx
5092; X32-SSE41-NEXT:    shll $30, %ecx
5093; X32-SSE41-NEXT:    sarl $31, %ecx
5094; X32-SSE41-NEXT:    movl %eax, %edx
5095; X32-SSE41-NEXT:    shll $31, %edx
5096; X32-SSE41-NEXT:    sarl $31, %edx
5097; X32-SSE41-NEXT:    movd %edx, %xmm1
5098; X32-SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
5099; X32-SSE41-NEXT:    movl %eax, %ecx
5100; X32-SSE41-NEXT:    shll $29, %ecx
5101; X32-SSE41-NEXT:    sarl $31, %ecx
5102; X32-SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
5103; X32-SSE41-NEXT:    movl %eax, %ecx
5104; X32-SSE41-NEXT:    shll $28, %ecx
5105; X32-SSE41-NEXT:    sarl $31, %ecx
5106; X32-SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
5107; X32-SSE41-NEXT:    movl %eax, %ecx
5108; X32-SSE41-NEXT:    shll $27, %ecx
5109; X32-SSE41-NEXT:    sarl $31, %ecx
5110; X32-SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
5111; X32-SSE41-NEXT:    movl %eax, %ecx
5112; X32-SSE41-NEXT:    shll $26, %ecx
5113; X32-SSE41-NEXT:    sarl $31, %ecx
5114; X32-SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
5115; X32-SSE41-NEXT:    movl %eax, %ecx
5116; X32-SSE41-NEXT:    shll $25, %ecx
5117; X32-SSE41-NEXT:    sarl $31, %ecx
5118; X32-SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
5119; X32-SSE41-NEXT:    movsbl %al, %ecx
5120; X32-SSE41-NEXT:    shrl $7, %ecx
5121; X32-SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
5122; X32-SSE41-NEXT:    movl %eax, %ecx
5123; X32-SSE41-NEXT:    shll $23, %ecx
5124; X32-SSE41-NEXT:    sarl $31, %ecx
5125; X32-SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
5126; X32-SSE41-NEXT:    movl %eax, %ecx
5127; X32-SSE41-NEXT:    shll $22, %ecx
5128; X32-SSE41-NEXT:    sarl $31, %ecx
5129; X32-SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
5130; X32-SSE41-NEXT:    movl %eax, %ecx
5131; X32-SSE41-NEXT:    shll $21, %ecx
5132; X32-SSE41-NEXT:    sarl $31, %ecx
5133; X32-SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
5134; X32-SSE41-NEXT:    movl %eax, %ecx
5135; X32-SSE41-NEXT:    shll $20, %ecx
5136; X32-SSE41-NEXT:    sarl $31, %ecx
5137; X32-SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
5138; X32-SSE41-NEXT:    movl %eax, %ecx
5139; X32-SSE41-NEXT:    shll $19, %ecx
5140; X32-SSE41-NEXT:    sarl $31, %ecx
5141; X32-SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
5142; X32-SSE41-NEXT:    movl %eax, %ecx
5143; X32-SSE41-NEXT:    shll $18, %ecx
5144; X32-SSE41-NEXT:    sarl $31, %ecx
5145; X32-SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
5146; X32-SSE41-NEXT:    movl %eax, %ecx
5147; X32-SSE41-NEXT:    shll $17, %ecx
5148; X32-SSE41-NEXT:    sarl $31, %ecx
5149; X32-SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
5150; X32-SSE41-NEXT:    shrl $15, %eax
5151; X32-SSE41-NEXT:    pinsrb $15, %eax, %xmm1
5152; X32-SSE41-NEXT:    popl %esi
5153; X32-SSE41-NEXT:    retl
5154entry:
5155 %X = load <32 x i1>, <32 x i1>* %ptr
5156 %Y = sext <32 x i1> %X to <32 x i8>
5157 ret <32 x i8> %Y
5158}
5159
5160define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
5161; SSE2-LABEL: load_sext_16i8_to_16i16:
5162; SSE2:       # %bb.0: # %entry
5163; SSE2-NEXT:    movdqa (%rdi), %xmm1
5164; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
5165; SSE2-NEXT:    psraw $8, %xmm0
5166; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
5167; SSE2-NEXT:    psraw $8, %xmm1
5168; SSE2-NEXT:    retq
5169;
5170; SSSE3-LABEL: load_sext_16i8_to_16i16:
5171; SSSE3:       # %bb.0: # %entry
5172; SSSE3-NEXT:    movdqa (%rdi), %xmm1
5173; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
5174; SSSE3-NEXT:    psraw $8, %xmm0
5175; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
5176; SSSE3-NEXT:    psraw $8, %xmm1
5177; SSSE3-NEXT:    retq
5178;
5179; SSE41-LABEL: load_sext_16i8_to_16i16:
5180; SSE41:       # %bb.0: # %entry
5181; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
5182; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
5183; SSE41-NEXT:    retq
5184;
5185; AVX1-LABEL: load_sext_16i8_to_16i16:
5186; AVX1:       # %bb.0: # %entry
5187; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm0
5188; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm1
5189; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5190; AVX1-NEXT:    retq
5191;
5192; AVX2-LABEL: load_sext_16i8_to_16i16:
5193; AVX2:       # %bb.0: # %entry
5194; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
5195; AVX2-NEXT:    retq
5196;
5197; AVX512-LABEL: load_sext_16i8_to_16i16:
5198; AVX512:       # %bb.0: # %entry
5199; AVX512-NEXT:    vpmovsxbw (%rdi), %ymm0
5200; AVX512-NEXT:    retq
5201;
5202; X32-SSE2-LABEL: load_sext_16i8_to_16i16:
5203; X32-SSE2:       # %bb.0: # %entry
5204; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5205; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
5206; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
5207; X32-SSE2-NEXT:    psraw $8, %xmm0
5208; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
5209; X32-SSE2-NEXT:    psraw $8, %xmm1
5210; X32-SSE2-NEXT:    retl
5211;
5212; X32-SSE41-LABEL: load_sext_16i8_to_16i16:
5213; X32-SSE41:       # %bb.0: # %entry
5214; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5215; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
5216; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
5217; X32-SSE41-NEXT:    retl
5218entry:
5219 %X = load <16 x i8>, <16 x i8>* %ptr
5220 %Y = sext <16 x i8> %X to <16 x i16>
5221 ret <16 x i16> %Y
5222}
5223
5224define <2 x i64> @load_sext_2i16_to_2i64(<2 x i16> *%ptr) {
5225; SSE2-LABEL: load_sext_2i16_to_2i64:
5226; SSE2:       # %bb.0: # %entry
5227; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5228; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5229; SSE2-NEXT:    pxor %xmm1, %xmm1
5230; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5231; SSE2-NEXT:    psrad $16, %xmm0
5232; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5233; SSE2-NEXT:    retq
5234;
5235; SSSE3-LABEL: load_sext_2i16_to_2i64:
5236; SSSE3:       # %bb.0: # %entry
5237; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5238; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5239; SSSE3-NEXT:    pxor %xmm1, %xmm1
5240; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
5241; SSSE3-NEXT:    psrad $16, %xmm0
5242; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5243; SSSE3-NEXT:    retq
5244;
5245; SSE41-LABEL: load_sext_2i16_to_2i64:
5246; SSE41:       # %bb.0: # %entry
5247; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
5248; SSE41-NEXT:    retq
5249;
5250; AVX-LABEL: load_sext_2i16_to_2i64:
5251; AVX:       # %bb.0: # %entry
5252; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
5253; AVX-NEXT:    retq
5254;
5255; X32-SSE2-LABEL: load_sext_2i16_to_2i64:
5256; X32-SSE2:       # %bb.0: # %entry
5257; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5258; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
5259; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
5260; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
5261; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5262; X32-SSE2-NEXT:    psrad $16, %xmm0
5263; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5264; X32-SSE2-NEXT:    retl
5265;
5266; X32-SSE41-LABEL: load_sext_2i16_to_2i64:
5267; X32-SSE41:       # %bb.0: # %entry
5268; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5269; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
5270; X32-SSE41-NEXT:    retl
5271entry:
5272 %X = load <2 x i16>, <2 x i16>* %ptr
5273 %Y = sext <2 x i16> %X to <2 x i64>
5274 ret <2 x i64> %Y
5275}
5276
5277define <4 x i32> @load_sext_4i16_to_4i32(<4 x i16> *%ptr) {
5278; SSE2-LABEL: load_sext_4i16_to_4i32:
5279; SSE2:       # %bb.0: # %entry
5280; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5281; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5282; SSE2-NEXT:    psrad $16, %xmm0
5283; SSE2-NEXT:    retq
5284;
5285; SSSE3-LABEL: load_sext_4i16_to_4i32:
5286; SSSE3:       # %bb.0: # %entry
5287; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5288; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5289; SSSE3-NEXT:    psrad $16, %xmm0
5290; SSSE3-NEXT:    retq
5291;
5292; SSE41-LABEL: load_sext_4i16_to_4i32:
5293; SSE41:       # %bb.0: # %entry
5294; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
5295; SSE41-NEXT:    retq
5296;
5297; AVX-LABEL: load_sext_4i16_to_4i32:
5298; AVX:       # %bb.0: # %entry
5299; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
5300; AVX-NEXT:    retq
5301;
5302; X32-SSE2-LABEL: load_sext_4i16_to_4i32:
5303; X32-SSE2:       # %bb.0: # %entry
5304; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5305; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5306; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5307; X32-SSE2-NEXT:    psrad $16, %xmm0
5308; X32-SSE2-NEXT:    retl
5309;
5310; X32-SSE41-LABEL: load_sext_4i16_to_4i32:
5311; X32-SSE41:       # %bb.0: # %entry
5312; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5313; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
5314; X32-SSE41-NEXT:    retl
5315entry:
5316 %X = load <4 x i16>, <4 x i16>* %ptr
5317 %Y = sext <4 x i16> %X to <4 x i32>
5318 ret <4 x i32> %Y
5319}
5320
5321define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
5322; SSE2-LABEL: load_sext_4i16_to_4i64:
5323; SSE2:       # %bb.0: # %entry
5324; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5325; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5326; SSE2-NEXT:    psrad $16, %xmm1
5327; SSE2-NEXT:    pxor %xmm2, %xmm2
5328; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5329; SSE2-NEXT:    movdqa %xmm1, %xmm0
5330; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
5331; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
5332; SSE2-NEXT:    retq
5333;
5334; SSSE3-LABEL: load_sext_4i16_to_4i64:
5335; SSSE3:       # %bb.0: # %entry
5336; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5337; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5338; SSSE3-NEXT:    psrad $16, %xmm1
5339; SSSE3-NEXT:    pxor %xmm2, %xmm2
5340; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5341; SSSE3-NEXT:    movdqa %xmm1, %xmm0
5342; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
5343; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
5344; SSSE3-NEXT:    retq
5345;
5346; SSE41-LABEL: load_sext_4i16_to_4i64:
5347; SSE41:       # %bb.0: # %entry
5348; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
5349; SSE41-NEXT:    pmovsxwq 4(%rdi), %xmm1
5350; SSE41-NEXT:    retq
5351;
5352; AVX1-LABEL: load_sext_4i16_to_4i64:
5353; AVX1:       # %bb.0: # %entry
5354; AVX1-NEXT:    vpmovsxwq 4(%rdi), %xmm0
5355; AVX1-NEXT:    vpmovsxwq (%rdi), %xmm1
5356; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5357; AVX1-NEXT:    retq
5358;
5359; AVX2-LABEL: load_sext_4i16_to_4i64:
5360; AVX2:       # %bb.0: # %entry
5361; AVX2-NEXT:    vpmovsxwq (%rdi), %ymm0
5362; AVX2-NEXT:    retq
5363;
5364; AVX512-LABEL: load_sext_4i16_to_4i64:
5365; AVX512:       # %bb.0: # %entry
5366; AVX512-NEXT:    vpmovsxwq (%rdi), %ymm0
5367; AVX512-NEXT:    retq
5368;
5369; X32-SSE2-LABEL: load_sext_4i16_to_4i64:
5370; X32-SSE2:       # %bb.0: # %entry
5371; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5372; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5373; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5374; X32-SSE2-NEXT:    psrad $16, %xmm1
5375; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5376; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5377; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
5378; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
5379; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
5380; X32-SSE2-NEXT:    retl
5381;
5382; X32-SSE41-LABEL: load_sext_4i16_to_4i64:
5383; X32-SSE41:       # %bb.0: # %entry
5384; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5385; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
5386; X32-SSE41-NEXT:    pmovsxwq 4(%eax), %xmm1
5387; X32-SSE41-NEXT:    retl
5388entry:
5389 %X = load <4 x i16>, <4 x i16>* %ptr
5390 %Y = sext <4 x i16> %X to <4 x i64>
5391 ret <4 x i64> %Y
5392}
5393
5394define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
5395; SSE2-LABEL: load_sext_8i16_to_8i32:
5396; SSE2:       # %bb.0: # %entry
5397; SSE2-NEXT:    movdqa (%rdi), %xmm1
5398; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5399; SSE2-NEXT:    psrad $16, %xmm0
5400; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
5401; SSE2-NEXT:    psrad $16, %xmm1
5402; SSE2-NEXT:    retq
5403;
5404; SSSE3-LABEL: load_sext_8i16_to_8i32:
5405; SSSE3:       # %bb.0: # %entry
5406; SSSE3-NEXT:    movdqa (%rdi), %xmm1
5407; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5408; SSSE3-NEXT:    psrad $16, %xmm0
5409; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
5410; SSSE3-NEXT:    psrad $16, %xmm1
5411; SSSE3-NEXT:    retq
5412;
5413; SSE41-LABEL: load_sext_8i16_to_8i32:
5414; SSE41:       # %bb.0: # %entry
5415; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
5416; SSE41-NEXT:    pmovsxwd 8(%rdi), %xmm1
5417; SSE41-NEXT:    retq
5418;
5419; AVX1-LABEL: load_sext_8i16_to_8i32:
5420; AVX1:       # %bb.0: # %entry
5421; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm0
5422; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm1
5423; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5424; AVX1-NEXT:    retq
5425;
5426; AVX2-LABEL: load_sext_8i16_to_8i32:
5427; AVX2:       # %bb.0: # %entry
5428; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm0
5429; AVX2-NEXT:    retq
5430;
5431; AVX512-LABEL: load_sext_8i16_to_8i32:
5432; AVX512:       # %bb.0: # %entry
5433; AVX512-NEXT:    vpmovsxwd (%rdi), %ymm0
5434; AVX512-NEXT:    retq
5435;
5436; X32-SSE2-LABEL: load_sext_8i16_to_8i32:
5437; X32-SSE2:       # %bb.0: # %entry
5438; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5439; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
5440; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5441; X32-SSE2-NEXT:    psrad $16, %xmm0
5442; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
5443; X32-SSE2-NEXT:    psrad $16, %xmm1
5444; X32-SSE2-NEXT:    retl
5445;
5446; X32-SSE41-LABEL: load_sext_8i16_to_8i32:
5447; X32-SSE41:       # %bb.0: # %entry
5448; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5449; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
5450; X32-SSE41-NEXT:    pmovsxwd 8(%eax), %xmm1
5451; X32-SSE41-NEXT:    retl
5452entry:
5453 %X = load <8 x i16>, <8 x i16>* %ptr
5454 %Y = sext <8 x i16> %X to <8 x i32>
5455 ret <8 x i32> %Y
5456}
5457
5458define <2 x i64> @load_sext_2i32_to_2i64(<2 x i32> *%ptr) {
5459; SSE2-LABEL: load_sext_2i32_to_2i64:
5460; SSE2:       # %bb.0: # %entry
5461; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5462; SSE2-NEXT:    pxor %xmm1, %xmm1
5463; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5464; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5465; SSE2-NEXT:    retq
5466;
5467; SSSE3-LABEL: load_sext_2i32_to_2i64:
5468; SSSE3:       # %bb.0: # %entry
5469; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5470; SSSE3-NEXT:    pxor %xmm1, %xmm1
5471; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
5472; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5473; SSSE3-NEXT:    retq
5474;
5475; SSE41-LABEL: load_sext_2i32_to_2i64:
5476; SSE41:       # %bb.0: # %entry
5477; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
5478; SSE41-NEXT:    retq
5479;
5480; AVX-LABEL: load_sext_2i32_to_2i64:
5481; AVX:       # %bb.0: # %entry
5482; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
5483; AVX-NEXT:    retq
5484;
5485; X32-SSE2-LABEL: load_sext_2i32_to_2i64:
5486; X32-SSE2:       # %bb.0: # %entry
5487; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5488; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
5489; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
5490; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
5491; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
5492; X32-SSE2-NEXT:    retl
5493;
5494; X32-SSE41-LABEL: load_sext_2i32_to_2i64:
5495; X32-SSE41:       # %bb.0: # %entry
5496; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5497; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
5498; X32-SSE41-NEXT:    retl
5499entry:
5500 %X = load <2 x i32>, <2 x i32>* %ptr
5501 %Y = sext <2 x i32> %X to <2 x i64>
5502 ret <2 x i64> %Y
5503}
5504
5505define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
5506; SSE2-LABEL: load_sext_4i32_to_4i64:
5507; SSE2:       # %bb.0: # %entry
5508; SSE2-NEXT:    movdqa (%rdi), %xmm0
5509; SSE2-NEXT:    pxor %xmm2, %xmm2
5510; SSE2-NEXT:    pxor %xmm3, %xmm3
5511; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5512; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5513; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5514; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5515; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5516; SSE2-NEXT:    retq
5517;
5518; SSSE3-LABEL: load_sext_4i32_to_4i64:
5519; SSSE3:       # %bb.0: # %entry
5520; SSSE3-NEXT:    movdqa (%rdi), %xmm0
5521; SSSE3-NEXT:    pxor %xmm2, %xmm2
5522; SSSE3-NEXT:    pxor %xmm3, %xmm3
5523; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5524; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5525; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5526; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5527; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5528; SSSE3-NEXT:    retq
5529;
5530; SSE41-LABEL: load_sext_4i32_to_4i64:
5531; SSE41:       # %bb.0: # %entry
5532; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
5533; SSE41-NEXT:    pmovsxdq 8(%rdi), %xmm1
5534; SSE41-NEXT:    retq
5535;
5536; AVX1-LABEL: load_sext_4i32_to_4i64:
5537; AVX1:       # %bb.0: # %entry
5538; AVX1-NEXT:    vpmovsxdq 8(%rdi), %xmm0
5539; AVX1-NEXT:    vpmovsxdq (%rdi), %xmm1
5540; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5541; AVX1-NEXT:    retq
5542;
5543; AVX2-LABEL: load_sext_4i32_to_4i64:
5544; AVX2:       # %bb.0: # %entry
5545; AVX2-NEXT:    vpmovsxdq (%rdi), %ymm0
5546; AVX2-NEXT:    retq
5547;
5548; AVX512-LABEL: load_sext_4i32_to_4i64:
5549; AVX512:       # %bb.0: # %entry
5550; AVX512-NEXT:    vpmovsxdq (%rdi), %ymm0
5551; AVX512-NEXT:    retq
5552;
5553; X32-SSE2-LABEL: load_sext_4i32_to_4i64:
5554; X32-SSE2:       # %bb.0: # %entry
5555; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5556; X32-SSE2-NEXT:    movdqa (%eax), %xmm0
5557; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5558; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5559; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5560; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5561; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5562; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5563; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5564; X32-SSE2-NEXT:    retl
5565;
5566; X32-SSE41-LABEL: load_sext_4i32_to_4i64:
5567; X32-SSE41:       # %bb.0: # %entry
5568; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5569; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
5570; X32-SSE41-NEXT:    pmovsxdq 8(%eax), %xmm1
5571; X32-SSE41-NEXT:    retl
5572entry:
5573 %X = load <4 x i32>, <4 x i32>* %ptr
5574 %Y = sext <4 x i32> %X to <4 x i64>
5575 ret <4 x i64> %Y
5576}
5577
5578define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
5579; SSE2-LABEL: sext_2i8_to_i32:
5580; SSE2:       # %bb.0: # %entry
5581; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5582; SSE2-NEXT:    psraw $8, %xmm0
5583; SSE2-NEXT:    movd %xmm0, %eax
5584; SSE2-NEXT:    retq
5585;
5586; SSSE3-LABEL: sext_2i8_to_i32:
5587; SSSE3:       # %bb.0: # %entry
5588; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5589; SSSE3-NEXT:    psraw $8, %xmm0
5590; SSSE3-NEXT:    movd %xmm0, %eax
5591; SSSE3-NEXT:    retq
5592;
5593; SSE41-LABEL: sext_2i8_to_i32:
5594; SSE41:       # %bb.0: # %entry
5595; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
5596; SSE41-NEXT:    movd %xmm0, %eax
5597; SSE41-NEXT:    retq
5598;
5599; AVX-LABEL: sext_2i8_to_i32:
5600; AVX:       # %bb.0: # %entry
5601; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
5602; AVX-NEXT:    vmovd %xmm0, %eax
5603; AVX-NEXT:    retq
5604;
5605; X32-SSE2-LABEL: sext_2i8_to_i32:
5606; X32-SSE2:       # %bb.0: # %entry
5607; X32-SSE2-NEXT:    pushl %eax
5608; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
5609; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5610; X32-SSE2-NEXT:    psraw $8, %xmm0
5611; X32-SSE2-NEXT:    movd %xmm0, %eax
5612; X32-SSE2-NEXT:    popl %ecx
5613; X32-SSE2-NEXT:    .cfi_def_cfa_offset 4
5614; X32-SSE2-NEXT:    retl
5615;
5616; X32-SSE41-LABEL: sext_2i8_to_i32:
5617; X32-SSE41:       # %bb.0: # %entry
5618; X32-SSE41-NEXT:    pushl %eax
5619; X32-SSE41-NEXT:    .cfi_def_cfa_offset 8
5620; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
5621; X32-SSE41-NEXT:    movd %xmm0, %eax
5622; X32-SSE41-NEXT:    popl %ecx
5623; X32-SSE41-NEXT:    .cfi_def_cfa_offset 4
5624; X32-SSE41-NEXT:    retl
5625entry:
5626  %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
5627  %Ex = sext <2 x i8> %Shuf to <2 x i16>
5628  %Bc = bitcast <2 x i16> %Ex to i32
5629  ret i32 %Bc
5630}
5631
5632define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
5633; SSE2-LABEL: sext_4i1_to_4i64:
5634; SSE2:       # %bb.0:
5635; SSE2-NEXT:    pslld $31, %xmm0
5636; SSE2-NEXT:    psrad $31, %xmm0
5637; SSE2-NEXT:    pxor %xmm2, %xmm2
5638; SSE2-NEXT:    pxor %xmm3, %xmm3
5639; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5640; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5641; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5642; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5643; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5644; SSE2-NEXT:    retq
5645;
5646; SSSE3-LABEL: sext_4i1_to_4i64:
5647; SSSE3:       # %bb.0:
5648; SSSE3-NEXT:    pslld $31, %xmm0
5649; SSSE3-NEXT:    psrad $31, %xmm0
5650; SSSE3-NEXT:    pxor %xmm2, %xmm2
5651; SSSE3-NEXT:    pxor %xmm3, %xmm3
5652; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5653; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5654; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5655; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5656; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5657; SSSE3-NEXT:    retq
5658;
5659; SSE41-LABEL: sext_4i1_to_4i64:
5660; SSE41:       # %bb.0:
5661; SSE41-NEXT:    pslld $31, %xmm0
5662; SSE41-NEXT:    psrad $31, %xmm0
5663; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5664; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5665; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5666; SSE41-NEXT:    movdqa %xmm2, %xmm0
5667; SSE41-NEXT:    retq
5668;
5669; AVX1-LABEL: sext_4i1_to_4i64:
5670; AVX1:       # %bb.0:
5671; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
5672; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
5673; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
5674; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5675; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
5676; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5677; AVX1-NEXT:    retq
5678;
5679; AVX2-LABEL: sext_4i1_to_4i64:
5680; AVX2:       # %bb.0:
5681; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
5682; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
5683; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
5684; AVX2-NEXT:    retq
5685;
5686; AVX512-LABEL: sext_4i1_to_4i64:
5687; AVX512:       # %bb.0:
5688; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
5689; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm0
5690; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
5691; AVX512-NEXT:    retq
5692;
5693; X32-SSE2-LABEL: sext_4i1_to_4i64:
5694; X32-SSE2:       # %bb.0:
5695; X32-SSE2-NEXT:    pslld $31, %xmm0
5696; X32-SSE2-NEXT:    psrad $31, %xmm0
5697; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5698; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5699; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5700; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5701; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5702; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5703; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5704; X32-SSE2-NEXT:    retl
5705;
5706; X32-SSE41-LABEL: sext_4i1_to_4i64:
5707; X32-SSE41:       # %bb.0:
5708; X32-SSE41-NEXT:    pslld $31, %xmm0
5709; X32-SSE41-NEXT:    psrad $31, %xmm0
5710; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5711; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5712; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5713; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
5714; X32-SSE41-NEXT:    retl
5715  %extmask = sext <4 x i1> %mask to <4 x i64>
5716  ret <4 x i64> %extmask
5717}
5718
5719define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
5720; SSE2-LABEL: sext_4i8_to_4i64:
5721; SSE2:       # %bb.0:
5722; SSE2-NEXT:    pslld $24, %xmm0
5723; SSE2-NEXT:    psrad $24, %xmm0
5724; SSE2-NEXT:    pxor %xmm2, %xmm2
5725; SSE2-NEXT:    pxor %xmm3, %xmm3
5726; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5727; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5728; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5729; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5730; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5731; SSE2-NEXT:    retq
5732;
5733; SSSE3-LABEL: sext_4i8_to_4i64:
5734; SSSE3:       # %bb.0:
5735; SSSE3-NEXT:    pslld $24, %xmm0
5736; SSSE3-NEXT:    psrad $24, %xmm0
5737; SSSE3-NEXT:    pxor %xmm2, %xmm2
5738; SSSE3-NEXT:    pxor %xmm3, %xmm3
5739; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
5740; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5741; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5742; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
5743; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5744; SSSE3-NEXT:    retq
5745;
5746; SSE41-LABEL: sext_4i8_to_4i64:
5747; SSE41:       # %bb.0:
5748; SSE41-NEXT:    pslld $24, %xmm0
5749; SSE41-NEXT:    psrad $24, %xmm0
5750; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5751; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5752; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5753; SSE41-NEXT:    movdqa %xmm2, %xmm0
5754; SSE41-NEXT:    retq
5755;
5756; AVX1-LABEL: sext_4i8_to_4i64:
5757; AVX1:       # %bb.0:
5758; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
5759; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
5760; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
5761; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5762; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
5763; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
5764; AVX1-NEXT:    retq
5765;
5766; AVX2-LABEL: sext_4i8_to_4i64:
5767; AVX2:       # %bb.0:
5768; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
5769; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
5770; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
5771; AVX2-NEXT:    retq
5772;
5773; AVX512-LABEL: sext_4i8_to_4i64:
5774; AVX512:       # %bb.0:
5775; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
5776; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
5777; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
5778; AVX512-NEXT:    retq
5779;
5780; X32-SSE2-LABEL: sext_4i8_to_4i64:
5781; X32-SSE2:       # %bb.0:
5782; X32-SSE2-NEXT:    pslld $24, %xmm0
5783; X32-SSE2-NEXT:    psrad $24, %xmm0
5784; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
5785; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
5786; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
5787; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
5788; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
5789; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
5790; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
5791; X32-SSE2-NEXT:    retl
5792;
5793; X32-SSE41-LABEL: sext_4i8_to_4i64:
5794; X32-SSE41:       # %bb.0:
5795; X32-SSE41-NEXT:    pslld $24, %xmm0
5796; X32-SSE41-NEXT:    psrad $24, %xmm0
5797; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
5798; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
5799; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
5800; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
5801; X32-SSE41-NEXT:    retl
5802  %extmask = sext <4 x i8> %mask to <4 x i64>
5803  ret <4 x i64> %extmask
5804}
5805
5806define <32 x i8> @sext_32xi1_to_32xi8(<32 x i16> %c1, <32 x i16> %c2)nounwind {
5807; SSE-LABEL: sext_32xi1_to_32xi8:
5808; SSE:       # %bb.0:
5809; SSE-NEXT:    pcmpeqw %xmm5, %xmm1
5810; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
5811; SSE-NEXT:    packsswb %xmm1, %xmm0
5812; SSE-NEXT:    pcmpeqw %xmm7, %xmm3
5813; SSE-NEXT:    pcmpeqw %xmm6, %xmm2
5814; SSE-NEXT:    packsswb %xmm3, %xmm2
5815; SSE-NEXT:    movdqa %xmm2, %xmm1
5816; SSE-NEXT:    retq
5817;
5818; AVX1-LABEL: sext_32xi1_to_32xi8:
5819; AVX1:       # %bb.0:
5820; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
5821; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
5822; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm5, %xmm4
5823; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
5824; AVX1-NEXT:    vpacksswb %xmm4, %xmm1, %xmm1
5825; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
5826; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
5827; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm4, %xmm3
5828; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
5829; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
5830; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
5831; AVX1-NEXT:    retq
5832;
5833; AVX2-LABEL: sext_32xi1_to_32xi8:
5834; AVX2:       # %bb.0:
5835; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
5836; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
5837; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
5838; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
5839; AVX2-NEXT:    retq
5840;
5841; AVX512F-LABEL: sext_32xi1_to_32xi8:
5842; AVX512F:       # %bb.0:
5843; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
5844; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
5845; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
5846; AVX512F-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
5847; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
5848; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
5849; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
5850; AVX512F-NEXT:    retq
5851;
5852; AVX512BW-LABEL: sext_32xi1_to_32xi8:
5853; AVX512BW:       # %bb.0:
5854; AVX512BW-NEXT:    vpcmpeqw %zmm1, %zmm0, %k0
5855; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
5856; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
5857; AVX512BW-NEXT:    retq
5858;
5859; X32-SSE2-LABEL: sext_32xi1_to_32xi8:
5860; X32-SSE2:       # %bb.0:
5861; X32-SSE2-NEXT:    pushl %ebp
5862; X32-SSE2-NEXT:    movl %esp, %ebp
5863; X32-SSE2-NEXT:    andl $-16, %esp
5864; X32-SSE2-NEXT:    subl $16, %esp
5865; X32-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
5866; X32-SSE2-NEXT:    pcmpeqw 40(%ebp), %xmm1
5867; X32-SSE2-NEXT:    pcmpeqw 24(%ebp), %xmm0
5868; X32-SSE2-NEXT:    packsswb %xmm1, %xmm0
5869; X32-SSE2-NEXT:    pcmpeqw 72(%ebp), %xmm3
5870; X32-SSE2-NEXT:    pcmpeqw 56(%ebp), %xmm2
5871; X32-SSE2-NEXT:    packsswb %xmm3, %xmm2
5872; X32-SSE2-NEXT:    movdqa %xmm2, %xmm1
5873; X32-SSE2-NEXT:    movl %ebp, %esp
5874; X32-SSE2-NEXT:    popl %ebp
5875; X32-SSE2-NEXT:    retl
5876;
5877; X32-SSE41-LABEL: sext_32xi1_to_32xi8:
5878; X32-SSE41:       # %bb.0:
5879; X32-SSE41-NEXT:    pushl %ebp
5880; X32-SSE41-NEXT:    movl %esp, %ebp
5881; X32-SSE41-NEXT:    andl $-16, %esp
5882; X32-SSE41-NEXT:    subl $16, %esp
5883; X32-SSE41-NEXT:    movdqa 8(%ebp), %xmm3
5884; X32-SSE41-NEXT:    pcmpeqw 40(%ebp), %xmm1
5885; X32-SSE41-NEXT:    pcmpeqw 24(%ebp), %xmm0
5886; X32-SSE41-NEXT:    packsswb %xmm1, %xmm0
5887; X32-SSE41-NEXT:    pcmpeqw 72(%ebp), %xmm3
5888; X32-SSE41-NEXT:    pcmpeqw 56(%ebp), %xmm2
5889; X32-SSE41-NEXT:    packsswb %xmm3, %xmm2
5890; X32-SSE41-NEXT:    movdqa %xmm2, %xmm1
5891; X32-SSE41-NEXT:    movl %ebp, %esp
5892; X32-SSE41-NEXT:    popl %ebp
5893; X32-SSE41-NEXT:    retl
5894  %a = icmp eq <32 x i16> %c1, %c2
5895  %b = sext <32 x i1> %a to <32 x i8>
5896  ret <32 x i8> %b
5897}
5898
5899define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
5900; SSE2-LABEL: sext_2i8_to_2i32:
5901; SSE2:       # %bb.0:
5902; SSE2-NEXT:    movzwl (%rdi), %eax
5903; SSE2-NEXT:    movd %eax, %xmm0
5904; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5905; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5906; SSE2-NEXT:    psrad $24, %xmm0
5907; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
5908; SSE2-NEXT:    paddq %xmm0, %xmm0
5909; SSE2-NEXT:    retq
5910;
5911; SSSE3-LABEL: sext_2i8_to_2i32:
5912; SSSE3:       # %bb.0:
5913; SSSE3-NEXT:    movzwl (%rdi), %eax
5914; SSSE3-NEXT:    movd %eax, %xmm0
5915; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5916; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5917; SSSE3-NEXT:    psrad $24, %xmm0
5918; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
5919; SSSE3-NEXT:    paddq %xmm0, %xmm0
5920; SSSE3-NEXT:    retq
5921;
5922; SSE41-LABEL: sext_2i8_to_2i32:
5923; SSE41:       # %bb.0:
5924; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
5925; SSE41-NEXT:    paddq %xmm0, %xmm0
5926; SSE41-NEXT:    retq
5927;
5928; AVX-LABEL: sext_2i8_to_2i32:
5929; AVX:       # %bb.0:
5930; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
5931; AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
5932; AVX-NEXT:    retq
5933;
5934; X32-SSE2-LABEL: sext_2i8_to_2i32:
5935; X32-SSE2:       # %bb.0:
5936; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
5937; X32-SSE2-NEXT:    movzwl (%eax), %eax
5938; X32-SSE2-NEXT:    movd %eax, %xmm0
5939; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
5940; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
5941; X32-SSE2-NEXT:    psrad $24, %xmm0
5942; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
5943; X32-SSE2-NEXT:    paddq %xmm0, %xmm0
5944; X32-SSE2-NEXT:    retl
5945;
5946; X32-SSE41-LABEL: sext_2i8_to_2i32:
5947; X32-SSE41:       # %bb.0:
5948; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
5949; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
5950; X32-SSE41-NEXT:    paddq %xmm0, %xmm0
5951; X32-SSE41-NEXT:    retl
5952  %x = load <2 x i8>, <2 x i8>* %addr, align 1
5953  %y = sext <2 x i8> %x to <2 x i32>
5954  %z = add <2 x i32>%y, %y
5955  ret <2 x i32>%z
5956}
5957
5958