1; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
6;
7; Just one 32-bit run to make sure we do reasonable things there.
8; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=i686 -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41
9
10define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
11; SSE2-LABEL: sext_8i16_to_8i32:
12; SSE2:       # BB#0: # %entry
13; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
14; SSE2-NEXT:    psrad $16, %xmm2
15; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
16; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
17; SSE2-NEXT:    psrad $16, %xmm1
18; SSE2-NEXT:    movdqa %xmm2, %xmm0
19; SSE2-NEXT:    retq
20;
21; SSSE3-LABEL: sext_8i16_to_8i32:
22; SSSE3:       # BB#0: # %entry
23; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
24; SSSE3-NEXT:    psrad $16, %xmm2
25; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
26; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
27; SSSE3-NEXT:    psrad $16, %xmm1
28; SSSE3-NEXT:    movdqa %xmm2, %xmm0
29; SSSE3-NEXT:    retq
30;
31; SSE41-LABEL: sext_8i16_to_8i32:
32; SSE41:       # BB#0: # %entry
33; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
34; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
35; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
36; SSE41-NEXT:    movdqa %xmm2, %xmm0
37; SSE41-NEXT:    retq
38;
39; AVX1-LABEL: sext_8i16_to_8i32:
40; AVX1:       # BB#0: # %entry
41; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
42; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
43; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
44; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
45; AVX1-NEXT:    retq
46;
47; AVX2-LABEL: sext_8i16_to_8i32:
48; AVX2:       # BB#0: # %entry
49; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
50; AVX2-NEXT:    retq
51;
52; X32-SSE41-LABEL: sext_8i16_to_8i32:
53; X32-SSE41:       # BB#0: # %entry
54; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
55; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
56; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
57; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
58; X32-SSE41-NEXT:    retl
59entry:
60  %B = sext <8 x i16> %A to <8 x i32>
61  ret <8 x i32>%B
62}
63
64define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
65; SSE2-LABEL: sext_4i32_to_4i64:
66; SSE2:       # BB#0: # %entry
67; SSE2-NEXT:    movdqa %xmm0, %xmm2
68; SSE2-NEXT:    psrad $31, %xmm2
69; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
70; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
71; SSE2-NEXT:    movdqa %xmm1, %xmm2
72; SSE2-NEXT:    psrad $31, %xmm2
73; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
74; SSE2-NEXT:    retq
75;
76; SSSE3-LABEL: sext_4i32_to_4i64:
77; SSSE3:       # BB#0: # %entry
78; SSSE3-NEXT:    movdqa %xmm0, %xmm2
79; SSSE3-NEXT:    psrad $31, %xmm2
80; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
81; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
82; SSSE3-NEXT:    movdqa %xmm1, %xmm2
83; SSSE3-NEXT:    psrad $31, %xmm2
84; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
85; SSSE3-NEXT:    retq
86;
87; SSE41-LABEL: sext_4i32_to_4i64:
88; SSE41:       # BB#0: # %entry
89; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
90; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
91; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
92; SSE41-NEXT:    movdqa %xmm2, %xmm0
93; SSE41-NEXT:    retq
94;
95; AVX1-LABEL: sext_4i32_to_4i64:
96; AVX1:       # BB#0: # %entry
97; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
98; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
99; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
100; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
101; AVX1-NEXT:    retq
102;
103; AVX2-LABEL: sext_4i32_to_4i64:
104; AVX2:       # BB#0: # %entry
105; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
106; AVX2-NEXT:    retq
107;
108; X32-SSE41-LABEL: sext_4i32_to_4i64:
109; X32-SSE41:       # BB#0: # %entry
110; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
111; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
112; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
113; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
114; X32-SSE41-NEXT:    retl
115entry:
116  %B = sext <4 x i32> %A to <4 x i64>
117  ret <4 x i64>%B
118}
119
120define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
121; SSE2-LABEL: sext_2i8_to_i32:
122; SSE2:       # BB#0: # %entry
123; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
124; SSE2-NEXT:    psraw $8, %xmm0
125; SSE2-NEXT:    movd %xmm0, %eax
126; SSE2-NEXT:    retq
127;
128; SSSE3-LABEL: sext_2i8_to_i32:
129; SSSE3:       # BB#0: # %entry
130; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
131; SSSE3-NEXT:    psraw $8, %xmm0
132; SSSE3-NEXT:    movd %xmm0, %eax
133; SSSE3-NEXT:    retq
134;
135; SSE41-LABEL: sext_2i8_to_i32:
136; SSE41:       # BB#0: # %entry
137; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
138; SSE41-NEXT:    movd %xmm0, %eax
139; SSE41-NEXT:    retq
140;
141; AVX-LABEL: sext_2i8_to_i32:
142; AVX:       # BB#0: # %entry
143; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
144; AVX-NEXT:    vmovd %xmm0, %eax
145; AVX-NEXT:    retq
146;
147; X32-SSE41-LABEL: sext_2i8_to_i32:
148; X32-SSE41:       # BB#0: # %entry
149; X32-SSE41:         pmovsxbw %xmm0, %xmm0
150; X32-SSE41-NEXT:    movd %xmm0, %eax
151; X32-SSE41-NEXT:    popl %edx
152; X32-SSE41-NEXT:    retl
153entry:
154  %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
155  %Ex = sext <2 x i8> %Shuf to <2 x i16>
156  %Bc = bitcast <2 x i16> %Ex to i32
157  ret i32 %Bc
158}
159
160define <4 x i32> @load_sext_test1(<4 x i16> *%ptr) {
161; SSE2-LABEL: load_sext_test1:
162; SSE2:       # BB#0: # %entry
163; SSE2-NEXT:    movq (%rdi), %xmm0
164; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
165; SSE2-NEXT:    psrad $16, %xmm0
166; SSE2-NEXT:    retq
167;
168; SSSE3-LABEL: load_sext_test1:
169; SSSE3:       # BB#0: # %entry
170; SSSE3-NEXT:    movq (%rdi), %xmm0
171; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
172; SSSE3-NEXT:    psrad $16, %xmm0
173; SSSE3-NEXT:    retq
174;
175; SSE41-LABEL: load_sext_test1:
176; SSE41:       # BB#0: # %entry
177; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
178; SSE41-NEXT:    retq
179;
180; AVX-LABEL: load_sext_test1:
181; AVX:       # BB#0: # %entry
182; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
183; AVX-NEXT:    retq
184;
185; X32-SSE41-LABEL: load_sext_test1:
186; X32-SSE41:       # BB#0: # %entry
187; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
188; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
189; X32-SSE41-NEXT:    retl
190entry:
191 %X = load <4 x i16>, <4 x i16>* %ptr
192 %Y = sext <4 x i16> %X to <4 x i32>
193 ret <4 x i32>%Y
194}
195
196define <4 x i32> @load_sext_test2(<4 x i8> *%ptr) {
197; SSE2-LABEL: load_sext_test2:
198; SSE2:       # BB#0: # %entry
199; SSE2-NEXT:    movd (%rdi), %xmm0
200; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
201; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
202; SSE2-NEXT:    psrad $24, %xmm0
203; SSE2-NEXT:    retq
204;
205; SSSE3-LABEL: load_sext_test2:
206; SSSE3:       # BB#0: # %entry
207; SSSE3-NEXT:    movd (%rdi), %xmm0
208; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
209; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
210; SSSE3-NEXT:    psrad $24, %xmm0
211; SSSE3-NEXT:    retq
212;
213; SSE41-LABEL: load_sext_test2:
214; SSE41:       # BB#0: # %entry
215; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
216; SSE41-NEXT:    retq
217;
218; AVX-LABEL: load_sext_test2:
219; AVX:       # BB#0: # %entry
220; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
221; AVX-NEXT:    retq
222;
223; X32-SSE41-LABEL: load_sext_test2:
224; X32-SSE41:       # BB#0: # %entry
225; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
226; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
227; X32-SSE41-NEXT:    retl
228entry:
229 %X = load <4 x i8>, <4 x i8>* %ptr
230 %Y = sext <4 x i8> %X to <4 x i32>
231 ret <4 x i32>%Y
232}
233
234define <2 x i64> @load_sext_test3(<2 x i8> *%ptr) {
235; SSE2-LABEL: load_sext_test3:
236; SSE2:       # BB#0: # %entry
237; SSE2-NEXT:    movzwl (%rdi), %eax
238; SSE2-NEXT:    movd %eax, %xmm0
239; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
240; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
241; SSE2-NEXT:    movdqa %xmm0, %xmm1
242; SSE2-NEXT:    psrad $31, %xmm1
243; SSE2-NEXT:    psrad $24, %xmm0
244; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
245; SSE2-NEXT:    retq
246;
247; SSSE3-LABEL: load_sext_test3:
248; SSSE3:       # BB#0: # %entry
249; SSSE3-NEXT:    movzwl (%rdi), %eax
250; SSSE3-NEXT:    movd %eax, %xmm0
251; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
252; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
253; SSSE3-NEXT:    movdqa %xmm0, %xmm1
254; SSSE3-NEXT:    psrad $31, %xmm1
255; SSSE3-NEXT:    psrad $24, %xmm0
256; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
257; SSSE3-NEXT:    retq
258;
259; SSE41-LABEL: load_sext_test3:
260; SSE41:       # BB#0: # %entry
261; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
262; SSE41-NEXT:    retq
263;
264; AVX-LABEL: load_sext_test3:
265; AVX:       # BB#0: # %entry
266; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
267; AVX-NEXT:    retq
268;
269; X32-SSE41-LABEL: load_sext_test3:
270; X32-SSE41:       # BB#0: # %entry
271; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
272; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
273; X32-SSE41-NEXT:    retl
274entry:
275 %X = load <2 x i8>, <2 x i8>* %ptr
276 %Y = sext <2 x i8> %X to <2 x i64>
277 ret <2 x i64>%Y
278}
279
280define <2 x i64> @load_sext_test4(<2 x i16> *%ptr) {
281; SSE2-LABEL: load_sext_test4:
282; SSE2:       # BB#0: # %entry
283; SSE2-NEXT:    movd (%rdi), %xmm0
284; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
285; SSE2-NEXT:    movdqa %xmm0, %xmm1
286; SSE2-NEXT:    psrad $31, %xmm1
287; SSE2-NEXT:    psrad $16, %xmm0
288; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
289; SSE2-NEXT:    retq
290;
291; SSSE3-LABEL: load_sext_test4:
292; SSSE3:       # BB#0: # %entry
293; SSSE3-NEXT:    movd (%rdi), %xmm0
294; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
295; SSSE3-NEXT:    movdqa %xmm0, %xmm1
296; SSSE3-NEXT:    psrad $31, %xmm1
297; SSSE3-NEXT:    psrad $16, %xmm0
298; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
299; SSSE3-NEXT:    retq
300;
301; SSE41-LABEL: load_sext_test4:
302; SSE41:       # BB#0: # %entry
303; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
304; SSE41-NEXT:    retq
305;
306; AVX-LABEL: load_sext_test4:
307; AVX:       # BB#0: # %entry
308; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
309; AVX-NEXT:    retq
310;
311; X32-SSE41-LABEL: load_sext_test4:
312; X32-SSE41:       # BB#0: # %entry
313; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
314; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
315; X32-SSE41-NEXT:    retl
316entry:
317 %X = load <2 x i16>, <2 x i16>* %ptr
318 %Y = sext <2 x i16> %X to <2 x i64>
319 ret <2 x i64>%Y
320}
321
322define <2 x i64> @load_sext_test5(<2 x i32> *%ptr) {
323; SSE2-LABEL: load_sext_test5:
324; SSE2:       # BB#0: # %entry
325; SSE2-NEXT:    movq (%rdi), %xmm0
326; SSE2-NEXT:    movdqa %xmm0, %xmm1
327; SSE2-NEXT:    psrad $31, %xmm1
328; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
329; SSE2-NEXT:    retq
330;
331; SSSE3-LABEL: load_sext_test5:
332; SSSE3:       # BB#0: # %entry
333; SSSE3-NEXT:    movq (%rdi), %xmm0
334; SSSE3-NEXT:    movdqa %xmm0, %xmm1
335; SSSE3-NEXT:    psrad $31, %xmm1
336; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
337; SSSE3-NEXT:    retq
338;
339; SSE41-LABEL: load_sext_test5:
340; SSE41:       # BB#0: # %entry
341; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
342; SSE41-NEXT:    retq
343;
344; AVX-LABEL: load_sext_test5:
345; AVX:       # BB#0: # %entry
346; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
347; AVX-NEXT:    retq
348;
349; X32-SSE41-LABEL: load_sext_test5:
350; X32-SSE41:       # BB#0: # %entry
351; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
352; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
353; X32-SSE41-NEXT:    retl
354entry:
355 %X = load <2 x i32>, <2 x i32>* %ptr
356 %Y = sext <2 x i32> %X to <2 x i64>
357 ret <2 x i64>%Y
358}
359
360define <8 x i16> @load_sext_test6(<8 x i8> *%ptr) {
361; SSE2-LABEL: load_sext_test6:
362; SSE2:       # BB#0: # %entry
363; SSE2-NEXT:    movq (%rdi), %xmm0
364; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
365; SSE2-NEXT:    psraw $8, %xmm0
366; SSE2-NEXT:    retq
367;
368; SSSE3-LABEL: load_sext_test6:
369; SSSE3:       # BB#0: # %entry
370; SSSE3-NEXT:    movq (%rdi), %xmm0
371; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
372; SSSE3-NEXT:    psraw $8, %xmm0
373; SSSE3-NEXT:    retq
374;
375; SSE41-LABEL: load_sext_test6:
376; SSE41:       # BB#0: # %entry
377; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
378; SSE41-NEXT:    retq
379;
380; AVX-LABEL: load_sext_test6:
381; AVX:       # BB#0: # %entry
382; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
383; AVX-NEXT:    retq
384;
385; X32-SSE41-LABEL: load_sext_test6:
386; X32-SSE41:       # BB#0: # %entry
387; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
388; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
389; X32-SSE41-NEXT:    retl
390entry:
391 %X = load <8 x i8>, <8 x i8>* %ptr
392 %Y = sext <8 x i8> %X to <8 x i16>
393 ret <8 x i16>%Y
394}
395
396define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
397; SSE2-LABEL: sext_4i1_to_4i64:
398; SSE2:       # BB#0:
399; SSE2-NEXT:    pslld $31, %xmm0
400; SSE2-NEXT:    psrad $31, %xmm0
401; SSE2-NEXT:    movdqa %xmm0, %xmm2
402; SSE2-NEXT:    psrad $31, %xmm2
403; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
404; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
405; SSE2-NEXT:    movdqa %xmm1, %xmm2
406; SSE2-NEXT:    psrad $31, %xmm2
407; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
408; SSE2-NEXT:    retq
409;
410; SSSE3-LABEL: sext_4i1_to_4i64:
411; SSSE3:       # BB#0:
412; SSSE3-NEXT:    pslld $31, %xmm0
413; SSSE3-NEXT:    psrad $31, %xmm0
414; SSSE3-NEXT:    movdqa %xmm0, %xmm2
415; SSSE3-NEXT:    psrad $31, %xmm2
416; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
417; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
418; SSSE3-NEXT:    movdqa %xmm1, %xmm2
419; SSSE3-NEXT:    psrad $31, %xmm2
420; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
421; SSSE3-NEXT:    retq
422;
423; SSE41-LABEL: sext_4i1_to_4i64:
424; SSE41:       # BB#0:
425; SSE41-NEXT:    pslld $31, %xmm0
426; SSE41-NEXT:    psrad $31, %xmm0
427; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
428; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
429; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
430; SSE41-NEXT:    movdqa %xmm2, %xmm0
431; SSE41-NEXT:    retq
432;
433; AVX1-LABEL: sext_4i1_to_4i64:
434; AVX1:       # BB#0:
435; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
436; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
437; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
438; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
439; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
440; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
441; AVX1-NEXT:    retq
442;
443; AVX2-LABEL: sext_4i1_to_4i64:
444; AVX2:       # BB#0:
445; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
446; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
447; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
448; AVX2-NEXT:    retq
449;
450; X32-SSE41-LABEL: sext_4i1_to_4i64:
451; X32-SSE41:       # BB#0:
452; X32-SSE41-NEXT:    pslld $31, %xmm0
453; X32-SSE41-NEXT:    psrad $31, %xmm0
454; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
455; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
456; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
457; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
458; X32-SSE41-NEXT:    retl
459  %extmask = sext <4 x i1> %mask to <4 x i64>
460  ret <4 x i64> %extmask
461}
462
463define <16 x i16> @sext_16i8_to_16i16(<16 x i8> *%ptr) {
464; SSE2-LABEL: sext_16i8_to_16i16:
465; SSE2:       # BB#0: # %entry
466; SSE2-NEXT:    movq (%rdi), %xmm0
467; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
468; SSE2-NEXT:    psraw $8, %xmm0
469; SSE2-NEXT:    movq 8(%rdi), %xmm1
470; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
471; SSE2-NEXT:    psraw $8, %xmm1
472; SSE2-NEXT:    retq
473;
474; SSSE3-LABEL: sext_16i8_to_16i16:
475; SSSE3:       # BB#0: # %entry
476; SSSE3-NEXT:    movq (%rdi), %xmm0
477; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
478; SSSE3-NEXT:    psraw $8, %xmm0
479; SSSE3-NEXT:    movq 8(%rdi), %xmm1
480; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
481; SSSE3-NEXT:    psraw $8, %xmm1
482; SSSE3-NEXT:    retq
483;
484; SSE41-LABEL: sext_16i8_to_16i16:
485; SSE41:       # BB#0: # %entry
486; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
487; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
488; SSE41-NEXT:    retq
489;
490; AVX1-LABEL: sext_16i8_to_16i16:
491; AVX1:       # BB#0: # %entry
492; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
493; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm1
494; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
495; AVX1-NEXT:    retq
496;
497; AVX2-LABEL: sext_16i8_to_16i16:
498; AVX2:       # BB#0: # %entry
499; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
500; AVX2-NEXT:    retq
501;
502; X32-SSE41-LABEL: sext_16i8_to_16i16:
503; X32-SSE41:       # BB#0: # %entry
504; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
505; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
506; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
507; X32-SSE41-NEXT:    retl
508entry:
509 %X = load <16 x i8>, <16 x i8>* %ptr
510 %Y = sext <16 x i8> %X to <16 x i16>
511 ret <16 x i16> %Y
512}
513
514define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
515; SSE2-LABEL: sext_4i8_to_4i64:
516; SSE2:       # BB#0:
517; SSE2-NEXT:    pslld $24, %xmm0
518; SSE2-NEXT:    psrad $24, %xmm0
519; SSE2-NEXT:    movdqa %xmm0, %xmm2
520; SSE2-NEXT:    psrad $31, %xmm2
521; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
522; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
523; SSE2-NEXT:    movdqa %xmm1, %xmm2
524; SSE2-NEXT:    psrad $31, %xmm2
525; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
526; SSE2-NEXT:    retq
527;
528; SSSE3-LABEL: sext_4i8_to_4i64:
529; SSSE3:       # BB#0:
530; SSSE3-NEXT:    pslld $24, %xmm0
531; SSSE3-NEXT:    psrad $24, %xmm0
532; SSSE3-NEXT:    movdqa %xmm0, %xmm2
533; SSSE3-NEXT:    psrad $31, %xmm2
534; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
535; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
536; SSSE3-NEXT:    movdqa %xmm1, %xmm2
537; SSSE3-NEXT:    psrad $31, %xmm2
538; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
539; SSSE3-NEXT:    retq
540;
541; SSE41-LABEL: sext_4i8_to_4i64:
542; SSE41:       # BB#0:
543; SSE41-NEXT:    pslld $24, %xmm0
544; SSE41-NEXT:    psrad $24, %xmm0
545; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
546; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
547; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
548; SSE41-NEXT:    movdqa %xmm2, %xmm0
549; SSE41-NEXT:    retq
550;
551; AVX1-LABEL: sext_4i8_to_4i64:
552; AVX1:       # BB#0:
553; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
554; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
555; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
556; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
557; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
558; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
559; AVX1-NEXT:    retq
560;
561; AVX2-LABEL: sext_4i8_to_4i64:
562; AVX2:       # BB#0:
563; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
564; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
565; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
566; AVX2-NEXT:    retq
567;
568; X32-SSE41-LABEL: sext_4i8_to_4i64:
569; X32-SSE41:       # BB#0:
570; X32-SSE41-NEXT:    pslld $24, %xmm0
571; X32-SSE41-NEXT:    psrad $24, %xmm0
572; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
573; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
574; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
575; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
576; X32-SSE41-NEXT:    retl
577  %extmask = sext <4 x i8> %mask to <4 x i64>
578  ret <4 x i64> %extmask
579}
580
581define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
582; SSE2-LABEL: load_sext_4i8_to_4i64:
583; SSE2:       # BB#0: # %entry
584; SSE2-NEXT:    movsbq 1(%rdi), %rax
585; SSE2-NEXT:    movd %rax, %xmm1
586; SSE2-NEXT:    movsbq (%rdi), %rax
587; SSE2-NEXT:    movd %rax, %xmm0
588; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
589; SSE2-NEXT:    movsbq 3(%rdi), %rax
590; SSE2-NEXT:    movd %rax, %xmm2
591; SSE2-NEXT:    movsbq 2(%rdi), %rax
592; SSE2-NEXT:    movd %rax, %xmm1
593; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
594; SSE2-NEXT:    retq
595;
596; SSSE3-LABEL: load_sext_4i8_to_4i64:
597; SSSE3:       # BB#0: # %entry
598; SSSE3-NEXT:    movsbq 1(%rdi), %rax
599; SSSE3-NEXT:    movd %rax, %xmm1
600; SSSE3-NEXT:    movsbq (%rdi), %rax
601; SSSE3-NEXT:    movd %rax, %xmm0
602; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
603; SSSE3-NEXT:    movsbq 3(%rdi), %rax
604; SSSE3-NEXT:    movd %rax, %xmm2
605; SSSE3-NEXT:    movsbq 2(%rdi), %rax
606; SSSE3-NEXT:    movd %rax, %xmm1
607; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
608; SSSE3-NEXT:    retq
609;
610; SSE41-LABEL: load_sext_4i8_to_4i64:
611; SSE41:       # BB#0: # %entry
612; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
613; SSE41-NEXT:    pmovsxbq 2(%rdi), %xmm1
614; SSE41-NEXT:    retq
615;
616; AVX1-LABEL: load_sext_4i8_to_4i64:
617; AVX1:       # BB#0: # %entry
618; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
619; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
620; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
621; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
622; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
623; AVX1-NEXT:    retq
624;
625; AVX2-LABEL: load_sext_4i8_to_4i64:
626; AVX2:       # BB#0: # %entry
627; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm0
628; AVX2-NEXT:    retq
629;
630; X32-SSE41-LABEL: load_sext_4i8_to_4i64:
631; X32-SSE41:       # BB#0: # %entry
632; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
633; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
634; X32-SSE41-NEXT:    pmovsxbq 2(%eax), %xmm1
635; X32-SSE41-NEXT:    retl
636entry:
637 %X = load <4 x i8>, <4 x i8>* %ptr
638 %Y = sext <4 x i8> %X to <4 x i64>
639 ret <4 x i64>%Y
640}
641
642define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
643; SSE2-LABEL: load_sext_4i16_to_4i64:
644; SSE2:       # BB#0: # %entry
645; SSE2-NEXT:    movswq 2(%rdi), %rax
646; SSE2-NEXT:    movd %rax, %xmm1
647; SSE2-NEXT:    movswq (%rdi), %rax
648; SSE2-NEXT:    movd %rax, %xmm0
649; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
650; SSE2-NEXT:    movswq 6(%rdi), %rax
651; SSE2-NEXT:    movd %rax, %xmm2
652; SSE2-NEXT:    movswq 4(%rdi), %rax
653; SSE2-NEXT:    movd %rax, %xmm1
654; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
655; SSE2-NEXT:    retq
656;
657; SSSE3-LABEL: load_sext_4i16_to_4i64:
658; SSSE3:       # BB#0: # %entry
659; SSSE3-NEXT:    movswq 2(%rdi), %rax
660; SSSE3-NEXT:    movd %rax, %xmm1
661; SSSE3-NEXT:    movswq (%rdi), %rax
662; SSSE3-NEXT:    movd %rax, %xmm0
663; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
664; SSSE3-NEXT:    movswq 6(%rdi), %rax
665; SSSE3-NEXT:    movd %rax, %xmm2
666; SSSE3-NEXT:    movswq 4(%rdi), %rax
667; SSSE3-NEXT:    movd %rax, %xmm1
668; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
669; SSSE3-NEXT:    retq
670;
671; SSE41-LABEL: load_sext_4i16_to_4i64:
672; SSE41:       # BB#0: # %entry
673; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
674; SSE41-NEXT:    pmovsxwq 4(%rdi), %xmm1
675; SSE41-NEXT:    retq
676;
677; AVX1-LABEL: load_sext_4i16_to_4i64:
678; AVX1:       # BB#0: # %entry
679; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm0
680; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
681; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
682; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
683; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
684; AVX1-NEXT:    retq
685;
686; AVX2-LABEL: load_sext_4i16_to_4i64:
687; AVX2:       # BB#0: # %entry
688; AVX2-NEXT:    vpmovsxwq (%rdi), %ymm0
689; AVX2-NEXT:    retq
690;
691; X32-SSE41-LABEL: load_sext_4i16_to_4i64:
692; X32-SSE41:       # BB#0: # %entry
693; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
694; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
695; X32-SSE41-NEXT:    pmovsxwq 4(%eax), %xmm1
696; X32-SSE41-NEXT:    retl
697entry:
698 %X = load <4 x i16>, <4 x i16>* %ptr
699 %Y = sext <4 x i16> %X to <4 x i64>
700 ret <4 x i64>%Y
701}
702