1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
8
9define <8 x i16> @zext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
10; SSE2-LABEL: zext_16i8_to_8i16:
11; SSE2:       # BB#0: # %entry
12; SSE2-NEXT:    pxor %xmm1, %xmm1
13; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
14; SSE2-NEXT:    retq
15;
16; SSSE3-LABEL: zext_16i8_to_8i16:
17; SSSE3:       # BB#0: # %entry
18; SSSE3-NEXT:    pxor %xmm1, %xmm1
19; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
20; SSSE3-NEXT:    retq
21;
22; SSE41-LABEL: zext_16i8_to_8i16:
23; SSE41:       # BB#0: # %entry
24; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
25; SSE41-NEXT:    retq
26;
27; AVX-LABEL: zext_16i8_to_8i16:
28; AVX:       # BB#0: # %entry
29; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
30; AVX-NEXT:    retq
31entry:
32  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
33  %C = zext <8 x i8> %B to <8 x i16>
34  ret <8 x i16> %C
35}
36
37; PR17654
38define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %A) {
39; SSE2-LABEL: zext_16i8_to_16i16:
40; SSE2:       # BB#0: # %entry
41; SSE2-NEXT:    movdqa %xmm0, %xmm1
42; SSE2-NEXT:    pxor %xmm2, %xmm2
43; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
44; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
45; SSE2-NEXT:    retq
46;
47; SSSE3-LABEL: zext_16i8_to_16i16:
48; SSSE3:       # BB#0: # %entry
49; SSSE3-NEXT:    movdqa %xmm0, %xmm1
50; SSSE3-NEXT:    pxor %xmm2, %xmm2
51; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
52; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
53; SSSE3-NEXT:    retq
54;
55; SSE41-LABEL: zext_16i8_to_16i16:
56; SSE41:       # BB#0: # %entry
57; SSE41-NEXT:    movdqa %xmm0, %xmm1
58; SSE41-NEXT:    pxor %xmm2, %xmm2
59; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
60; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
61; SSE41-NEXT:    retq
62;
63; AVX1-LABEL: zext_16i8_to_16i16:
64; AVX1:       # BB#0: # %entry
65; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
66; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
67; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
68; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
69; AVX1-NEXT:    retq
70;
71; AVX2-LABEL: zext_16i8_to_16i16:
72; AVX2:       # BB#0: # %entry
73; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
74; AVX2-NEXT:    retq
75;
76; AVX512-LABEL: zext_16i8_to_16i16:
77; AVX512:       # BB#0: # %entry
78; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
79; AVX512-NEXT:    retq
80entry:
81  %B = zext <16 x i8> %A to <16 x i16>
82  ret <16 x i16> %B
83}
84
85define <4 x i32> @zext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
86; SSE2-LABEL: zext_16i8_to_4i32:
87; SSE2:       # BB#0: # %entry
88; SSE2-NEXT:    pxor %xmm1, %xmm1
89; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
90; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
91; SSE2-NEXT:    retq
92;
93; SSSE3-LABEL: zext_16i8_to_4i32:
94; SSSE3:       # BB#0: # %entry
95; SSSE3-NEXT:    pxor %xmm1, %xmm1
96; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
97; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
98; SSSE3-NEXT:    retq
99;
100; SSE41-LABEL: zext_16i8_to_4i32:
101; SSE41:       # BB#0: # %entry
102; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
103; SSE41-NEXT:    retq
104;
105; AVX-LABEL: zext_16i8_to_4i32:
106; AVX:       # BB#0: # %entry
107; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
108; AVX-NEXT:    retq
109entry:
110  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
111  %C = zext <4 x i8> %B to <4 x i32>
112  ret <4 x i32> %C
113}
114
115define <8 x i32> @zext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
116; SSE2-LABEL: zext_16i8_to_8i32:
117; SSE2:       # BB#0: # %entry
118; SSE2-NEXT:    movdqa %xmm0, %xmm1
119; SSE2-NEXT:    pxor %xmm2, %xmm2
120; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
121; SSE2-NEXT:    movdqa %xmm1, %xmm0
122; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
123; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
124; SSE2-NEXT:    retq
125;
126; SSSE3-LABEL: zext_16i8_to_8i32:
127; SSSE3:       # BB#0: # %entry
128; SSSE3-NEXT:    movdqa %xmm0, %xmm1
129; SSSE3-NEXT:    pxor %xmm2, %xmm2
130; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
131; SSSE3-NEXT:    movdqa %xmm1, %xmm0
132; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
133; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
134; SSSE3-NEXT:    retq
135;
136; SSE41-LABEL: zext_16i8_to_8i32:
137; SSE41:       # BB#0: # %entry
138; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
139; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
140; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
141; SSE41-NEXT:    movdqa %xmm2, %xmm0
142; SSE41-NEXT:    retq
143;
144; AVX1-LABEL: zext_16i8_to_8i32:
145; AVX1:       # BB#0: # %entry
146; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
147; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
148; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
149; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
150; AVX1-NEXT:    retq
151;
152; AVX2-LABEL: zext_16i8_to_8i32:
153; AVX2:       # BB#0: # %entry
154; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
155; AVX2-NEXT:    retq
156;
157; AVX512-LABEL: zext_16i8_to_8i32:
158; AVX512:       # BB#0: # %entry
159; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
160; AVX512-NEXT:    retq
161entry:
162  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
163  %C = zext <8 x i8> %B to <8 x i32>
164  ret <8 x i32> %C
165}
166
167define <2 x i64> @zext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
168; SSE2-LABEL: zext_16i8_to_2i64:
169; SSE2:       # BB#0: # %entry
170; SSE2-NEXT:    pxor %xmm1, %xmm1
171; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
172; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
173; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
174; SSE2-NEXT:    retq
175;
176; SSSE3-LABEL: zext_16i8_to_2i64:
177; SSSE3:       # BB#0: # %entry
178; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
179; SSSE3-NEXT:    retq
180;
181; SSE41-LABEL: zext_16i8_to_2i64:
182; SSE41:       # BB#0: # %entry
183; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
184; SSE41-NEXT:    retq
185;
186; AVX-LABEL: zext_16i8_to_2i64:
187; AVX:       # BB#0: # %entry
188; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
189; AVX-NEXT:    retq
190entry:
191  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
192  %C = zext <2 x i8> %B to <2 x i64>
193  ret <2 x i64> %C
194}
195
196define <4 x i64> @zext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
197; SSE2-LABEL: zext_16i8_to_4i64:
198; SSE2:       # BB#0: # %entry
199; SSE2-NEXT:    movdqa %xmm0, %xmm1
200; SSE2-NEXT:    pxor %xmm2, %xmm2
201; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
202; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
203; SSE2-NEXT:    movdqa %xmm1, %xmm0
204; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
205; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
206; SSE2-NEXT:    retq
207;
208; SSSE3-LABEL: zext_16i8_to_4i64:
209; SSSE3:       # BB#0: # %entry
210; SSSE3-NEXT:    movdqa %xmm0, %xmm1
211; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
212; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
213; SSSE3-NEXT:    retq
214;
215; SSE41-LABEL: zext_16i8_to_4i64:
216; SSE41:       # BB#0: # %entry
217; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
218; SSE41-NEXT:    psrld $16, %xmm0
219; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
220; SSE41-NEXT:    movdqa %xmm2, %xmm0
221; SSE41-NEXT:    retq
222;
223; AVX1-LABEL: zext_16i8_to_4i64:
224; AVX1:       # BB#0: # %entry
225; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
226; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
227; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
228; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
229; AVX1-NEXT:    retq
230;
231; AVX2-LABEL: zext_16i8_to_4i64:
232; AVX2:       # BB#0: # %entry
233; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
234; AVX2-NEXT:    retq
235;
236; AVX512-LABEL: zext_16i8_to_4i64:
237; AVX512:       # BB#0: # %entry
238; AVX512-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
239; AVX512-NEXT:    retq
240entry:
241  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
242  %C = zext <4 x i8> %B to <4 x i64>
243  ret <4 x i64> %C
244}
245
246define <4 x i32> @zext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
247; SSE2-LABEL: zext_8i16_to_4i32:
248; SSE2:       # BB#0: # %entry
249; SSE2-NEXT:    pxor %xmm1, %xmm1
250; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
251; SSE2-NEXT:    retq
252;
253; SSSE3-LABEL: zext_8i16_to_4i32:
254; SSSE3:       # BB#0: # %entry
255; SSSE3-NEXT:    pxor %xmm1, %xmm1
256; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
257; SSSE3-NEXT:    retq
258;
259; SSE41-LABEL: zext_8i16_to_4i32:
260; SSE41:       # BB#0: # %entry
261; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
262; SSE41-NEXT:    retq
263;
264; AVX-LABEL: zext_8i16_to_4i32:
265; AVX:       # BB#0: # %entry
266; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
267; AVX-NEXT:    retq
268entry:
269  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
270  %C = zext <4 x i16> %B to <4 x i32>
271  ret <4 x i32> %C
272}
273
274define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
275; SSE2-LABEL: zext_8i16_to_8i32:
276; SSE2:       # BB#0: # %entry
277; SSE2-NEXT:    movdqa %xmm0, %xmm1
278; SSE2-NEXT:    pxor %xmm2, %xmm2
279; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
280; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
281; SSE2-NEXT:    retq
282;
283; SSSE3-LABEL: zext_8i16_to_8i32:
284; SSSE3:       # BB#0: # %entry
285; SSSE3-NEXT:    movdqa %xmm0, %xmm1
286; SSSE3-NEXT:    pxor %xmm2, %xmm2
287; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
288; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
289; SSSE3-NEXT:    retq
290;
291; SSE41-LABEL: zext_8i16_to_8i32:
292; SSE41:       # BB#0: # %entry
293; SSE41-NEXT:    movdqa %xmm0, %xmm1
294; SSE41-NEXT:    pxor %xmm2, %xmm2
295; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
296; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
297; SSE41-NEXT:    retq
298;
299; AVX1-LABEL: zext_8i16_to_8i32:
300; AVX1:       # BB#0: # %entry
301; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
302; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
303; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
304; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
305; AVX1-NEXT:    retq
306;
307; AVX2-LABEL: zext_8i16_to_8i32:
308; AVX2:       # BB#0: # %entry
309; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
310; AVX2-NEXT:    retq
311;
312; AVX512-LABEL: zext_8i16_to_8i32:
313; AVX512:       # BB#0: # %entry
314; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
315; AVX512-NEXT:    retq
316entry:
317  %B = zext <8 x i16> %A to <8 x i32>
318  ret <8 x i32>%B
319}
320
321define <2 x i64> @zext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
322; SSE2-LABEL: zext_8i16_to_2i64:
323; SSE2:       # BB#0: # %entry
324; SSE2-NEXT:    pxor %xmm1, %xmm1
325; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
326; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
327; SSE2-NEXT:    retq
328;
329; SSSE3-LABEL: zext_8i16_to_2i64:
330; SSSE3:       # BB#0: # %entry
331; SSSE3-NEXT:    pxor %xmm1, %xmm1
332; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
333; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
334; SSSE3-NEXT:    retq
335;
336; SSE41-LABEL: zext_8i16_to_2i64:
337; SSE41:       # BB#0: # %entry
338; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
339; SSE41-NEXT:    retq
340;
341; AVX-LABEL: zext_8i16_to_2i64:
342; AVX:       # BB#0: # %entry
343; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
344; AVX-NEXT:    retq
345entry:
346  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
347  %C = zext <2 x i16> %B to <2 x i64>
348  ret <2 x i64> %C
349}
350
351define <4 x i64> @zext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
352; SSE2-LABEL: zext_8i16_to_4i64:
353; SSE2:       # BB#0: # %entry
354; SSE2-NEXT:    movdqa %xmm0, %xmm1
355; SSE2-NEXT:    pxor %xmm2, %xmm2
356; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
357; SSE2-NEXT:    movdqa %xmm1, %xmm0
358; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
359; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
360; SSE2-NEXT:    retq
361;
362; SSSE3-LABEL: zext_8i16_to_4i64:
363; SSSE3:       # BB#0: # %entry
364; SSSE3-NEXT:    movdqa %xmm0, %xmm1
365; SSSE3-NEXT:    pxor %xmm2, %xmm2
366; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
367; SSSE3-NEXT:    movdqa %xmm1, %xmm0
368; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
369; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
370; SSSE3-NEXT:    retq
371;
372; SSE41-LABEL: zext_8i16_to_4i64:
373; SSE41:       # BB#0: # %entry
374; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
375; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
376; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
377; SSE41-NEXT:    movdqa %xmm2, %xmm0
378; SSE41-NEXT:    retq
379;
380; AVX1-LABEL: zext_8i16_to_4i64:
381; AVX1:       # BB#0: # %entry
382; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
383; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
384; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
385; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
386; AVX1-NEXT:    retq
387;
388; AVX2-LABEL: zext_8i16_to_4i64:
389; AVX2:       # BB#0: # %entry
390; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
391; AVX2-NEXT:    retq
392;
393; AVX512-LABEL: zext_8i16_to_4i64:
394; AVX512:       # BB#0: # %entry
395; AVX512-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
396; AVX512-NEXT:    retq
397entry:
398  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
399  %C = zext <4 x i16> %B to <4 x i64>
400  ret <4 x i64> %C
401}
402
403define <2 x i64> @zext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
404; SSE2-LABEL: zext_4i32_to_2i64:
405; SSE2:       # BB#0: # %entry
406; SSE2-NEXT:    pxor %xmm1, %xmm1
407; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
408; SSE2-NEXT:    retq
409;
410; SSSE3-LABEL: zext_4i32_to_2i64:
411; SSSE3:       # BB#0: # %entry
412; SSSE3-NEXT:    pxor %xmm1, %xmm1
413; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
414; SSSE3-NEXT:    retq
415;
416; SSE41-LABEL: zext_4i32_to_2i64:
417; SSE41:       # BB#0: # %entry
418; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
419; SSE41-NEXT:    retq
420;
421; AVX-LABEL: zext_4i32_to_2i64:
422; AVX:       # BB#0: # %entry
423; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
424; AVX-NEXT:    retq
425entry:
426  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
427  %C = zext <2 x i32> %B to <2 x i64>
428  ret <2 x i64> %C
429}
430
431define <4 x i64> @zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
432; SSE2-LABEL: zext_4i32_to_4i64:
433; SSE2:       # BB#0: # %entry
434; SSE2-NEXT:    movdqa %xmm0, %xmm1
435; SSE2-NEXT:    pxor %xmm2, %xmm2
436; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
437; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
438; SSE2-NEXT:    retq
439;
440; SSSE3-LABEL: zext_4i32_to_4i64:
441; SSSE3:       # BB#0: # %entry
442; SSSE3-NEXT:    movdqa %xmm0, %xmm1
443; SSSE3-NEXT:    pxor %xmm2, %xmm2
444; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
445; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
446; SSSE3-NEXT:    retq
447;
448; SSE41-LABEL: zext_4i32_to_4i64:
449; SSE41:       # BB#0: # %entry
450; SSE41-NEXT:    movdqa %xmm0, %xmm1
451; SSE41-NEXT:    pxor %xmm2, %xmm2
452; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
453; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
454; SSE41-NEXT:    retq
455;
456; AVX1-LABEL: zext_4i32_to_4i64:
457; AVX1:       # BB#0: # %entry
458; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
459; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
460; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
461; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
462; AVX1-NEXT:    retq
463;
464; AVX2-LABEL: zext_4i32_to_4i64:
465; AVX2:       # BB#0: # %entry
466; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
467; AVX2-NEXT:    retq
468;
469; AVX512-LABEL: zext_4i32_to_4i64:
470; AVX512:       # BB#0: # %entry
471; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
472; AVX512-NEXT:    retq
473entry:
474  %B = zext <4 x i32> %A to <4 x i64>
475  ret <4 x i64>%B
476}
477
478define <2 x i64> @load_zext_2i8_to_2i64(<2 x i8> *%ptr) {
479; SSE2-LABEL: load_zext_2i8_to_2i64:
480; SSE2:       # BB#0: # %entry
481; SSE2-NEXT:    movzwl (%rdi), %eax
482; SSE2-NEXT:    movd %eax, %xmm0
483; SSE2-NEXT:    pxor %xmm1, %xmm1
484; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
485; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
486; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
487; SSE2-NEXT:    retq
488;
489; SSSE3-LABEL: load_zext_2i8_to_2i64:
490; SSSE3:       # BB#0: # %entry
491; SSSE3-NEXT:    movzwl (%rdi), %eax
492; SSSE3-NEXT:    movd %eax, %xmm0
493; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
494; SSSE3-NEXT:    retq
495;
496; SSE41-LABEL: load_zext_2i8_to_2i64:
497; SSE41:       # BB#0: # %entry
498; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
499; SSE41-NEXT:    retq
500;
501; AVX-LABEL: load_zext_2i8_to_2i64:
502; AVX:       # BB#0: # %entry
503; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
504; AVX-NEXT:    retq
505entry:
506 %X = load <2 x i8>, <2 x i8>* %ptr
507 %Y = zext <2 x i8> %X to <2 x i64>
508 ret <2 x i64> %Y
509}
510
511define <4 x i32> @load_zext_4i8_to_4i32(<4 x i8> *%ptr) {
512; SSE2-LABEL: load_zext_4i8_to_4i32:
513; SSE2:       # BB#0: # %entry
514; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
515; SSE2-NEXT:    pxor %xmm1, %xmm1
516; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
517; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
518; SSE2-NEXT:    retq
519;
520; SSSE3-LABEL: load_zext_4i8_to_4i32:
521; SSSE3:       # BB#0: # %entry
522; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
523; SSSE3-NEXT:    pxor %xmm1, %xmm1
524; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
525; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
526; SSSE3-NEXT:    retq
527;
528; SSE41-LABEL: load_zext_4i8_to_4i32:
529; SSE41:       # BB#0: # %entry
530; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
531; SSE41-NEXT:    retq
532;
533; AVX-LABEL: load_zext_4i8_to_4i32:
534; AVX:       # BB#0: # %entry
535; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
536; AVX-NEXT:    retq
537entry:
538 %X = load <4 x i8>, <4 x i8>* %ptr
539 %Y = zext <4 x i8> %X to <4 x i32>
540 ret <4 x i32> %Y
541}
542
543define <4 x i64> @load_zext_4i8_to_4i64(<4 x i8> *%ptr) {
544; SSE2-LABEL: load_zext_4i8_to_4i64:
545; SSE2:       # BB#0: # %entry
546; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
547; SSE2-NEXT:    pxor %xmm2, %xmm2
548; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
549; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
550; SSE2-NEXT:    movdqa %xmm1, %xmm0
551; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
552; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
553; SSE2-NEXT:    retq
554;
555; SSSE3-LABEL: load_zext_4i8_to_4i64:
556; SSSE3:       # BB#0: # %entry
557; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
558; SSSE3-NEXT:    movdqa %xmm1, %xmm0
559; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
560; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
561; SSSE3-NEXT:    retq
562;
563; SSE41-LABEL: load_zext_4i8_to_4i64:
564; SSE41:       # BB#0: # %entry
565; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
566; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
567; SSE41-NEXT:    retq
568;
569; AVX1-LABEL: load_zext_4i8_to_4i64:
570; AVX1:       # BB#0: # %entry
571; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
572; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
573; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
574; AVX1-NEXT:    retq
575;
576; AVX2-LABEL: load_zext_4i8_to_4i64:
577; AVX2:       # BB#0: # %entry
578; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
579; AVX2-NEXT:    retq
580;
581; AVX512-LABEL: load_zext_4i8_to_4i64:
582; AVX512:       # BB#0: # %entry
583; AVX512-NEXT:    vpmovzxbq {{.*#+}} ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
584; AVX512-NEXT:    retq
585entry:
586 %X = load <4 x i8>, <4 x i8>* %ptr
587 %Y = zext <4 x i8> %X to <4 x i64>
588 ret <4 x i64> %Y
589}
590
591define <8 x i16> @load_zext_8i8_to_8i16(<8 x i8> *%ptr) {
592; SSE2-LABEL: load_zext_8i8_to_8i16:
593; SSE2:       # BB#0: # %entry
594; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
595; SSE2-NEXT:    pxor %xmm1, %xmm1
596; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
597; SSE2-NEXT:    retq
598;
599; SSSE3-LABEL: load_zext_8i8_to_8i16:
600; SSSE3:       # BB#0: # %entry
601; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
602; SSSE3-NEXT:    pxor %xmm1, %xmm1
603; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
604; SSSE3-NEXT:    retq
605;
606; SSE41-LABEL: load_zext_8i8_to_8i16:
607; SSE41:       # BB#0: # %entry
608; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
609; SSE41-NEXT:    retq
610;
611; AVX-LABEL: load_zext_8i8_to_8i16:
612; AVX:       # BB#0: # %entry
613; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
614; AVX-NEXT:    retq
615entry:
616 %X = load <8 x i8>, <8 x i8>* %ptr
617 %Y = zext <8 x i8> %X to <8 x i16>
618 ret <8 x i16> %Y
619}
620
621define <8 x i32> @load_zext_8i8_to_8i32(<8 x i8> *%ptr) {
622; SSE2-LABEL: load_zext_8i8_to_8i32:
623; SSE2:       # BB#0: # %entry
624; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
625; SSE2-NEXT:    pxor %xmm2, %xmm2
626; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
627; SSE2-NEXT:    movdqa %xmm1, %xmm0
628; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
629; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
630; SSE2-NEXT:    retq
631;
632; SSSE3-LABEL: load_zext_8i8_to_8i32:
633; SSSE3:       # BB#0: # %entry
634; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
635; SSSE3-NEXT:    pxor %xmm2, %xmm2
636; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
637; SSSE3-NEXT:    movdqa %xmm1, %xmm0
638; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
639; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
640; SSSE3-NEXT:    retq
641;
642; SSE41-LABEL: load_zext_8i8_to_8i32:
643; SSE41:       # BB#0: # %entry
644; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
645; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
646; SSE41-NEXT:    retq
647;
648; AVX1-LABEL: load_zext_8i8_to_8i32:
649; AVX1:       # BB#0: # %entry
650; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
651; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
652; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
653; AVX1-NEXT:    retq
654;
655; AVX2-LABEL: load_zext_8i8_to_8i32:
656; AVX2:       # BB#0: # %entry
657; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
658; AVX2-NEXT:    retq
659;
660; AVX512-LABEL: load_zext_8i8_to_8i32:
661; AVX512:       # BB#0: # %entry
662; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
663; AVX512-NEXT:    retq
664entry:
665 %X = load <8 x i8>, <8 x i8>* %ptr
666 %Y = zext <8 x i8> %X to <8 x i32>
667 ret <8 x i32> %Y
668}
669
670define <8 x i64> @load_zext_8i8_to_8i64(<8 x i8> *%ptr) {
671; SSE2-LABEL: load_zext_8i8_to_8i64:
672; SSE2:       # BB#0: # %entry
673; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
674; SSE2-NEXT:    pxor %xmm4, %xmm4
675; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
676; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
677; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
678; SSE2-NEXT:    movdqa %xmm1, %xmm0
679; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
680; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
681; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
682; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
683; SSE2-NEXT:    movdqa %xmm3, %xmm2
684; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
685; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
686; SSE2-NEXT:    retq
687;
688; SSSE3-LABEL: load_zext_8i8_to_8i64:
689; SSSE3:       # BB#0: # %entry
690; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
691; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [0,128,128,128,128,128,128,128,1,128,128,128,128,128,128,128]
692; SSSE3-NEXT:    movdqa %xmm1, %xmm0
693; SSSE3-NEXT:    pshufb %xmm4, %xmm0
694; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2,128,128,128,128,128,128,128,3,128,128,128,128,128,128,128]
695; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
696; SSSE3-NEXT:    pshufb %xmm5, %xmm1
697; SSSE3-NEXT:    movdqa %xmm3, %xmm2
698; SSSE3-NEXT:    pshufb %xmm4, %xmm2
699; SSSE3-NEXT:    pshufb %xmm5, %xmm3
700; SSSE3-NEXT:    retq
701;
702; SSE41-LABEL: load_zext_8i8_to_8i64:
703; SSE41:       # BB#0: # %entry
704; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
705; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
706; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
707; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
708; SSE41-NEXT:    retq
709;
710; AVX1-LABEL: load_zext_8i8_to_8i64:
711; AVX1:       # BB#0: # %entry
712; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
713; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
714; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
715; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
716; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
717; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
718; AVX1-NEXT:    retq
719;
720; AVX2-LABEL: load_zext_8i8_to_8i64:
721; AVX2:       # BB#0: # %entry
722; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
723; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
724; AVX2-NEXT:    retq
725;
726; AVX512-LABEL: load_zext_8i8_to_8i64:
727; AVX512:       # BB#0: # %entry
728; AVX512-NEXT:    vpmovzxbq {{.*#+}} zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
729; AVX512-NEXT:    retq
730entry:
731 %X = load <8 x i8>, <8 x i8>* %ptr
732 %Y = zext <8 x i8> %X to <8 x i64>
733 ret <8 x i64> %Y
734}
735
736define <16 x i16> @load_zext_16i8_to_16i16(<16 x i8> *%ptr) {
737; SSE2-LABEL: load_zext_16i8_to_16i16:
738; SSE2:       # BB#0: # %entry
739; SSE2-NEXT:    movdqa (%rdi), %xmm1
740; SSE2-NEXT:    pxor %xmm2, %xmm2
741; SSE2-NEXT:    movdqa %xmm1, %xmm0
742; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
743; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
744; SSE2-NEXT:    retq
745;
746; SSSE3-LABEL: load_zext_16i8_to_16i16:
747; SSSE3:       # BB#0: # %entry
748; SSSE3-NEXT:    movdqa (%rdi), %xmm1
749; SSSE3-NEXT:    pxor %xmm2, %xmm2
750; SSSE3-NEXT:    movdqa %xmm1, %xmm0
751; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
752; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
753; SSSE3-NEXT:    retq
754;
755; SSE41-LABEL: load_zext_16i8_to_16i16:
756; SSE41:       # BB#0: # %entry
757; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
758; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
759; SSE41-NEXT:    retq
760;
761; AVX1-LABEL: load_zext_16i8_to_16i16:
762; AVX1:       # BB#0: # %entry
763; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
764; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
765; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
766; AVX1-NEXT:    retq
767;
768; AVX2-LABEL: load_zext_16i8_to_16i16:
769; AVX2:       # BB#0: # %entry
770; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
771; AVX2-NEXT:    retq
772;
773; AVX512-LABEL: load_zext_16i8_to_16i16:
774; AVX512:       # BB#0: # %entry
775; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
776; AVX512-NEXT:    retq
777entry:
778 %X = load <16 x i8>, <16 x i8>* %ptr
779 %Y = zext <16 x i8> %X to <16 x i16>
780 ret <16 x i16> %Y
781}
782
783define <2 x i64> @load_zext_2i16_to_2i64(<2 x i16> *%ptr) {
784; SSE2-LABEL: load_zext_2i16_to_2i64:
785; SSE2:       # BB#0: # %entry
786; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
787; SSE2-NEXT:    pxor %xmm1, %xmm1
788; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
789; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
790; SSE2-NEXT:    retq
791;
792; SSSE3-LABEL: load_zext_2i16_to_2i64:
793; SSSE3:       # BB#0: # %entry
794; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
795; SSSE3-NEXT:    pxor %xmm1, %xmm1
796; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
797; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
798; SSSE3-NEXT:    retq
799;
800; SSE41-LABEL: load_zext_2i16_to_2i64:
801; SSE41:       # BB#0: # %entry
802; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
803; SSE41-NEXT:    retq
804;
805; AVX-LABEL: load_zext_2i16_to_2i64:
806; AVX:       # BB#0: # %entry
807; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
808; AVX-NEXT:    retq
809entry:
810 %X = load <2 x i16>, <2 x i16>* %ptr
811 %Y = zext <2 x i16> %X to <2 x i64>
812 ret <2 x i64> %Y
813}
814
815define <4 x i32> @load_zext_4i16_to_4i32(<4 x i16> *%ptr) {
816; SSE2-LABEL: load_zext_4i16_to_4i32:
817; SSE2:       # BB#0: # %entry
818; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
819; SSE2-NEXT:    pxor %xmm1, %xmm1
820; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
821; SSE2-NEXT:    retq
822;
823; SSSE3-LABEL: load_zext_4i16_to_4i32:
824; SSSE3:       # BB#0: # %entry
825; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
826; SSSE3-NEXT:    pxor %xmm1, %xmm1
827; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
828; SSSE3-NEXT:    retq
829;
830; SSE41-LABEL: load_zext_4i16_to_4i32:
831; SSE41:       # BB#0: # %entry
832; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
833; SSE41-NEXT:    retq
834;
835; AVX-LABEL: load_zext_4i16_to_4i32:
836; AVX:       # BB#0: # %entry
837; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
838; AVX-NEXT:    retq
839entry:
840 %X = load <4 x i16>, <4 x i16>* %ptr
841 %Y = zext <4 x i16> %X to <4 x i32>
842 ret <4 x i32> %Y
843}
844
845define <4 x i64> @load_zext_4i16_to_4i64(<4 x i16> *%ptr) {
846; SSE2-LABEL: load_zext_4i16_to_4i64:
847; SSE2:       # BB#0: # %entry
848; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
849; SSE2-NEXT:    pxor %xmm2, %xmm2
850; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
851; SSE2-NEXT:    movdqa %xmm1, %xmm0
852; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
853; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
854; SSE2-NEXT:    retq
855;
856; SSSE3-LABEL: load_zext_4i16_to_4i64:
857; SSSE3:       # BB#0: # %entry
858; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
859; SSSE3-NEXT:    pxor %xmm2, %xmm2
860; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
861; SSSE3-NEXT:    movdqa %xmm1, %xmm0
862; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
863; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
864; SSSE3-NEXT:    retq
865;
866; SSE41-LABEL: load_zext_4i16_to_4i64:
867; SSE41:       # BB#0: # %entry
868; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
869; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
870; SSE41-NEXT:    retq
871;
872; AVX1-LABEL: load_zext_4i16_to_4i64:
873; AVX1:       # BB#0: # %entry
874; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
875; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
876; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
877; AVX1-NEXT:    retq
878;
879; AVX2-LABEL: load_zext_4i16_to_4i64:
880; AVX2:       # BB#0: # %entry
881; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
882; AVX2-NEXT:    retq
883;
884; AVX512-LABEL: load_zext_4i16_to_4i64:
885; AVX512:       # BB#0: # %entry
886; AVX512-NEXT:    vpmovzxwq {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
887; AVX512-NEXT:    retq
888entry:
889 %X = load <4 x i16>, <4 x i16>* %ptr
890 %Y = zext <4 x i16> %X to <4 x i64>
891 ret <4 x i64> %Y
892}
893
894define <8 x i32> @load_zext_8i16_to_8i32(<8 x i16> *%ptr) {
895; SSE2-LABEL: load_zext_8i16_to_8i32:
896; SSE2:       # BB#0: # %entry
897; SSE2-NEXT:    movdqa (%rdi), %xmm1
898; SSE2-NEXT:    pxor %xmm2, %xmm2
899; SSE2-NEXT:    movdqa %xmm1, %xmm0
900; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
901; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
902; SSE2-NEXT:    retq
903;
904; SSSE3-LABEL: load_zext_8i16_to_8i32:
905; SSSE3:       # BB#0: # %entry
906; SSSE3-NEXT:    movdqa (%rdi), %xmm1
907; SSSE3-NEXT:    pxor %xmm2, %xmm2
908; SSSE3-NEXT:    movdqa %xmm1, %xmm0
909; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
910; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
911; SSSE3-NEXT:    retq
912;
913; SSE41-LABEL: load_zext_8i16_to_8i32:
914; SSE41:       # BB#0: # %entry
915; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
916; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
917; SSE41-NEXT:    retq
918;
919; AVX1-LABEL: load_zext_8i16_to_8i32:
920; AVX1:       # BB#0: # %entry
921; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
922; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
923; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
924; AVX1-NEXT:    retq
925;
926; AVX2-LABEL: load_zext_8i16_to_8i32:
927; AVX2:       # BB#0: # %entry
928; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
929; AVX2-NEXT:    retq
930;
931; AVX512-LABEL: load_zext_8i16_to_8i32:
932; AVX512:       # BB#0: # %entry
933; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
934; AVX512-NEXT:    retq
935entry:
936 %X = load <8 x i16>, <8 x i16>* %ptr
937 %Y = zext <8 x i16> %X to <8 x i32>
938 ret <8 x i32> %Y
939}
940
941define <2 x i64> @load_zext_2i32_to_2i64(<2 x i32> *%ptr) {
942; SSE2-LABEL: load_zext_2i32_to_2i64:
943; SSE2:       # BB#0: # %entry
944; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
945; SSE2-NEXT:    pxor %xmm1, %xmm1
946; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
947; SSE2-NEXT:    retq
948;
949; SSSE3-LABEL: load_zext_2i32_to_2i64:
950; SSSE3:       # BB#0: # %entry
951; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
952; SSSE3-NEXT:    pxor %xmm1, %xmm1
953; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
954; SSSE3-NEXT:    retq
955;
956; SSE41-LABEL: load_zext_2i32_to_2i64:
957; SSE41:       # BB#0: # %entry
958; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
959; SSE41-NEXT:    retq
960;
961; AVX-LABEL: load_zext_2i32_to_2i64:
962; AVX:       # BB#0: # %entry
963; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
964; AVX-NEXT:    retq
965entry:
966 %X = load <2 x i32>, <2 x i32>* %ptr
967 %Y = zext <2 x i32> %X to <2 x i64>
968 ret <2 x i64> %Y
969}
970
971define <4 x i64> @load_zext_4i32_to_4i64(<4 x i32> *%ptr) {
972; SSE2-LABEL: load_zext_4i32_to_4i64:
973; SSE2:       # BB#0: # %entry
974; SSE2-NEXT:    movdqa (%rdi), %xmm1
975; SSE2-NEXT:    pxor %xmm2, %xmm2
976; SSE2-NEXT:    movdqa %xmm1, %xmm0
977; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
978; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
979; SSE2-NEXT:    retq
980;
981; SSSE3-LABEL: load_zext_4i32_to_4i64:
982; SSSE3:       # BB#0: # %entry
983; SSSE3-NEXT:    movdqa (%rdi), %xmm1
984; SSSE3-NEXT:    pxor %xmm2, %xmm2
985; SSSE3-NEXT:    movdqa %xmm1, %xmm0
986; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
987; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
988; SSSE3-NEXT:    retq
989;
990; SSE41-LABEL: load_zext_4i32_to_4i64:
991; SSE41:       # BB#0: # %entry
992; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
993; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
994; SSE41-NEXT:    retq
995;
996; AVX1-LABEL: load_zext_4i32_to_4i64:
997; AVX1:       # BB#0: # %entry
998; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
999; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
1000; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1001; AVX1-NEXT:    retq
1002;
1003; AVX2-LABEL: load_zext_4i32_to_4i64:
1004; AVX2:       # BB#0: # %entry
1005; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1006; AVX2-NEXT:    retq
1007;
1008; AVX512-LABEL: load_zext_4i32_to_4i64:
1009; AVX512:       # BB#0: # %entry
1010; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1011; AVX512-NEXT:    retq
1012entry:
1013 %X = load <4 x i32>, <4 x i32>* %ptr
1014 %Y = zext <4 x i32> %X to <4 x i64>
1015 ret <4 x i64> %Y
1016}
1017
1018define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
1019; SSE2-LABEL: zext_8i8_to_8i32:
1020; SSE2:       # BB#0: # %entry
1021; SSE2-NEXT:    movdqa %xmm0, %xmm1
1022; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
1023; SSE2-NEXT:    pxor %xmm2, %xmm2
1024; SSE2-NEXT:    movdqa %xmm1, %xmm0
1025; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1026; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1027; SSE2-NEXT:    retq
1028;
1029; SSSE3-LABEL: zext_8i8_to_8i32:
1030; SSSE3:       # BB#0: # %entry
1031; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1032; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
1033; SSSE3-NEXT:    pxor %xmm2, %xmm2
1034; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1035; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1036; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1037; SSSE3-NEXT:    retq
1038;
1039; SSE41-LABEL: zext_8i8_to_8i32:
1040; SSE41:       # BB#0: # %entry
1041; SSE41-NEXT:    movdqa %xmm0, %xmm1
1042; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
1043; SSE41-NEXT:    pxor %xmm2, %xmm2
1044; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1045; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1046; SSE41-NEXT:    retq
1047;
1048; AVX1-LABEL: zext_8i8_to_8i32:
1049; AVX1:       # BB#0: # %entry
1050; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1051; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1052; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
1053; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1054; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1055; AVX1-NEXT:    retq
1056;
1057; AVX2-LABEL: zext_8i8_to_8i32:
1058; AVX2:       # BB#0: # %entry
1059; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1060; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1061; AVX2-NEXT:    retq
1062;
1063; AVX512-LABEL: zext_8i8_to_8i32:
1064; AVX512:       # BB#0: # %entry
1065; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
1066; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1067; AVX512-NEXT:    retq
1068entry:
1069  %t = zext <8 x i8> %z to <8 x i32>
1070  ret <8 x i32> %t
1071}
1072
1073define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
1074; SSE2-LABEL: shuf_zext_8i16_to_8i32:
1075; SSE2:       # BB#0: # %entry
1076; SSE2-NEXT:    movdqa %xmm0, %xmm1
1077; SSE2-NEXT:    pxor %xmm2, %xmm2
1078; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1079; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1080; SSE2-NEXT:    retq
1081;
1082; SSSE3-LABEL: shuf_zext_8i16_to_8i32:
1083; SSSE3:       # BB#0: # %entry
1084; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1085; SSSE3-NEXT:    pxor %xmm2, %xmm2
1086; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1087; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1088; SSSE3-NEXT:    retq
1089;
1090; SSE41-LABEL: shuf_zext_8i16_to_8i32:
1091; SSE41:       # BB#0: # %entry
1092; SSE41-NEXT:    movdqa %xmm0, %xmm1
1093; SSE41-NEXT:    pxor %xmm2, %xmm2
1094; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1095; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1096; SSE41-NEXT:    retq
1097;
1098; AVX1-LABEL: shuf_zext_8i16_to_8i32:
1099; AVX1:       # BB#0: # %entry
1100; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1101; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
1102; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1103; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1104; AVX1-NEXT:    retq
1105;
1106; AVX2-LABEL: shuf_zext_8i16_to_8i32:
1107; AVX2:       # BB#0: # %entry
1108; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1109; AVX2-NEXT:    retq
1110;
1111; AVX512-LABEL: shuf_zext_8i16_to_8i32:
1112; AVX512:       # BB#0: # %entry
1113; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1114; AVX512-NEXT:    retq
1115entry:
1116  %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> <i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8, i32 4, i32 8, i32 5, i32 8, i32 6, i32 8, i32 7, i32 8>
1117  %Z = bitcast <16 x i16> %B to <8 x i32>
1118  ret <8 x i32> %Z
1119}
1120
1121define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
1122; SSE2-LABEL: shuf_zext_4i32_to_4i64:
1123; SSE2:       # BB#0: # %entry
1124; SSE2-NEXT:    movdqa %xmm0, %xmm1
1125; SSE2-NEXT:    pxor %xmm2, %xmm2
1126; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1127; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1128; SSE2-NEXT:    retq
1129;
1130; SSSE3-LABEL: shuf_zext_4i32_to_4i64:
1131; SSSE3:       # BB#0: # %entry
1132; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1133; SSSE3-NEXT:    pxor %xmm2, %xmm2
1134; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
1135; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1136; SSSE3-NEXT:    retq
1137;
1138; SSE41-LABEL: shuf_zext_4i32_to_4i64:
1139; SSE41:       # BB#0: # %entry
1140; SSE41-NEXT:    movdqa %xmm0, %xmm1
1141; SSE41-NEXT:    pxor %xmm2, %xmm2
1142; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
1143; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1144; SSE41-NEXT:    retq
1145;
1146; AVX1-LABEL: shuf_zext_4i32_to_4i64:
1147; AVX1:       # BB#0: # %entry
1148; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
1149; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
1150; AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
1151; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,0]
1152; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1153; AVX1-NEXT:    retq
1154;
1155; AVX2-LABEL: shuf_zext_4i32_to_4i64:
1156; AVX2:       # BB#0: # %entry
1157; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1158; AVX2-NEXT:    retq
1159;
1160; AVX512-LABEL: shuf_zext_4i32_to_4i64:
1161; AVX512:       # BB#0: # %entry
1162; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1163; AVX512-NEXT:    retq
1164entry:
1165  %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 4, i32 1, i32 4, i32 2, i32 4, i32 3, i32 4>
1166  %Z = bitcast <8 x i32> %B to <4 x i64>
1167  ret <4 x i64> %Z
1168}
1169
1170define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
1171; SSE2-LABEL: shuf_zext_8i8_to_8i32:
1172; SSE2:       # BB#0: # %entry
1173; SSE2-NEXT:    movdqa %xmm0, %xmm1
1174; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
1175; SSE2-NEXT:    packuswb %xmm1, %xmm1
1176; SSE2-NEXT:    pxor %xmm2, %xmm2
1177; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1178; SSE2-NEXT:    movdqa %xmm1, %xmm0
1179; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1180; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1181; SSE2-NEXT:    retq
1182;
1183; SSSE3-LABEL: shuf_zext_8i8_to_8i32:
1184; SSSE3:       # BB#0: # %entry
1185; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1186; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[2],zero,xmm1[4],zero,xmm1[6],zero,xmm1[8],zero,xmm1[10],zero,xmm1[12],zero,xmm1[14],zero
1187; SSSE3-NEXT:    pxor %xmm2, %xmm2
1188; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1189; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1190; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1191; SSSE3-NEXT:    retq
1192;
1193; SSE41-LABEL: shuf_zext_8i8_to_8i32:
1194; SSE41:       # BB#0: # %entry
1195; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1196; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1197; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1198; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1199; SSE41-NEXT:    movdqa %xmm2, %xmm0
1200; SSE41-NEXT:    retq
1201;
1202; AVX1-LABEL: shuf_zext_8i8_to_8i32:
1203; AVX1:       # BB#0: # %entry
1204; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1205; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1206; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1207; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1208; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1209; AVX1-NEXT:    retq
1210;
1211; AVX2-LABEL: shuf_zext_8i8_to_8i32:
1212; AVX2:       # BB#0: # %entry
1213; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1214; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
1215; AVX2-NEXT:    retq
1216;
1217; AVX512-LABEL: shuf_zext_8i8_to_8i32:
1218; AVX512:       # BB#0: # %entry
1219; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1220; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
1221; AVX512-NEXT:    retq
1222entry:
1223  %B = shufflevector <8 x i8> %A, <8 x i8> zeroinitializer, <32 x i32> <i32 0, i32 8, i32 8, i32 8, i32 1, i32 8, i32 8, i32 8, i32 2, i32 8, i32 8, i32 8, i32 3, i32 8, i32 8, i32 8, i32 4, i32 8, i32 8, i32 8, i32 5, i32 8, i32 8, i32 8, i32 6, i32 8, i32 8, i32 8, i32 7, i32 8, i32 8, i32 8>
1224  %Z = bitcast <32 x i8> %B to <8 x i32>
1225  ret <8 x i32> %Z
1226}
1227
1228define <2 x i64> @shuf_zext_16i8_to_2i64_offset6(<16 x i8> %A) nounwind uwtable readnone ssp {
1229; SSE2-LABEL: shuf_zext_16i8_to_2i64_offset6:
1230; SSE2:       # BB#0: # %entry
1231; SSE2-NEXT:    pxor %xmm1, %xmm1
1232; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
1233; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
1234; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1235; SSE2-NEXT:    retq
1236;
1237; SSSE3-LABEL: shuf_zext_16i8_to_2i64_offset6:
1238; SSSE3:       # BB#0: # %entry
1239; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero
1240; SSSE3-NEXT:    retq
1241;
1242; SSE41-LABEL: shuf_zext_16i8_to_2i64_offset6:
1243; SSE41:       # BB#0: # %entry
1244; SSE41-NEXT:    psrlq $48, %xmm0
1245; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
1246; SSE41-NEXT:    retq
1247;
1248; AVX-LABEL: shuf_zext_16i8_to_2i64_offset6:
1249; AVX:       # BB#0: # %entry
1250; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm0
1251; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
1252; AVX-NEXT:    retq
1253entry:
1254  %B = shufflevector <16 x i8> %A, <16 x i8> zeroinitializer, <16 x i32> <i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
1255  %Z = bitcast <16 x i8> %B to <2 x i64>
1256  ret <2 x i64> %Z
1257}
1258
1259define <4 x i64> @shuf_zext_16i8_to_4i64_offset11(<16 x i8> %A) nounwind uwtable readnone ssp {
1260; SSE2-LABEL: shuf_zext_16i8_to_4i64_offset11:
1261; SSE2:       # BB#0: # %entry
1262; SSE2-NEXT:    movdqa %xmm0, %xmm1
1263; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero
1264; SSE2-NEXT:    pxor %xmm2, %xmm2
1265; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1266; SSE2-NEXT:    movdqa %xmm1, %xmm0
1267; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1268; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1269; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1270; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1271; SSE2-NEXT:    retq
1272;
1273; SSSE3-LABEL: shuf_zext_16i8_to_4i64_offset11:
1274; SSSE3:       # BB#0: # %entry
1275; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1276; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[11],zero,zero,zero,zero,zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero
1277; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[13],zero,zero,zero,zero,zero,zero,zero,xmm1[14],zero,zero,zero,zero,zero,zero,zero
1278; SSSE3-NEXT:    retq
1279;
1280; SSE41-LABEL: shuf_zext_16i8_to_4i64_offset11:
1281; SSE41:       # BB#0: # %entry
1282; SSE41-NEXT:    movdqa %xmm0, %xmm1
1283; SSE41-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1284; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
1285; SSE41-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1286; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
1287; SSE41-NEXT:    movdqa %xmm2, %xmm0
1288; SSE41-NEXT:    retq
1289;
1290; AVX1-LABEL: shuf_zext_16i8_to_4i64_offset11:
1291; AVX1:       # BB#0: # %entry
1292; AVX1-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1293; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
1294; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1295; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
1296; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1297; AVX1-NEXT:    retq
1298;
1299; AVX2-LABEL: shuf_zext_16i8_to_4i64_offset11:
1300; AVX2:       # BB#0: # %entry
1301; AVX2-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1302; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
1303; AVX2-NEXT:    retq
1304;
1305; AVX512-LABEL: shuf_zext_16i8_to_4i64_offset11:
1306; AVX512:       # BB#0: # %entry
1307; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1308; AVX512-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
1309; AVX512-NEXT:    retq
1310entry:
1311  %B = shufflevector <16 x i8> %A, <16 x i8> zeroinitializer, <32 x i32> <i32 11, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 12, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 13, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 14, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
1312  %Z = bitcast <32 x i8> %B to <4 x i64>
1313  ret <4 x i64> %Z
1314}
1315
1316define <2 x i64> @shuf_zext_8i16_to_2i64_offset6(<8 x i16> %A) nounwind uwtable readnone ssp {
1317; SSE2-LABEL: shuf_zext_8i16_to_2i64_offset6:
1318; SSE2:       # BB#0: # %entry
1319; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1320; SSE2-NEXT:    pxor %xmm1, %xmm1
1321; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1322; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1323; SSE2-NEXT:    retq
1324;
1325; SSSE3-LABEL: shuf_zext_8i16_to_2i64_offset6:
1326; SSSE3:       # BB#0: # %entry
1327; SSSE3-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1328; SSSE3-NEXT:    pxor %xmm1, %xmm1
1329; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1330; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1331; SSSE3-NEXT:    retq
1332;
1333; SSE41-LABEL: shuf_zext_8i16_to_2i64_offset6:
1334; SSE41:       # BB#0: # %entry
1335; SSE41-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1336; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1337; SSE41-NEXT:    retq
1338;
1339; AVX-LABEL: shuf_zext_8i16_to_2i64_offset6:
1340; AVX:       # BB#0: # %entry
1341; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1342; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1343; AVX-NEXT:    retq
1344entry:
1345  %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <8 x i32> <i32 3, i32 8, i32 8, i32 8, i32 4, i32 8, i32 8, i32 8>
1346  %Z = bitcast <8 x i16> %B to <2 x i64>
1347  ret <2 x i64> %Z
1348}
1349
1350define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable readnone ssp {
1351; SSE2-LABEL: shuf_zext_8i16_to_4i64_offset2:
1352; SSE2:       # BB#0: # %entry
1353; SSE2-NEXT:    movdqa %xmm0, %xmm1
1354; SSE2-NEXT:    pxor %xmm2, %xmm2
1355; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1356; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1357; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1358; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1359; SSE2-NEXT:    retq
1360;
1361; SSSE3-LABEL: shuf_zext_8i16_to_4i64_offset2:
1362; SSSE3:       # BB#0: # %entry
1363; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1364; SSSE3-NEXT:    pxor %xmm2, %xmm2
1365; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1366; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1367; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1368; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1369; SSSE3-NEXT:    retq
1370;
1371; SSE41-LABEL: shuf_zext_8i16_to_4i64_offset2:
1372; SSE41:       # BB#0: # %entry
1373; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1374; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
1375; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1376; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1377; SSE41-NEXT:    movdqa %xmm2, %xmm0
1378; SSE41-NEXT:    retq
1379;
1380; AVX1-LABEL: shuf_zext_8i16_to_4i64_offset2:
1381; AVX1:       # BB#0: # %entry
1382; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1383; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
1384; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1385; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1386; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1387; AVX1-NEXT:    retq
1388;
1389; AVX2-LABEL: shuf_zext_8i16_to_4i64_offset2:
1390; AVX2:       # BB#0: # %entry
1391; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,2,2,3]
1392; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1393; AVX2-NEXT:    retq
1394;
1395; AVX512-LABEL: shuf_zext_8i16_to_4i64_offset2:
1396; AVX512:       # BB#0: # %entry
1397; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,2,2,3]
1398; AVX512-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
1399; AVX512-NEXT:    retq
1400entry:
1401  %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> <i32 2, i32 8, i32 8, i32 8, i32 3, i32 8, i32 8, i32 8, i32 4, i32 8, i32 8, i32 8, i32 5, i32 8, i32 8, i32 8>
1402  %Z = bitcast <16 x i16> %B to <4 x i64>
1403  ret <4 x i64> %Z
1404}
1405
1406define <4 x i32> @shuf_zext_8i16_to_4i32_offset1(<8 x i16> %A) nounwind uwtable readnone ssp {
1407; SSE-LABEL: shuf_zext_8i16_to_4i32_offset1:
1408; SSE:       # BB#0: # %entry
1409; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1410; SSE-NEXT:    pxor %xmm1, %xmm1
1411; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1412; SSE-NEXT:    retq
1413;
1414; AVX-LABEL: shuf_zext_8i16_to_4i32_offset1:
1415; AVX:       # BB#0: # %entry
1416; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1417; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1418; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1419; AVX-NEXT:    retq
1420entry:
1421  %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <8 x i32> <i32 1, i32 8, i32 2, i32 8, i32 3, i32 8, i32 4, i32 8>
1422  %Z = bitcast <8 x i16> %B to <4 x i32>
1423  ret <4 x i32> %Z
1424}
1425
1426define <8 x i32> @shuf_zext_8i16_to_8i32_offset3(<8 x i16> %A) nounwind uwtable readnone ssp {
1427; SSE2-LABEL: shuf_zext_8i16_to_8i32_offset3:
1428; SSE2:       # BB#0: # %entry
1429; SSE2-NEXT:    movdqa %xmm0, %xmm1
1430; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1431; SSE2-NEXT:    pxor %xmm2, %xmm2
1432; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1433; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1434; SSE2-NEXT:    retq
1435;
1436; SSSE3-LABEL: shuf_zext_8i16_to_8i32_offset3:
1437; SSSE3:       # BB#0: # %entry
1438; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1439; SSSE3-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1440; SSSE3-NEXT:    pxor %xmm2, %xmm2
1441; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1442; SSSE3-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1443; SSSE3-NEXT:    retq
1444;
1445; SSE41-LABEL: shuf_zext_8i16_to_8i32_offset3:
1446; SSE41:       # BB#0: # %entry
1447; SSE41-NEXT:    movdqa %xmm0, %xmm1
1448; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
1449; SSE41-NEXT:    pxor %xmm2, %xmm2
1450; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1451; SSE41-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1452; SSE41-NEXT:    retq
1453;
1454; AVX1-LABEL: shuf_zext_8i16_to_8i32_offset3:
1455; AVX1:       # BB#0: # %entry
1456; AVX1-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
1457; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1458; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1459; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1460; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1461; AVX1-NEXT:    retq
1462;
1463; AVX2-LABEL: shuf_zext_8i16_to_8i32_offset3:
1464; AVX2:       # BB#0: # %entry
1465; AVX2-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1466; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1467; AVX2-NEXT:    retq
1468;
1469; AVX512-LABEL: shuf_zext_8i16_to_8i32_offset3:
1470; AVX512:       # BB#0: # %entry
1471; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
1472; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1473; AVX512-NEXT:    retq
1474entry:
1475  %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> <i32 3, i32 8, i32 4, i32 8, i32 5, i32 8, i32 6, i32 8, i32 7, i32 8, i32 undef, i32 8, i32 undef, i32 8, i32 undef, i32 8>
1476  %Z = bitcast <16 x i16> %B to <8 x i32>
1477  ret <8 x i32> %Z
1478}
1479
1480define <8 x i32> @shuf_zext_16i16_to_8i32_offset8(<16 x i16> %A) nounwind uwtable readnone ssp {
1481; SSE2-LABEL: shuf_zext_16i16_to_8i32_offset8:
1482; SSE2:       # BB#0: # %entry
1483; SSE2-NEXT:    pxor %xmm2, %xmm2
1484; SSE2-NEXT:    movdqa %xmm1, %xmm0
1485; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1486; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1487; SSE2-NEXT:    retq
1488;
1489; SSSE3-LABEL: shuf_zext_16i16_to_8i32_offset8:
1490; SSSE3:       # BB#0: # %entry
1491; SSSE3-NEXT:    pxor %xmm2, %xmm2
1492; SSSE3-NEXT:    movdqa %xmm1, %xmm0
1493; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1494; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1495; SSSE3-NEXT:    retq
1496;
1497; SSE41-LABEL: shuf_zext_16i16_to_8i32_offset8:
1498; SSE41:       # BB#0: # %entry
1499; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
1500; SSE41-NEXT:    pxor %xmm2, %xmm2
1501; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
1502; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
1503; SSE41-NEXT:    movdqa %xmm2, %xmm1
1504; SSE41-NEXT:    retq
1505;
1506; AVX1-LABEL: shuf_zext_16i16_to_8i32_offset8:
1507; AVX1:       # BB#0: # %entry
1508; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1509; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
1510; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1511; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
1512; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1513; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1514; AVX1-NEXT:    retq
1515;
1516; AVX2-LABEL: shuf_zext_16i16_to_8i32_offset8:
1517; AVX2:       # BB#0: # %entry
1518; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
1519; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1520; AVX2-NEXT:    retq
1521;
1522; AVX512-LABEL: shuf_zext_16i16_to_8i32_offset8:
1523; AVX512:       # BB#0: # %entry
1524; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
1525; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1526; AVX512-NEXT:    retq
1527entry:
1528  %B = shufflevector <16 x i16> %A, <16 x i16> zeroinitializer, <16 x i32> <i32 8, i32 16, i32 9, i32 16, i32 10, i32 16, i32 11, i32 16, i32 12, i32 16, i32 undef, i32 16, i32 14, i32 16, i32 undef, i32 16>
1529  %Z = bitcast <16 x i16> %B to <8 x i32>
1530  ret <8 x i32> %Z
1531}
1532
1533define <2 x i64> @shuf_zext_4i32_to_2i64_offset2(<4 x i32> %A) nounwind uwtable readnone ssp {
1534; SSE-LABEL: shuf_zext_4i32_to_2i64_offset2:
1535; SSE:       # BB#0: # %entry
1536; SSE-NEXT:    pxor %xmm1, %xmm1
1537; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1538; SSE-NEXT:    retq
1539;
1540; AVX-LABEL: shuf_zext_4i32_to_2i64_offset2:
1541; AVX:       # BB#0: # %entry
1542; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1543; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1544; AVX-NEXT:    retq
1545entry:
1546  %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <4 x i32> <i32 2, i32 4, i32 3, i32 4>
1547  %Z = bitcast <4 x i32> %B to <2 x i64>
1548  ret <2 x i64> %Z
1549}
1550
1551define <4 x i64> @shuf_zext_4i32_to_4i64_offset1(<4 x i32> %A) nounwind uwtable readnone ssp {
1552; SSE2-LABEL: shuf_zext_4i32_to_4i64_offset1:
1553; SSE2:       # BB#0: # %entry
1554; SSE2-NEXT:    movdqa %xmm0, %xmm1
1555; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,0,4294967295,0]
1556; SSE2-NEXT:    pand %xmm1, %xmm0
1557; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1558; SSE2-NEXT:    retq
1559;
1560; SSSE3-LABEL: shuf_zext_4i32_to_4i64_offset1:
1561; SSSE3:       # BB#0: # %entry
1562; SSSE3-NEXT:    movdqa %xmm0, %xmm1
1563; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,0,4294967295,0]
1564; SSSE3-NEXT:    pand %xmm1, %xmm0
1565; SSSE3-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1566; SSSE3-NEXT:    retq
1567;
1568; SSE41-LABEL: shuf_zext_4i32_to_4i64_offset1:
1569; SSE41:       # BB#0: # %entry
1570; SSE41-NEXT:    movdqa %xmm0, %xmm1
1571; SSE41-NEXT:    pxor %xmm0, %xmm0
1572; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5],xmm0[6,7]
1573; SSE41-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1574; SSE41-NEXT:    retq
1575;
1576; AVX1-LABEL: shuf_zext_4i32_to_4i64_offset1:
1577; AVX1:       # BB#0: # %entry
1578; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[3],zero,zero,zero
1579; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
1580; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
1581; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1582; AVX1-NEXT:    retq
1583;
1584; AVX2-LABEL: shuf_zext_4i32_to_4i64_offset1:
1585; AVX2:       # BB#0: # %entry
1586; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,2,3,3]
1587; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1588; AVX2-NEXT:    retq
1589;
1590; AVX512-LABEL: shuf_zext_4i32_to_4i64_offset1:
1591; AVX512:       # BB#0: # %entry
1592; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,2,3,3]
1593; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1594; AVX512-NEXT:    retq
1595entry:
1596  %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <8 x i32> <i32 undef, i32 4, i32 2, i32 4, i32 3, i32 4, i32 undef, i32 4>
1597  %Z = bitcast <8 x i32> %B to <4 x i64>
1598  ret <4 x i64> %Z
1599}
1600