1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512VL --check-prefix=AVX512DQVL
9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512VL --check-prefix=AVX512BWVL
10;
11; 32-bit runs to make sure we do reasonable things for i64 shifts.
12; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=ALL --check-prefix=X32-AVX --check-prefix=X32-AVX1
13; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-AVX --check-prefix=X32-AVX2
14
15;
16; Variable Shifts
17;
18
19define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
20; AVX1-LABEL: var_shift_v4i64:
21; AVX1:       # %bb.0:
22; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
23; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
24; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
25; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
26; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
27; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
28; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
29; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
30; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
31; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
32; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
33; AVX1-NEXT:    retq
34;
35; AVX2-LABEL: var_shift_v4i64:
36; AVX2:       # %bb.0:
37; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
38; AVX2-NEXT:    retq
39;
40; XOPAVX1-LABEL: var_shift_v4i64:
41; XOPAVX1:       # %bb.0:
42; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
43; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
44; XOPAVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
45; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
46; XOPAVX1-NEXT:    vpshlq %xmm2, %xmm4, %xmm2
47; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm3, %xmm1
48; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
49; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
50; XOPAVX1-NEXT:    retq
51;
52; XOPAVX2-LABEL: var_shift_v4i64:
53; XOPAVX2:       # %bb.0:
54; XOPAVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
55; XOPAVX2-NEXT:    retq
56;
57; AVX512-LABEL: var_shift_v4i64:
58; AVX512:       # %bb.0:
59; AVX512-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
60; AVX512-NEXT:    retq
61;
62; AVX512VL-LABEL: var_shift_v4i64:
63; AVX512VL:       # %bb.0:
64; AVX512VL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
65; AVX512VL-NEXT:    retq
66;
67; X32-AVX1-LABEL: var_shift_v4i64:
68; X32-AVX1:       # %bb.0:
69; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
70; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
71; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
72; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
73; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
74; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
75; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
76; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
77; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
78; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
79; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
80; X32-AVX1-NEXT:    retl
81;
82; X32-AVX2-LABEL: var_shift_v4i64:
83; X32-AVX2:       # %bb.0:
84; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
85; X32-AVX2-NEXT:    retl
86  %shift = lshr <4 x i64> %a, %b
87  ret <4 x i64> %shift
88}
89
90define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
91; AVX1-LABEL: var_shift_v8i32:
92; AVX1:       # %bb.0:
93; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
94; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
95; AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
96; AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
97; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
98; AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
99; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
100; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
101; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
102; AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
103; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
104; AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
105; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
106; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
107; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
108; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
109; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
110; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
111; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
112; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
113; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
114; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
115; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
116; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
117; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
118; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
119; AVX1-NEXT:    retq
120;
121; AVX2-LABEL: var_shift_v8i32:
122; AVX2:       # %bb.0:
123; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
124; AVX2-NEXT:    retq
125;
126; XOPAVX1-LABEL: var_shift_v8i32:
127; XOPAVX1:       # %bb.0:
128; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
129; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
130; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
131; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
132; XOPAVX1-NEXT:    vpshld %xmm2, %xmm4, %xmm2
133; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
134; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
135; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
136; XOPAVX1-NEXT:    retq
137;
138; XOPAVX2-LABEL: var_shift_v8i32:
139; XOPAVX2:       # %bb.0:
140; XOPAVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
141; XOPAVX2-NEXT:    retq
142;
143; AVX512-LABEL: var_shift_v8i32:
144; AVX512:       # %bb.0:
145; AVX512-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
146; AVX512-NEXT:    retq
147;
148; AVX512VL-LABEL: var_shift_v8i32:
149; AVX512VL:       # %bb.0:
150; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
151; AVX512VL-NEXT:    retq
152;
153; X32-AVX1-LABEL: var_shift_v8i32:
154; X32-AVX1:       # %bb.0:
155; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
156; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
157; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
158; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
159; X32-AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
160; X32-AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
161; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
162; X32-AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
163; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
164; X32-AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
165; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
166; X32-AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
167; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
168; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
169; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
170; X32-AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
171; X32-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
172; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
173; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
174; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
175; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
176; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
177; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
178; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
179; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
180; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
181; X32-AVX1-NEXT:    retl
182;
183; X32-AVX2-LABEL: var_shift_v8i32:
184; X32-AVX2:       # %bb.0:
185; X32-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
186; X32-AVX2-NEXT:    retl
187  %shift = lshr <8 x i32> %a, %b
188  ret <8 x i32> %shift
189}
190
191define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
192; AVX1-LABEL: var_shift_v16i16:
193; AVX1:       # %bb.0:
194; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
195; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
196; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
197; AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
198; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
199; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
200; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
201; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
202; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
203; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
204; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
205; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
206; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
207; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
208; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
209; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
210; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
211; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
212; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
213; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
214; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
215; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
216; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
217; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
218; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
219; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
220; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
221; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
222; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
223; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
224; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
225; AVX1-NEXT:    retq
226;
227; AVX2-LABEL: var_shift_v16i16:
228; AVX2:       # %bb.0:
229; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
230; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
231; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
232; AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
233; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
234; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
235; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
236; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
237; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
238; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
239; AVX2-NEXT:    retq
240;
241; XOPAVX1-LABEL: var_shift_v16i16:
242; XOPAVX1:       # %bb.0:
243; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
244; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
245; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
246; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
247; XOPAVX1-NEXT:    vpshlw %xmm2, %xmm4, %xmm2
248; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
249; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
250; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
251; XOPAVX1-NEXT:    retq
252;
253; XOPAVX2-LABEL: var_shift_v16i16:
254; XOPAVX2:       # %bb.0:
255; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
256; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
257; XOPAVX2-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
258; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
259; XOPAVX2-NEXT:    vpshlw %xmm2, %xmm4, %xmm2
260; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
261; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
262; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
263; XOPAVX2-NEXT:    retq
264;
265; AVX512DQ-LABEL: var_shift_v16i16:
266; AVX512DQ:       # %bb.0:
267; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
268; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
269; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
270; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
271; AVX512DQ-NEXT:    retq
272;
273; AVX512BW-LABEL: var_shift_v16i16:
274; AVX512BW:       # %bb.0:
275; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
276; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
277; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
278; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
279; AVX512BW-NEXT:    retq
280;
281; AVX512DQVL-LABEL: var_shift_v16i16:
282; AVX512DQVL:       # %bb.0:
283; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
284; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
285; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
286; AVX512DQVL-NEXT:    vpmovdw %zmm0, %ymm0
287; AVX512DQVL-NEXT:    retq
288;
289; AVX512BWVL-LABEL: var_shift_v16i16:
290; AVX512BWVL:       # %bb.0:
291; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
292; AVX512BWVL-NEXT:    retq
293;
294; X32-AVX1-LABEL: var_shift_v16i16:
295; X32-AVX1:       # %bb.0:
296; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
297; X32-AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
298; X32-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
299; X32-AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
300; X32-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
301; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
302; X32-AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
303; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
304; X32-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
305; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
306; X32-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
307; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
308; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
309; X32-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
310; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
311; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
312; X32-AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
313; X32-AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
314; X32-AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
315; X32-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
316; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
317; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
318; X32-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
319; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
320; X32-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
321; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
322; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
323; X32-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
324; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
325; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
326; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
327; X32-AVX1-NEXT:    retl
328;
329; X32-AVX2-LABEL: var_shift_v16i16:
330; X32-AVX2:       # %bb.0:
331; X32-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
332; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
333; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
334; X32-AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
335; X32-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
336; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
337; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
338; X32-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
339; X32-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
340; X32-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
341; X32-AVX2-NEXT:    retl
342  %shift = lshr <16 x i16> %a, %b
343  ret <16 x i16> %shift
344}
345
346define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
347; AVX1-LABEL: var_shift_v32i8:
348; AVX1:       # %bb.0:
349; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
350; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
351; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
352; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
353; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
354; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
355; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
356; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
357; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
358; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
359; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
360; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
361; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
362; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
363; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
364; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
365; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
366; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
367; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
368; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
369; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
370; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
371; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
372; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
373; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
374; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
375; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
376; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
377; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
378; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
379; AVX1-NEXT:    retq
380;
381; AVX2-LABEL: var_shift_v32i8:
382; AVX2:       # %bb.0:
383; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
384; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
385; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
386; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
387; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
388; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
389; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
390; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
391; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
392; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
393; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
394; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
395; AVX2-NEXT:    retq
396;
397; XOPAVX1-LABEL: var_shift_v32i8:
398; XOPAVX1:       # %bb.0:
399; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
400; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
401; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
402; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
403; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
404; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
405; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
406; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
407; XOPAVX1-NEXT:    retq
408;
409; XOPAVX2-LABEL: var_shift_v32i8:
410; XOPAVX2:       # %bb.0:
411; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
412; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
413; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
414; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
415; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
416; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
417; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
418; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
419; XOPAVX2-NEXT:    retq
420;
421; AVX512DQ-LABEL: var_shift_v32i8:
422; AVX512DQ:       # %bb.0:
423; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
424; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm2
425; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
426; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
427; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm2
428; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
429; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
430; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
431; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm2
432; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
433; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
434; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
435; AVX512DQ-NEXT:    retq
436;
437; AVX512BW-LABEL: var_shift_v32i8:
438; AVX512BW:       # %bb.0:
439; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
440; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
441; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
442; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
443; AVX512BW-NEXT:    retq
444;
445; AVX512DQVL-LABEL: var_shift_v32i8:
446; AVX512DQVL:       # %bb.0:
447; AVX512DQVL-NEXT:    vpsllw $5, %ymm1, %ymm1
448; AVX512DQVL-NEXT:    vpsrlw $4, %ymm0, %ymm2
449; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
450; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
451; AVX512DQVL-NEXT:    vpsrlw $2, %ymm0, %ymm2
452; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
453; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
454; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
455; AVX512DQVL-NEXT:    vpsrlw $1, %ymm0, %ymm2
456; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
457; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
458; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
459; AVX512DQVL-NEXT:    retq
460;
461; AVX512BWVL-LABEL: var_shift_v32i8:
462; AVX512BWVL:       # %bb.0:
463; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
464; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
465; AVX512BWVL-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
466; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
467; AVX512BWVL-NEXT:    retq
468;
469; X32-AVX1-LABEL: var_shift_v32i8:
470; X32-AVX1:       # %bb.0:
471; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
472; X32-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
473; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
474; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
475; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
476; X32-AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
477; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
478; X32-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
479; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
480; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
481; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
482; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
483; X32-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
484; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
485; X32-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
486; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
487; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
488; X32-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
489; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
490; X32-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
491; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
492; X32-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
493; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
494; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
495; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
496; X32-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
497; X32-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
498; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
499; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
500; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
501; X32-AVX1-NEXT:    retl
502;
503; X32-AVX2-LABEL: var_shift_v32i8:
504; X32-AVX2:       # %bb.0:
505; X32-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
506; X32-AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
507; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
508; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
509; X32-AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
510; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
511; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
512; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
513; X32-AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
514; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
515; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
516; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
517; X32-AVX2-NEXT:    retl
518  %shift = lshr <32 x i8> %a, %b
519  ret <32 x i8> %shift
520}
521
522;
523; Uniform Variable Shifts
524;
525
526define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
527; AVX1-LABEL: splatvar_shift_v4i64:
528; AVX1:       # %bb.0:
529; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
530; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
531; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
532; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
533; AVX1-NEXT:    retq
534;
535; AVX2-LABEL: splatvar_shift_v4i64:
536; AVX2:       # %bb.0:
537; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
538; AVX2-NEXT:    retq
539;
540; XOPAVX1-LABEL: splatvar_shift_v4i64:
541; XOPAVX1:       # %bb.0:
542; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
543; XOPAVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
544; XOPAVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
545; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
546; XOPAVX1-NEXT:    retq
547;
548; XOPAVX2-LABEL: splatvar_shift_v4i64:
549; XOPAVX2:       # %bb.0:
550; XOPAVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
551; XOPAVX2-NEXT:    retq
552;
553; AVX512-LABEL: splatvar_shift_v4i64:
554; AVX512:       # %bb.0:
555; AVX512-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
556; AVX512-NEXT:    retq
557;
558; AVX512VL-LABEL: splatvar_shift_v4i64:
559; AVX512VL:       # %bb.0:
560; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
561; AVX512VL-NEXT:    retq
562;
563; X32-AVX1-LABEL: splatvar_shift_v4i64:
564; X32-AVX1:       # %bb.0:
565; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
566; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
567; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
568; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
569; X32-AVX1-NEXT:    retl
570;
571; X32-AVX2-LABEL: splatvar_shift_v4i64:
572; X32-AVX2:       # %bb.0:
573; X32-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
574; X32-AVX2-NEXT:    retl
575  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
576  %shift = lshr <4 x i64> %a, %splat
577  ret <4 x i64> %shift
578}
579
580define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
581; AVX1-LABEL: splatvar_shift_v8i32:
582; AVX1:       # %bb.0:
583; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
584; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
585; AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
586; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
587; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
588; AVX1-NEXT:    retq
589;
590; AVX2-LABEL: splatvar_shift_v8i32:
591; AVX2:       # %bb.0:
592; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
593; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
594; AVX2-NEXT:    retq
595;
596; XOPAVX1-LABEL: splatvar_shift_v8i32:
597; XOPAVX1:       # %bb.0:
598; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
599; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
600; XOPAVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
601; XOPAVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
602; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
603; XOPAVX1-NEXT:    retq
604;
605; XOPAVX2-LABEL: splatvar_shift_v8i32:
606; XOPAVX2:       # %bb.0:
607; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
608; XOPAVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
609; XOPAVX2-NEXT:    retq
610;
611; AVX512-LABEL: splatvar_shift_v8i32:
612; AVX512:       # %bb.0:
613; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
614; AVX512-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
615; AVX512-NEXT:    retq
616;
617; AVX512VL-LABEL: splatvar_shift_v8i32:
618; AVX512VL:       # %bb.0:
619; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
620; AVX512VL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
621; AVX512VL-NEXT:    retq
622;
623; X32-AVX1-LABEL: splatvar_shift_v8i32:
624; X32-AVX1:       # %bb.0:
625; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
626; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
627; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
628; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
629; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
630; X32-AVX1-NEXT:    retl
631;
632; X32-AVX2-LABEL: splatvar_shift_v8i32:
633; X32-AVX2:       # %bb.0:
634; X32-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
635; X32-AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
636; X32-AVX2-NEXT:    retl
637  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
638  %shift = lshr <8 x i32> %a, %splat
639  ret <8 x i32> %shift
640}
641
642define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
643; AVX1-LABEL: splatvar_shift_v16i16:
644; AVX1:       # %bb.0:
645; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
646; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
647; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
648; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
649; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
650; AVX1-NEXT:    retq
651;
652; AVX2-LABEL: splatvar_shift_v16i16:
653; AVX2:       # %bb.0:
654; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
655; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
656; AVX2-NEXT:    retq
657;
658; XOPAVX1-LABEL: splatvar_shift_v16i16:
659; XOPAVX1:       # %bb.0:
660; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
661; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
662; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
663; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
664; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
665; XOPAVX1-NEXT:    retq
666;
667; XOPAVX2-LABEL: splatvar_shift_v16i16:
668; XOPAVX2:       # %bb.0:
669; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
670; XOPAVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
671; XOPAVX2-NEXT:    retq
672;
673; AVX512-LABEL: splatvar_shift_v16i16:
674; AVX512:       # %bb.0:
675; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
676; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
677; AVX512-NEXT:    retq
678;
679; AVX512VL-LABEL: splatvar_shift_v16i16:
680; AVX512VL:       # %bb.0:
681; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
682; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
683; AVX512VL-NEXT:    retq
684;
685; X32-AVX1-LABEL: splatvar_shift_v16i16:
686; X32-AVX1:       # %bb.0:
687; X32-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
688; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
689; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
690; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
691; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
692; X32-AVX1-NEXT:    retl
693;
694; X32-AVX2-LABEL: splatvar_shift_v16i16:
695; X32-AVX2:       # %bb.0:
696; X32-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
697; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
698; X32-AVX2-NEXT:    retl
699  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
700  %shift = lshr <16 x i16> %a, %splat
701  ret <16 x i16> %shift
702}
703
704define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
705; AVX1-LABEL: splatvar_shift_v32i8:
706; AVX1:       # %bb.0:
707; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
708; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
709; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
710; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
711; AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
712; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
713; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
714; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
715; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
716; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
717; AVX1-NEXT:    retq
718;
719; AVX2-LABEL: splatvar_shift_v32i8:
720; AVX2:       # %bb.0:
721; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
722; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
723; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
724; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
725; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
726; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
727; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
728; AVX2-NEXT:    retq
729;
730; XOPAVX1-LABEL: splatvar_shift_v32i8:
731; XOPAVX1:       # %bb.0:
732; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
733; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
734; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
735; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
736; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm2, %xmm2
737; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
738; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
739; XOPAVX1-NEXT:    retq
740;
741; XOPAVX2-LABEL: splatvar_shift_v32i8:
742; XOPAVX2:       # %bb.0:
743; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
744; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
745; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
746; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
747; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
748; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
749; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
750; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
751; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
752; XOPAVX2-NEXT:    retq
753;
754; AVX512DQ-LABEL: splatvar_shift_v32i8:
755; AVX512DQ:       # %bb.0:
756; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
757; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
758; AVX512DQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
759; AVX512DQ-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
760; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
761; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
762; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
763; AVX512DQ-NEXT:    retq
764;
765; AVX512BW-LABEL: splatvar_shift_v32i8:
766; AVX512BW:       # %bb.0:
767; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
768; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
769; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
770; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
771; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
772; AVX512BW-NEXT:    retq
773;
774; AVX512DQVL-LABEL: splatvar_shift_v32i8:
775; AVX512DQVL:       # %bb.0:
776; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
777; AVX512DQVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
778; AVX512DQVL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
779; AVX512DQVL-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
780; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
781; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %ymm1
782; AVX512DQVL-NEXT:    vpand %ymm1, %ymm0, %ymm0
783; AVX512DQVL-NEXT:    retq
784;
785; AVX512BWVL-LABEL: splatvar_shift_v32i8:
786; AVX512BWVL:       # %bb.0:
787; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %ymm1
788; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
789; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
790; AVX512BWVL-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
791; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
792; AVX512BWVL-NEXT:    retq
793;
794; X32-AVX1-LABEL: splatvar_shift_v32i8:
795; X32-AVX1:       # %bb.0:
796; X32-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
797; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
798; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
799; X32-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
800; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
801; X32-AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
802; X32-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
803; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
804; X32-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
805; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
806; X32-AVX1-NEXT:    retl
807;
808; X32-AVX2-LABEL: splatvar_shift_v32i8:
809; X32-AVX2:       # %bb.0:
810; X32-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
811; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
812; X32-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
813; X32-AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
814; X32-AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
815; X32-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
816; X32-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
817; X32-AVX2-NEXT:    retl
818  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
819  %shift = lshr <32 x i8> %a, %splat
820  ret <32 x i8> %shift
821}
822
823;
824; Constant Shifts
825;
826
827define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
828; AVX1-LABEL: constant_shift_v4i64:
829; AVX1:       # %bb.0:
830; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
831; AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
832; AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
833; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
834; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
835; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
836; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
837; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
838; AVX1-NEXT:    retq
839;
840; AVX2-LABEL: constant_shift_v4i64:
841; AVX2:       # %bb.0:
842; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
843; AVX2-NEXT:    retq
844;
845; XOPAVX1-LABEL: constant_shift_v4i64:
846; XOPAVX1:       # %bb.0:
847; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm1
848; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
849; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
850; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
851; XOPAVX1-NEXT:    retq
852;
853; XOPAVX2-LABEL: constant_shift_v4i64:
854; XOPAVX2:       # %bb.0:
855; XOPAVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
856; XOPAVX2-NEXT:    retq
857;
858; AVX512-LABEL: constant_shift_v4i64:
859; AVX512:       # %bb.0:
860; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
861; AVX512-NEXT:    retq
862;
863; AVX512VL-LABEL: constant_shift_v4i64:
864; AVX512VL:       # %bb.0:
865; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
866; AVX512VL-NEXT:    retq
867;
868; X32-AVX1-LABEL: constant_shift_v4i64:
869; X32-AVX1:       # %bb.0:
870; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
871; X32-AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
872; X32-AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
873; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
874; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
875; X32-AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
876; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
877; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
878; X32-AVX1-NEXT:    retl
879;
880; X32-AVX2-LABEL: constant_shift_v4i64:
881; X32-AVX2:       # %bb.0:
882; X32-AVX2-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0
883; X32-AVX2-NEXT:    retl
884  %shift = lshr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
885  ret <4 x i64> %shift
886}
887
888define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
889; AVX1-LABEL: constant_shift_v8i32:
890; AVX1:       # %bb.0:
891; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
892; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
893; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
894; AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
895; AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
896; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
897; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
898; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
899; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
900; AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
901; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
902; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
903; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
904; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
905; AVX1-NEXT:    retq
906;
907; AVX2-LABEL: constant_shift_v8i32:
908; AVX2:       # %bb.0:
909; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
910; AVX2-NEXT:    retq
911;
912; XOPAVX1-LABEL: constant_shift_v8i32:
913; XOPAVX1:       # %bb.0:
914; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm1
915; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
916; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
917; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
918; XOPAVX1-NEXT:    retq
919;
920; XOPAVX2-LABEL: constant_shift_v8i32:
921; XOPAVX2:       # %bb.0:
922; XOPAVX2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
923; XOPAVX2-NEXT:    retq
924;
925; AVX512-LABEL: constant_shift_v8i32:
926; AVX512:       # %bb.0:
927; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
928; AVX512-NEXT:    retq
929;
930; AVX512VL-LABEL: constant_shift_v8i32:
931; AVX512VL:       # %bb.0:
932; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
933; AVX512VL-NEXT:    retq
934;
935; X32-AVX1-LABEL: constant_shift_v8i32:
936; X32-AVX1:       # %bb.0:
937; X32-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
938; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
939; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
940; X32-AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
941; X32-AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
942; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
943; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
944; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
945; X32-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
946; X32-AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
947; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
948; X32-AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
949; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
950; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
951; X32-AVX1-NEXT:    retl
952;
953; X32-AVX2-LABEL: constant_shift_v8i32:
954; X32-AVX2:       # %bb.0:
955; X32-AVX2-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0
956; X32-AVX2-NEXT:    retl
957  %shift = lshr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
958  ret <8 x i32> %shift
959}
960
961define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
962; AVX1-LABEL: constant_shift_v16i16:
963; AVX1:       # %bb.0:
964; AVX1-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
965; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
966; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
967; AVX1-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm0
968; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
969; AVX1-NEXT:    retq
970;
971; AVX2-LABEL: constant_shift_v16i16:
972; AVX2:       # %bb.0:
973; AVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
974; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
975; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
976; AVX2-NEXT:    retq
977;
978; XOPAVX1-LABEL: constant_shift_v16i16:
979; XOPAVX1:       # %bb.0:
980; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm1
981; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
982; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
983; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
984; XOPAVX1-NEXT:    retq
985;
986; XOPAVX2-LABEL: constant_shift_v16i16:
987; XOPAVX2:       # %bb.0:
988; XOPAVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
989; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
990; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
991; XOPAVX2-NEXT:    retq
992;
993; AVX512DQ-LABEL: constant_shift_v16i16:
994; AVX512DQ:       # %bb.0:
995; AVX512DQ-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
996; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
997; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
998; AVX512DQ-NEXT:    retq
999;
1000; AVX512BW-LABEL: constant_shift_v16i16:
1001; AVX512BW:       # %bb.0:
1002; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
1003; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
1004; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
1005; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
1006; AVX512BW-NEXT:    retq
1007;
1008; AVX512DQVL-LABEL: constant_shift_v16i16:
1009; AVX512DQVL:       # %bb.0:
1010; AVX512DQVL-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
1011; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
1012; AVX512DQVL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1013; AVX512DQVL-NEXT:    retq
1014;
1015; AVX512BWVL-LABEL: constant_shift_v16i16:
1016; AVX512BWVL:       # %bb.0:
1017; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
1018; AVX512BWVL-NEXT:    retq
1019;
1020; X32-AVX1-LABEL: constant_shift_v16i16:
1021; X32-AVX1:       # %bb.0:
1022; X32-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm1
1023; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
1024; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1025; X32-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm0
1026; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1027; X32-AVX1-NEXT:    retl
1028;
1029; X32-AVX2-LABEL: constant_shift_v16i16:
1030; X32-AVX2:       # %bb.0:
1031; X32-AVX2-NEXT:    vpmulhuw {{\.LCPI.*}}, %ymm0, %ymm1
1032; X32-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
1033; X32-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1034; X32-AVX2-NEXT:    retl
1035  %shift = lshr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
1036  ret <16 x i16> %shift
1037}
1038
1039define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
1040; AVX1-LABEL: constant_shift_v32i8:
1041; AVX1:       # %bb.0:
1042; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1043; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1044; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1045; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,4,8,16,32,64,128,256]
1046; AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
1047; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
1048; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1049; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [256,128,64,32,16,8,4,2]
1050; AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
1051; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
1052; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
1053; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1054; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
1055; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1056; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1057; AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
1058; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1059; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
1060; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1061; AVX1-NEXT:    retq
1062;
1063; AVX2-LABEL: constant_shift_v32i8:
1064; AVX2:       # %bb.0:
1065; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1066; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
1067; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
1068; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1069; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
1070; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
1071; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1072; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
1073; AVX2-NEXT:    retq
1074;
1075; XOPAVX1-LABEL: constant_shift_v32i8:
1076; XOPAVX1:       # %bb.0:
1077; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1078; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,249,250,251,252,253,254,255,0]
1079; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm1, %xmm1
1080; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm0, %xmm0
1081; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1082; XOPAVX1-NEXT:    retq
1083;
1084; XOPAVX2-LABEL: constant_shift_v32i8:
1085; XOPAVX2:       # %bb.0:
1086; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1087; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,249,250,251,252,253,254,255,0]
1088; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm1, %xmm1
1089; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm0, %xmm0
1090; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
1091; XOPAVX2-NEXT:    retq
1092;
1093; AVX512DQ-LABEL: constant_shift_v32i8:
1094; AVX512DQ:       # %bb.0:
1095; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1096; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
1097; AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
1098; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
1099; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
1100; AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
1101; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
1102; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
1103; AVX512DQ-NEXT:    retq
1104;
1105; AVX512BW-LABEL: constant_shift_v32i8:
1106; AVX512BW:       # %bb.0:
1107; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1108; AVX512BW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
1109; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
1110; AVX512BW-NEXT:    retq
1111;
1112; AVX512DQVL-LABEL: constant_shift_v32i8:
1113; AVX512DQVL:       # %bb.0:
1114; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1115; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
1116; AVX512DQVL-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
1117; AVX512DQVL-NEXT:    vpsrlw $8, %ymm2, %ymm2
1118; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
1119; AVX512DQVL-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
1120; AVX512DQVL-NEXT:    vpsrlw $8, %ymm0, %ymm0
1121; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
1122; AVX512DQVL-NEXT:    retq
1123;
1124; AVX512BWVL-LABEL: constant_shift_v32i8:
1125; AVX512BWVL:       # %bb.0:
1126; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
1127; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
1128; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
1129; AVX512BWVL-NEXT:    retq
1130;
1131; X32-AVX1-LABEL: constant_shift_v32i8:
1132; X32-AVX1:       # %bb.0:
1133; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1134; X32-AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1135; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
1136; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,4,8,16,32,64,128,256]
1137; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
1138; X32-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
1139; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1140; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [256,128,64,32,16,8,4,2]
1141; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
1142; X32-AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
1143; X32-AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
1144; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1145; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
1146; X32-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
1147; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1148; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
1149; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
1150; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
1151; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1152; X32-AVX1-NEXT:    retl
1153;
1154; X32-AVX2-LABEL: constant_shift_v32i8:
1155; X32-AVX2:       # %bb.0:
1156; X32-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1157; X32-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
1158; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm2, %ymm2
1159; X32-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
1160; X32-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
1161; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
1162; X32-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
1163; X32-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
1164; X32-AVX2-NEXT:    retl
1165  %shift = lshr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
1166  ret <32 x i8> %shift
1167}
1168
1169;
1170; Uniform Constant Shifts
1171;
1172
1173define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
1174; AVX1-LABEL: splatconstant_shift_v4i64:
1175; AVX1:       # %bb.0:
1176; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
1177; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1178; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
1179; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1180; AVX1-NEXT:    retq
1181;
1182; AVX2-LABEL: splatconstant_shift_v4i64:
1183; AVX2:       # %bb.0:
1184; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
1185; AVX2-NEXT:    retq
1186;
1187; XOPAVX1-LABEL: splatconstant_shift_v4i64:
1188; XOPAVX1:       # %bb.0:
1189; XOPAVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
1190; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1191; XOPAVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
1192; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1193; XOPAVX1-NEXT:    retq
1194;
1195; XOPAVX2-LABEL: splatconstant_shift_v4i64:
1196; XOPAVX2:       # %bb.0:
1197; XOPAVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
1198; XOPAVX2-NEXT:    retq
1199;
1200; AVX512-LABEL: splatconstant_shift_v4i64:
1201; AVX512:       # %bb.0:
1202; AVX512-NEXT:    vpsrlq $7, %ymm0, %ymm0
1203; AVX512-NEXT:    retq
1204;
1205; AVX512VL-LABEL: splatconstant_shift_v4i64:
1206; AVX512VL:       # %bb.0:
1207; AVX512VL-NEXT:    vpsrlq $7, %ymm0, %ymm0
1208; AVX512VL-NEXT:    retq
1209;
1210; X32-AVX1-LABEL: splatconstant_shift_v4i64:
1211; X32-AVX1:       # %bb.0:
1212; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
1213; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1214; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
1215; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1216; X32-AVX1-NEXT:    retl
1217;
1218; X32-AVX2-LABEL: splatconstant_shift_v4i64:
1219; X32-AVX2:       # %bb.0:
1220; X32-AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
1221; X32-AVX2-NEXT:    retl
1222  %shift = lshr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
1223  ret <4 x i64> %shift
1224}
1225
1226define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
1227; AVX1-LABEL: splatconstant_shift_v8i32:
1228; AVX1:       # %bb.0:
1229; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
1230; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1231; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
1232; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1233; AVX1-NEXT:    retq
1234;
1235; AVX2-LABEL: splatconstant_shift_v8i32:
1236; AVX2:       # %bb.0:
1237; AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
1238; AVX2-NEXT:    retq
1239;
1240; XOPAVX1-LABEL: splatconstant_shift_v8i32:
1241; XOPAVX1:       # %bb.0:
1242; XOPAVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
1243; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1244; XOPAVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
1245; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1246; XOPAVX1-NEXT:    retq
1247;
1248; XOPAVX2-LABEL: splatconstant_shift_v8i32:
1249; XOPAVX2:       # %bb.0:
1250; XOPAVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
1251; XOPAVX2-NEXT:    retq
1252;
1253; AVX512-LABEL: splatconstant_shift_v8i32:
1254; AVX512:       # %bb.0:
1255; AVX512-NEXT:    vpsrld $5, %ymm0, %ymm0
1256; AVX512-NEXT:    retq
1257;
1258; AVX512VL-LABEL: splatconstant_shift_v8i32:
1259; AVX512VL:       # %bb.0:
1260; AVX512VL-NEXT:    vpsrld $5, %ymm0, %ymm0
1261; AVX512VL-NEXT:    retq
1262;
1263; X32-AVX1-LABEL: splatconstant_shift_v8i32:
1264; X32-AVX1:       # %bb.0:
1265; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
1266; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1267; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
1268; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1269; X32-AVX1-NEXT:    retl
1270;
1271; X32-AVX2-LABEL: splatconstant_shift_v8i32:
1272; X32-AVX2:       # %bb.0:
1273; X32-AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
1274; X32-AVX2-NEXT:    retl
1275  %shift = lshr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
1276  ret <8 x i32> %shift
1277}
1278
1279define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
1280; AVX1-LABEL: splatconstant_shift_v16i16:
1281; AVX1:       # %bb.0:
1282; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
1283; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1284; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1285; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1286; AVX1-NEXT:    retq
1287;
1288; AVX2-LABEL: splatconstant_shift_v16i16:
1289; AVX2:       # %bb.0:
1290; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1291; AVX2-NEXT:    retq
1292;
1293; XOPAVX1-LABEL: splatconstant_shift_v16i16:
1294; XOPAVX1:       # %bb.0:
1295; XOPAVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
1296; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1297; XOPAVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1298; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1299; XOPAVX1-NEXT:    retq
1300;
1301; XOPAVX2-LABEL: splatconstant_shift_v16i16:
1302; XOPAVX2:       # %bb.0:
1303; XOPAVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1304; XOPAVX2-NEXT:    retq
1305;
1306; AVX512-LABEL: splatconstant_shift_v16i16:
1307; AVX512:       # %bb.0:
1308; AVX512-NEXT:    vpsrlw $3, %ymm0, %ymm0
1309; AVX512-NEXT:    retq
1310;
1311; AVX512VL-LABEL: splatconstant_shift_v16i16:
1312; AVX512VL:       # %bb.0:
1313; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
1314; AVX512VL-NEXT:    retq
1315;
1316; X32-AVX1-LABEL: splatconstant_shift_v16i16:
1317; X32-AVX1:       # %bb.0:
1318; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
1319; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1320; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1321; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1322; X32-AVX1-NEXT:    retl
1323;
1324; X32-AVX2-LABEL: splatconstant_shift_v16i16:
1325; X32-AVX2:       # %bb.0:
1326; X32-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1327; X32-AVX2-NEXT:    retl
1328  %shift = lshr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
1329  ret <16 x i16> %shift
1330}
1331
1332define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
1333; AVX1-LABEL: splatconstant_shift_v32i8:
1334; AVX1:       # %bb.0:
1335; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1336; AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
1337; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
1338; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
1339; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1340; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
1341; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1342; AVX1-NEXT:    retq
1343;
1344; AVX2-LABEL: splatconstant_shift_v32i8:
1345; AVX2:       # %bb.0:
1346; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1347; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1348; AVX2-NEXT:    retq
1349;
1350; XOPAVX1-LABEL: splatconstant_shift_v32i8:
1351; XOPAVX1:       # %bb.0:
1352; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1353; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253]
1354; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm1, %xmm1
1355; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm0, %xmm0
1356; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1357; XOPAVX1-NEXT:    retq
1358;
1359; XOPAVX2-LABEL: splatconstant_shift_v32i8:
1360; XOPAVX2:       # %bb.0:
1361; XOPAVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1362; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1363; XOPAVX2-NEXT:    retq
1364;
1365; AVX512-LABEL: splatconstant_shift_v32i8:
1366; AVX512:       # %bb.0:
1367; AVX512-NEXT:    vpsrlw $3, %ymm0, %ymm0
1368; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1369; AVX512-NEXT:    retq
1370;
1371; AVX512VL-LABEL: splatconstant_shift_v32i8:
1372; AVX512VL:       # %bb.0:
1373; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
1374; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1375; AVX512VL-NEXT:    retq
1376;
1377; X32-AVX1-LABEL: splatconstant_shift_v32i8:
1378; X32-AVX1:       # %bb.0:
1379; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1380; X32-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
1381; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
1382; X32-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
1383; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1384; X32-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
1385; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1386; X32-AVX1-NEXT:    retl
1387;
1388; X32-AVX2-LABEL: splatconstant_shift_v32i8:
1389; X32-AVX2:       # %bb.0:
1390; X32-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1391; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
1392; X32-AVX2-NEXT:    retl
1393  %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
1394  ret <32 x i8> %shift
1395}
1396
1397define <4 x i32> @sh_trunc_sh_vec(<4 x i64> %x) {
1398; AVX1-LABEL: sh_trunc_sh_vec:
1399; AVX1:       # %bb.0:
1400; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1401; AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
1402; AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
1403; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1404; AVX1-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
1405; AVX1-NEXT:    vzeroupper
1406; AVX1-NEXT:    retq
1407;
1408; AVX2-LABEL: sh_trunc_sh_vec:
1409; AVX2:       # %bb.0:
1410; AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
1411; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1412; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1413; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
1414; AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
1415; AVX2-NEXT:    vzeroupper
1416; AVX2-NEXT:    retq
1417;
1418; XOPAVX1-LABEL: sh_trunc_sh_vec:
1419; XOPAVX1:       # %bb.0:
1420; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1421; XOPAVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
1422; XOPAVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
1423; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1424; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
1425; XOPAVX1-NEXT:    vzeroupper
1426; XOPAVX1-NEXT:    retq
1427;
1428; XOPAVX2-LABEL: sh_trunc_sh_vec:
1429; XOPAVX2:       # %bb.0:
1430; XOPAVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
1431; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1432; XOPAVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1433; XOPAVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
1434; XOPAVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
1435; XOPAVX2-NEXT:    vzeroupper
1436; XOPAVX2-NEXT:    retq
1437;
1438; AVX512-LABEL: sh_trunc_sh_vec:
1439; AVX512:       # %bb.0:
1440; AVX512-NEXT:    vpsrlq $36, %ymm0, %ymm0
1441; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
1442; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
1443; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
1444; AVX512-NEXT:    vzeroupper
1445; AVX512-NEXT:    retq
1446;
1447; AVX512VL-LABEL: sh_trunc_sh_vec:
1448; AVX512VL:       # %bb.0:
1449; AVX512VL-NEXT:    vpsrlq $36, %ymm0, %ymm0
1450; AVX512VL-NEXT:    vpmovqd %ymm0, %xmm0
1451; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
1452; AVX512VL-NEXT:    vzeroupper
1453; AVX512VL-NEXT:    retq
1454;
1455; X32-AVX1-LABEL: sh_trunc_sh_vec:
1456; X32-AVX1:       # %bb.0:
1457; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1458; X32-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
1459; X32-AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
1460; X32-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1461; X32-AVX1-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
1462; X32-AVX1-NEXT:    vzeroupper
1463; X32-AVX1-NEXT:    retl
1464;
1465; X32-AVX2-LABEL: sh_trunc_sh_vec:
1466; X32-AVX2:       # %bb.0:
1467; X32-AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
1468; X32-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1469; X32-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1470; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
1471; X32-AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
1472; X32-AVX2-NEXT:    vzeroupper
1473; X32-AVX2-NEXT:    retl
1474  %s = lshr <4 x i64> %x, <i64 24, i64 24, i64 24, i64 24>
1475  %t = trunc <4 x i64> %s to <4 x i32>
1476  %r = lshr <4 x i32> %t, <i32 12, i32 12, i32 12, i32 12>
1477  ret <4 x i32> %r
1478}
1479