1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ
7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512VL --check-prefix=AVX512DQVL
9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512VL --check-prefix=AVX512BWVL
10
11;
12; Variable Shifts
13;
14
15define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
16; AVX1-LABEL: var_shift_v4i64:
17; AVX1:       # BB#0:
18; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
19; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
20; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
21; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
22; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
23; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
24; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
25; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
26; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
27; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
28; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
29; AVX1-NEXT:    retq
30;
31; AVX2-LABEL: var_shift_v4i64:
32; AVX2:       # BB#0:
33; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
34; AVX2-NEXT:    retq
35;
36; XOPAVX1-LABEL: var_shift_v4i64:
37; XOPAVX1:       # BB#0:
38; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
39; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
40; XOPAVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
41; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
42; XOPAVX1-NEXT:    vpshlq %xmm2, %xmm4, %xmm2
43; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm3, %xmm1
44; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
45; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
46; XOPAVX1-NEXT:    retq
47;
48; XOPAVX2-LABEL: var_shift_v4i64:
49; XOPAVX2:       # BB#0:
50; XOPAVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
51; XOPAVX2-NEXT:    retq
52;
53; AVX512-LABEL: var_shift_v4i64:
54; AVX512:       # BB#0:
55; AVX512-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
56; AVX512-NEXT:    retq
57;
58; AVX512VL-LABEL: var_shift_v4i64:
59; AVX512VL:       # BB#0:
60; AVX512VL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
61; AVX512VL-NEXT:    retq
62  %shift = lshr <4 x i64> %a, %b
63  ret <4 x i64> %shift
64}
65
66define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
67; AVX1-LABEL: var_shift_v8i32:
68; AVX1:       # BB#0:
69; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
70; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
71; AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
72; AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
73; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
74; AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
75; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
76; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
77; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
78; AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
79; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
80; AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
81; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
82; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
83; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
84; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
85; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
86; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
87; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
88; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
89; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
90; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
91; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
92; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
93; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
94; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
95; AVX1-NEXT:    retq
96;
97; AVX2-LABEL: var_shift_v8i32:
98; AVX2:       # BB#0:
99; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
100; AVX2-NEXT:    retq
101;
102; XOPAVX1-LABEL: var_shift_v8i32:
103; XOPAVX1:       # BB#0:
104; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
105; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
106; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
107; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
108; XOPAVX1-NEXT:    vpshld %xmm2, %xmm4, %xmm2
109; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
110; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
111; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
112; XOPAVX1-NEXT:    retq
113;
114; XOPAVX2-LABEL: var_shift_v8i32:
115; XOPAVX2:       # BB#0:
116; XOPAVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
117; XOPAVX2-NEXT:    retq
118;
119; AVX512-LABEL: var_shift_v8i32:
120; AVX512:       # BB#0:
121; AVX512-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
122; AVX512-NEXT:    retq
123;
124; AVX512VL-LABEL: var_shift_v8i32:
125; AVX512VL:       # BB#0:
126; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
127; AVX512VL-NEXT:    retq
128  %shift = lshr <8 x i32> %a, %b
129  ret <8 x i32> %shift
130}
131
132define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
133; AVX1-LABEL: var_shift_v16i16:
134; AVX1:       # BB#0:
135; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
136; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
137; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
138; AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
139; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
140; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
141; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
142; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
143; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
144; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
145; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
146; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
147; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
148; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
149; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
150; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
151; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
152; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
153; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
154; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
155; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
156; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
157; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
158; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
159; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
160; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
161; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
162; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
163; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
164; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
165; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
166; AVX1-NEXT:    retq
167;
168; AVX2-LABEL: var_shift_v16i16:
169; AVX2:       # BB#0:
170; AVX2-NEXT:    vpxor %ymm2, %ymm2, %ymm2
171; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
172; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
173; AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
174; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
175; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
176; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
177; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
178; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
179; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
180; AVX2-NEXT:    retq
181;
182; XOPAVX1-LABEL: var_shift_v16i16:
183; XOPAVX1:       # BB#0:
184; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
185; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
186; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
187; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
188; XOPAVX1-NEXT:    vpshlw %xmm2, %xmm4, %xmm2
189; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
190; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
191; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
192; XOPAVX1-NEXT:    retq
193;
194; XOPAVX2-LABEL: var_shift_v16i16:
195; XOPAVX2:       # BB#0:
196; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
197; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
198; XOPAVX2-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
199; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
200; XOPAVX2-NEXT:    vpshlw %xmm2, %xmm4, %xmm2
201; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
202; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
203; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
204; XOPAVX2-NEXT:    retq
205;
206; AVX512DQ-LABEL: var_shift_v16i16:
207; AVX512DQ:       # BB#0:
208; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
209; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
210; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
211; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
212; AVX512DQ-NEXT:    retq
213;
214; AVX512BW-LABEL: var_shift_v16i16:
215; AVX512BW:       # BB#0:
216; AVX512BW-NEXT:    # kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
217; AVX512BW-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
218; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
219; AVX512BW-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
220; AVX512BW-NEXT:    retq
221;
222; AVX512DQVL-LABEL: var_shift_v16i16:
223; AVX512DQVL:       # BB#0:
224; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
225; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
226; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
227; AVX512DQVL-NEXT:    vpmovdw %zmm0, %ymm0
228; AVX512DQVL-NEXT:    retq
229;
230; AVX512BWVL-LABEL: var_shift_v16i16:
231; AVX512BWVL:       # BB#0:
232; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
233; AVX512BWVL-NEXT:    retq
234  %shift = lshr <16 x i16> %a, %b
235  ret <16 x i16> %shift
236}
237
238define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
239; AVX1-LABEL: var_shift_v32i8:
240; AVX1:       # BB#0:
241; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
242; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
243; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
244; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
245; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
246; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
247; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
248; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
249; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
250; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
251; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
252; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
253; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
254; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
255; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
256; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
257; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
258; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
259; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
260; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
261; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
262; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
263; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
264; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
265; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
266; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
267; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
268; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
269; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
270; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
271; AVX1-NEXT:    retq
272;
273; AVX2-LABEL: var_shift_v32i8:
274; AVX2:       # BB#0:
275; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
276; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
277; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
278; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
279; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
280; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
281; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
282; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
283; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
284; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
285; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
286; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
287; AVX2-NEXT:    retq
288;
289; XOPAVX1-LABEL: var_shift_v32i8:
290; XOPAVX1:       # BB#0:
291; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
292; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
293; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
294; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
295; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
296; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
297; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
298; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
299; XOPAVX1-NEXT:    retq
300;
301; XOPAVX2-LABEL: var_shift_v32i8:
302; XOPAVX2:       # BB#0:
303; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
304; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
305; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
306; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
307; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
308; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
309; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
310; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
311; XOPAVX2-NEXT:    retq
312;
313; AVX512DQ-LABEL: var_shift_v32i8:
314; AVX512DQ:       # BB#0:
315; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
316; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm2
317; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
318; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
319; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm2
320; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
321; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
322; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
323; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm2
324; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
325; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
326; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
327; AVX512DQ-NEXT:    retq
328;
329; AVX512BW-LABEL: var_shift_v32i8:
330; AVX512BW:       # BB#0:
331; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
332; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
333; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
334; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
335; AVX512BW-NEXT:    retq
336;
337; AVX512DQVL-LABEL: var_shift_v32i8:
338; AVX512DQVL:       # BB#0:
339; AVX512DQVL-NEXT:    vpsllw $5, %ymm1, %ymm1
340; AVX512DQVL-NEXT:    vpsrlw $4, %ymm0, %ymm2
341; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
342; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
343; AVX512DQVL-NEXT:    vpsrlw $2, %ymm0, %ymm2
344; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
345; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
346; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
347; AVX512DQVL-NEXT:    vpsrlw $1, %ymm0, %ymm2
348; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
349; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
350; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
351; AVX512DQVL-NEXT:    retq
352;
353; AVX512BWVL-LABEL: var_shift_v32i8:
354; AVX512BWVL:       # BB#0:
355; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
356; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
357; AVX512BWVL-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
358; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
359; AVX512BWVL-NEXT:    retq
360  %shift = lshr <32 x i8> %a, %b
361  ret <32 x i8> %shift
362}
363
364;
365; Uniform Variable Shifts
366;
367
368define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
369; AVX1-LABEL: splatvar_shift_v4i64:
370; AVX1:       # BB#0:
371; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
372; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
373; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
374; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
375; AVX1-NEXT:    retq
376;
377; AVX2-LABEL: splatvar_shift_v4i64:
378; AVX2:       # BB#0:
379; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
380; AVX2-NEXT:    retq
381;
382; XOPAVX1-LABEL: splatvar_shift_v4i64:
383; XOPAVX1:       # BB#0:
384; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
385; XOPAVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
386; XOPAVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
387; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
388; XOPAVX1-NEXT:    retq
389;
390; XOPAVX2-LABEL: splatvar_shift_v4i64:
391; XOPAVX2:       # BB#0:
392; XOPAVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
393; XOPAVX2-NEXT:    retq
394;
395; AVX512-LABEL: splatvar_shift_v4i64:
396; AVX512:       # BB#0:
397; AVX512-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
398; AVX512-NEXT:    retq
399;
400; AVX512VL-LABEL: splatvar_shift_v4i64:
401; AVX512VL:       # BB#0:
402; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
403; AVX512VL-NEXT:    retq
404  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
405  %shift = lshr <4 x i64> %a, %splat
406  ret <4 x i64> %shift
407}
408
409define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
410; AVX1-LABEL: splatvar_shift_v8i32:
411; AVX1:       # BB#0:
412; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
413; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
414; AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
415; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
416; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
417; AVX1-NEXT:    retq
418;
419; AVX2-LABEL: splatvar_shift_v8i32:
420; AVX2:       # BB#0:
421; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
422; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
423; AVX2-NEXT:    retq
424;
425; XOPAVX1-LABEL: splatvar_shift_v8i32:
426; XOPAVX1:       # BB#0:
427; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
428; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
429; XOPAVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
430; XOPAVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
431; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
432; XOPAVX1-NEXT:    retq
433;
434; XOPAVX2-LABEL: splatvar_shift_v8i32:
435; XOPAVX2:       # BB#0:
436; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
437; XOPAVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
438; XOPAVX2-NEXT:    retq
439;
440; AVX512-LABEL: splatvar_shift_v8i32:
441; AVX512:       # BB#0:
442; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
443; AVX512-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
444; AVX512-NEXT:    retq
445;
446; AVX512VL-LABEL: splatvar_shift_v8i32:
447; AVX512VL:       # BB#0:
448; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
449; AVX512VL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
450; AVX512VL-NEXT:    retq
451  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
452  %shift = lshr <8 x i32> %a, %splat
453  ret <8 x i32> %shift
454}
455
456define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
457; AVX1-LABEL: splatvar_shift_v16i16:
458; AVX1:       # BB#0:
459; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
460; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
461; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
462; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
463; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
464; AVX1-NEXT:    retq
465;
466; AVX2-LABEL: splatvar_shift_v16i16:
467; AVX2:       # BB#0:
468; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
469; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
470; AVX2-NEXT:    retq
471;
472; XOPAVX1-LABEL: splatvar_shift_v16i16:
473; XOPAVX1:       # BB#0:
474; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
475; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
476; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
477; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
478; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
479; XOPAVX1-NEXT:    retq
480;
481; XOPAVX2-LABEL: splatvar_shift_v16i16:
482; XOPAVX2:       # BB#0:
483; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
484; XOPAVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
485; XOPAVX2-NEXT:    retq
486;
487; AVX512-LABEL: splatvar_shift_v16i16:
488; AVX512:       # BB#0:
489; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
490; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
491; AVX512-NEXT:    retq
492;
493; AVX512VL-LABEL: splatvar_shift_v16i16:
494; AVX512VL:       # BB#0:
495; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
496; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
497; AVX512VL-NEXT:    retq
498  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
499  %shift = lshr <16 x i16> %a, %splat
500  ret <16 x i16> %shift
501}
502
503define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
504; AVX1-LABEL: splatvar_shift_v32i8:
505; AVX1:       # BB#0:
506; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
507; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
508; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
509; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
510; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
511; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
512; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
513; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm2, %xmm2
514; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
515; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
516; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
517; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm6
518; AVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm2, %xmm2
519; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
520; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
521; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
522; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm4
523; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm2, %xmm2
524; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
525; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
526; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
527; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
528; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
529; AVX1-NEXT:    vpblendvb %xmm6, %xmm1, %xmm0, %xmm0
530; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
531; AVX1-NEXT:    vpand %xmm7, %xmm1, %xmm1
532; AVX1-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
533; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
534; AVX1-NEXT:    retq
535;
536; AVX2-LABEL: splatvar_shift_v32i8:
537; AVX2:       # BB#0:
538; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
539; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
540; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
541; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
542; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
543; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
544; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
545; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
546; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
547; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
548; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
549; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
550; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
551; AVX2-NEXT:    retq
552;
553; XOPAVX1-LABEL: splatvar_shift_v32i8:
554; XOPAVX1:       # BB#0:
555; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
556; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
557; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
558; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
559; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm2, %xmm2
560; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
561; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
562; XOPAVX1-NEXT:    retq
563;
564; XOPAVX2-LABEL: splatvar_shift_v32i8:
565; XOPAVX2:       # BB#0:
566; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
567; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
568; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
569; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
570; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
571; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
572; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
573; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
574; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
575; XOPAVX2-NEXT:    retq
576;
577; AVX512DQ-LABEL: splatvar_shift_v32i8:
578; AVX512DQ:       # BB#0:
579; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
580; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm2
581; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
582; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
583; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
584; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm2
585; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
586; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
587; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
588; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm2
589; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
590; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
591; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
592; AVX512DQ-NEXT:    retq
593;
594; AVX512BW-LABEL: splatvar_shift_v32i8:
595; AVX512BW:       # BB#0:
596; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
597; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
598; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
599; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
600; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
601; AVX512BW-NEXT:    retq
602;
603; AVX512DQVL-LABEL: splatvar_shift_v32i8:
604; AVX512DQVL:       # BB#0:
605; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %ymm1
606; AVX512DQVL-NEXT:    vpsrlw $4, %ymm0, %ymm2
607; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
608; AVX512DQVL-NEXT:    vpsllw $5, %ymm1, %ymm1
609; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
610; AVX512DQVL-NEXT:    vpsrlw $2, %ymm0, %ymm2
611; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
612; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
613; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
614; AVX512DQVL-NEXT:    vpsrlw $1, %ymm0, %ymm2
615; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
616; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
617; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
618; AVX512DQVL-NEXT:    retq
619;
620; AVX512BWVL-LABEL: splatvar_shift_v32i8:
621; AVX512BWVL:       # BB#0:
622; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %ymm1
623; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
624; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
625; AVX512BWVL-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
626; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
627; AVX512BWVL-NEXT:    retq
628  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
629  %shift = lshr <32 x i8> %a, %splat
630  ret <32 x i8> %shift
631}
632
633;
634; Constant Shifts
635;
636
637define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
638; AVX1-LABEL: constant_shift_v4i64:
639; AVX1:       # BB#0:
640; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
641; AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
642; AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
643; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
644; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
645; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
646; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
647; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
648; AVX1-NEXT:    retq
649;
650; AVX2-LABEL: constant_shift_v4i64:
651; AVX2:       # BB#0:
652; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
653; AVX2-NEXT:    retq
654;
655; XOPAVX1-LABEL: constant_shift_v4i64:
656; XOPAVX1:       # BB#0:
657; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
658; XOPAVX1-NEXT:    vpsubq {{.*}}(%rip), %xmm1, %xmm2
659; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
660; XOPAVX1-NEXT:    vpshlq %xmm2, %xmm3, %xmm2
661; XOPAVX1-NEXT:    vpsubq {{.*}}(%rip), %xmm1, %xmm1
662; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
663; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
664; XOPAVX1-NEXT:    retq
665;
666; XOPAVX2-LABEL: constant_shift_v4i64:
667; XOPAVX2:       # BB#0:
668; XOPAVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
669; XOPAVX2-NEXT:    retq
670;
671; AVX512-LABEL: constant_shift_v4i64:
672; AVX512:       # BB#0:
673; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
674; AVX512-NEXT:    retq
675;
676; AVX512VL-LABEL: constant_shift_v4i64:
677; AVX512VL:       # BB#0:
678; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
679; AVX512VL-NEXT:    retq
680  %shift = lshr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
681  ret <4 x i64> %shift
682}
683
684define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
685; AVX1-LABEL: constant_shift_v8i32:
686; AVX1:       # BB#0:
687; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
688; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
689; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
690; AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
691; AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
692; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
693; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
694; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
695; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
696; AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
697; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
698; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
699; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
700; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
701; AVX1-NEXT:    retq
702;
703; AVX2-LABEL: constant_shift_v8i32:
704; AVX2:       # BB#0:
705; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
706; AVX2-NEXT:    retq
707;
708; XOPAVX1-LABEL: constant_shift_v8i32:
709; XOPAVX1:       # BB#0:
710; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm1
711; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
712; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
713; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
714; XOPAVX1-NEXT:    retq
715;
716; XOPAVX2-LABEL: constant_shift_v8i32:
717; XOPAVX2:       # BB#0:
718; XOPAVX2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
719; XOPAVX2-NEXT:    retq
720;
721; AVX512-LABEL: constant_shift_v8i32:
722; AVX512:       # BB#0:
723; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
724; AVX512-NEXT:    retq
725;
726; AVX512VL-LABEL: constant_shift_v8i32:
727; AVX512VL:       # BB#0:
728; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
729; AVX512VL-NEXT:    retq
730  %shift = lshr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
731  ret <8 x i32> %shift
732}
733
734define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
735; AVX1-LABEL: constant_shift_v16i16:
736; AVX1:       # BB#0:
737; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
738; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
739; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
740; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
741; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
742; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
743; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
744; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
745; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
746; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
747; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm2
748; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
749; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm2
750; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
751; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
752; AVX1-NEXT:    retq
753;
754; AVX2-LABEL: constant_shift_v16i16:
755; AVX2:       # BB#0:
756; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
757; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
758; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
759; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
760; AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
761; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
762; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
763; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
764; AVX2-NEXT:    vpsrlvd %ymm2, %ymm0, %ymm0
765; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
766; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
767; AVX2-NEXT:    retq
768;
769; XOPAVX1-LABEL: constant_shift_v16i16:
770; XOPAVX1:       # BB#0:
771; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
772; XOPAVX1-NEXT:    vpsubw {{.*}}(%rip), %xmm1, %xmm2
773; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
774; XOPAVX1-NEXT:    vpshlw %xmm2, %xmm3, %xmm2
775; XOPAVX1-NEXT:    vpsubw {{.*}}(%rip), %xmm1, %xmm1
776; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
777; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
778; XOPAVX1-NEXT:    retq
779;
780; XOPAVX2-LABEL: constant_shift_v16i16:
781; XOPAVX2:       # BB#0:
782; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
783; XOPAVX2-NEXT:    vpsubw {{.*}}(%rip), %xmm1, %xmm2
784; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
785; XOPAVX2-NEXT:    vpshlw %xmm2, %xmm3, %xmm2
786; XOPAVX2-NEXT:    vpsubw {{.*}}(%rip), %xmm1, %xmm1
787; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
788; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
789; XOPAVX2-NEXT:    retq
790;
791; AVX512DQ-LABEL: constant_shift_v16i16:
792; AVX512DQ:       # BB#0:
793; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
794; AVX512DQ-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
795; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
796; AVX512DQ-NEXT:    retq
797;
798; AVX512BW-LABEL: constant_shift_v16i16:
799; AVX512BW:       # BB#0:
800; AVX512BW-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
801; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
802; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
803; AVX512BW-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
804; AVX512BW-NEXT:    retq
805;
806; AVX512DQVL-LABEL: constant_shift_v16i16:
807; AVX512DQVL:       # BB#0:
808; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
809; AVX512DQVL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
810; AVX512DQVL-NEXT:    vpmovdw %zmm0, %ymm0
811; AVX512DQVL-NEXT:    retq
812;
813; AVX512BWVL-LABEL: constant_shift_v16i16:
814; AVX512BWVL:       # BB#0:
815; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
816; AVX512BWVL-NEXT:    retq
817  %shift = lshr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
818  ret <16 x i16> %shift
819}
820
821define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
822; AVX1-LABEL: constant_shift_v32i8:
823; AVX1:       # BB#0:
824; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
825; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
826; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
827; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
828; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
829; AVX1-NEXT:    vpsllw $5, %xmm4, %xmm4
830; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm1, %xmm1
831; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
832; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
833; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
834; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm6
835; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm1, %xmm1
836; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
837; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
838; AVX1-NEXT:    vpand %xmm7, %xmm2, %xmm2
839; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm3
840; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
841; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
842; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
843; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm0, %xmm0
844; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm2
845; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
846; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm0, %xmm0
847; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm2
848; AVX1-NEXT:    vpand %xmm7, %xmm2, %xmm2
849; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
850; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
851; AVX1-NEXT:    retq
852;
853; AVX2-LABEL: constant_shift_v32i8:
854; AVX2:       # BB#0:
855; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
856; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
857; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
858; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
859; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
860; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
861; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
862; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
863; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
864; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
865; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
866; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
867; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
868; AVX2-NEXT:    retq
869;
870; XOPAVX1-LABEL: constant_shift_v32i8:
871; XOPAVX1:       # BB#0:
872; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
873; XOPAVX1-NEXT:    vpsubb {{.*}}(%rip), %xmm1, %xmm1
874; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
875; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm2, %xmm2
876; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
877; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
878; XOPAVX1-NEXT:    retq
879;
880; XOPAVX2-LABEL: constant_shift_v32i8:
881; XOPAVX2:       # BB#0:
882; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
883; XOPAVX2-NEXT:    vpsubb {{.*}}(%rip), %xmm1, %xmm1
884; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
885; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm2, %xmm2
886; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
887; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
888; XOPAVX2-NEXT:    retq
889;
890; AVX512DQ-LABEL: constant_shift_v32i8:
891; AVX512DQ:       # BB#0:
892; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
893; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
894; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm2
895; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
896; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
897; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm2
898; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
899; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
900; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
901; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm2
902; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
903; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
904; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
905; AVX512DQ-NEXT:    retq
906;
907; AVX512BW-LABEL: constant_shift_v32i8:
908; AVX512BW:       # BB#0:
909; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
910; AVX512BW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
911; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
912; AVX512BW-NEXT:    retq
913;
914; AVX512DQVL-LABEL: constant_shift_v32i8:
915; AVX512DQVL:       # BB#0:
916; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
917; AVX512DQVL-NEXT:    vpsllw $5, %ymm1, %ymm1
918; AVX512DQVL-NEXT:    vpsrlw $4, %ymm0, %ymm2
919; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
920; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
921; AVX512DQVL-NEXT:    vpsrlw $2, %ymm0, %ymm2
922; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
923; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
924; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
925; AVX512DQVL-NEXT:    vpsrlw $1, %ymm0, %ymm2
926; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
927; AVX512DQVL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
928; AVX512DQVL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
929; AVX512DQVL-NEXT:    retq
930;
931; AVX512BWVL-LABEL: constant_shift_v32i8:
932; AVX512BWVL:       # BB#0:
933; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
934; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
935; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
936; AVX512BWVL-NEXT:    retq
937  %shift = lshr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
938  ret <32 x i8> %shift
939}
940
941;
942; Uniform Constant Shifts
943;
944
945define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
946; AVX1-LABEL: splatconstant_shift_v4i64:
947; AVX1:       # BB#0:
948; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
949; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
950; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
951; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
952; AVX1-NEXT:    retq
953;
954; AVX2-LABEL: splatconstant_shift_v4i64:
955; AVX2:       # BB#0:
956; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
957; AVX2-NEXT:    retq
958;
959; XOPAVX1-LABEL: splatconstant_shift_v4i64:
960; XOPAVX1:       # BB#0:
961; XOPAVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
962; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
963; XOPAVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
964; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
965; XOPAVX1-NEXT:    retq
966;
967; XOPAVX2-LABEL: splatconstant_shift_v4i64:
968; XOPAVX2:       # BB#0:
969; XOPAVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
970; XOPAVX2-NEXT:    retq
971;
972; AVX512-LABEL: splatconstant_shift_v4i64:
973; AVX512:       # BB#0:
974; AVX512-NEXT:    vpsrlq $7, %ymm0, %ymm0
975; AVX512-NEXT:    retq
976;
977; AVX512VL-LABEL: splatconstant_shift_v4i64:
978; AVX512VL:       # BB#0:
979; AVX512VL-NEXT:    vpsrlq $7, %ymm0, %ymm0
980; AVX512VL-NEXT:    retq
981  %shift = lshr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
982  ret <4 x i64> %shift
983}
984
985define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
986; AVX1-LABEL: splatconstant_shift_v8i32:
987; AVX1:       # BB#0:
988; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
989; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
990; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
991; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
992; AVX1-NEXT:    retq
993;
994; AVX2-LABEL: splatconstant_shift_v8i32:
995; AVX2:       # BB#0:
996; AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
997; AVX2-NEXT:    retq
998;
999; XOPAVX1-LABEL: splatconstant_shift_v8i32:
1000; XOPAVX1:       # BB#0:
1001; XOPAVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
1002; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1003; XOPAVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
1004; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1005; XOPAVX1-NEXT:    retq
1006;
1007; XOPAVX2-LABEL: splatconstant_shift_v8i32:
1008; XOPAVX2:       # BB#0:
1009; XOPAVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
1010; XOPAVX2-NEXT:    retq
1011;
1012; AVX512-LABEL: splatconstant_shift_v8i32:
1013; AVX512:       # BB#0:
1014; AVX512-NEXT:    vpsrld $5, %ymm0, %ymm0
1015; AVX512-NEXT:    retq
1016;
1017; AVX512VL-LABEL: splatconstant_shift_v8i32:
1018; AVX512VL:       # BB#0:
1019; AVX512VL-NEXT:    vpsrld $5, %ymm0, %ymm0
1020; AVX512VL-NEXT:    retq
1021  %shift = lshr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
1022  ret <8 x i32> %shift
1023}
1024
1025define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
1026; AVX1-LABEL: splatconstant_shift_v16i16:
1027; AVX1:       # BB#0:
1028; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
1029; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1030; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1031; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1032; AVX1-NEXT:    retq
1033;
1034; AVX2-LABEL: splatconstant_shift_v16i16:
1035; AVX2:       # BB#0:
1036; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1037; AVX2-NEXT:    retq
1038;
1039; XOPAVX1-LABEL: splatconstant_shift_v16i16:
1040; XOPAVX1:       # BB#0:
1041; XOPAVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
1042; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
1043; XOPAVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1044; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1045; XOPAVX1-NEXT:    retq
1046;
1047; XOPAVX2-LABEL: splatconstant_shift_v16i16:
1048; XOPAVX2:       # BB#0:
1049; XOPAVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1050; XOPAVX2-NEXT:    retq
1051;
1052; AVX512-LABEL: splatconstant_shift_v16i16:
1053; AVX512:       # BB#0:
1054; AVX512-NEXT:    vpsrlw $3, %ymm0, %ymm0
1055; AVX512-NEXT:    retq
1056;
1057; AVX512VL-LABEL: splatconstant_shift_v16i16:
1058; AVX512VL:       # BB#0:
1059; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
1060; AVX512VL-NEXT:    retq
1061  %shift = lshr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
1062  ret <16 x i16> %shift
1063}
1064
1065define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
1066; AVX1-LABEL: splatconstant_shift_v32i8:
1067; AVX1:       # BB#0:
1068; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1069; AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
1070; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
1071; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
1072; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
1073; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
1074; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1075; AVX1-NEXT:    retq
1076;
1077; AVX2-LABEL: splatconstant_shift_v32i8:
1078; AVX2:       # BB#0:
1079; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1080; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1081; AVX2-NEXT:    retq
1082;
1083; XOPAVX1-LABEL: splatconstant_shift_v32i8:
1084; XOPAVX1:       # BB#0:
1085; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1086; XOPAVX1-NEXT:    vpsubb {{.*}}(%rip), %xmm1, %xmm1
1087; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
1088; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm2, %xmm2
1089; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
1090; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1091; XOPAVX1-NEXT:    retq
1092;
1093; XOPAVX2-LABEL: splatconstant_shift_v32i8:
1094; XOPAVX2:       # BB#0:
1095; XOPAVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
1096; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1097; XOPAVX2-NEXT:    retq
1098;
1099; AVX512-LABEL: splatconstant_shift_v32i8:
1100; AVX512:       # BB#0:
1101; AVX512-NEXT:    vpsrlw $3, %ymm0, %ymm0
1102; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1103; AVX512-NEXT:    retq
1104;
1105; AVX512VL-LABEL: splatconstant_shift_v32i8:
1106; AVX512VL:       # BB#0:
1107; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
1108; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
1109; AVX512VL-NEXT:    retq
1110  %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
1111  ret <32 x i8> %shift
1112}
1113