1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx                       | FileCheck %s --check-prefixes=ALL,AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2                      | FileCheck %s --check-prefixes=ALL,AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,avx512vl,avx512f | FileCheck %s --check-prefixes=ALL,AVX512
5
6; PR37427 - https://bugs.llvm.org/show_bug.cgi?id=37427
7
8define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
9; AVX1-LABEL: eq_zero:
10; AVX1:       # %bb.0:
11; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
12; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
13; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
14; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
15; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
16; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
17; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
18; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
19; AVX1-NEXT:    retq
20;
21; AVX2-LABEL: eq_zero:
22; AVX2:       # %bb.0:
23; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
24; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
25; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
26; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
27; AVX2-NEXT:    retq
28;
29; AVX512-LABEL: eq_zero:
30; AVX512:       # %bb.0:
31; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
32; AVX512-NEXT:    vptestnmw %xmm2, %xmm2, %k1
33; AVX512-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
34; AVX512-NEXT:    retq
35  %load = load <8 x i8>, <8 x i8>* %p
36  %cmp = icmp eq <8 x i8> %load, zeroinitializer
37  %sel = select <8 x i1> %cmp, <8 x i32> %x, <8 x i32> %y
38  ret <8 x i32> %sel
39}
40
41define <4 x i64> @ne_zero(<4 x i16>* %p, <4 x i64> %x, <4 x i64> %y) {
42; AVX1-LABEL: ne_zero:
43; AVX1:       # %bb.0:
44; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
45; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
46; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
47; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
48; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
49; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
50; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
51; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
52; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
53; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
54; AVX1-NEXT:    retq
55;
56; AVX2-LABEL: ne_zero:
57; AVX2:       # %bb.0:
58; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
59; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
60; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm2, %ymm2
61; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
62; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
63; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
64; AVX2-NEXT:    retq
65;
66; AVX512-LABEL: ne_zero:
67; AVX512:       # %bb.0:
68; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
69; AVX512-NEXT:    vptestmd %xmm2, %xmm2, %k1
70; AVX512-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
71; AVX512-NEXT:    retq
72  %load = load <4 x i16>, <4 x i16>* %p
73  %cmp = icmp ne <4 x i16> %load, zeroinitializer
74  %sel = select <4 x i1> %cmp, <4 x i64> %x, <4 x i64> %y
75  ret <4 x i64> %sel
76}
77
78define <16 x i16> @sgt_zero(<16 x i8>* %p, <16 x i16> %x, <16 x i16> %y) {
79; AVX1-LABEL: sgt_zero:
80; AVX1:       # %bb.0:
81; AVX1-NEXT:    vmovdqa (%rdi), %xmm2
82; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
83; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm2
84; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm3
85; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
86; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
87; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
88; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
89; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
90; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
91; AVX1-NEXT:    retq
92;
93; AVX2-LABEL: sgt_zero:
94; AVX2:       # %bb.0:
95; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm2
96; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
97; AVX2-NEXT:    vpcmpgtw %ymm3, %ymm2, %ymm2
98; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
99; AVX2-NEXT:    retq
100;
101; AVX512-LABEL: sgt_zero:
102; AVX512:       # %bb.0:
103; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
104; AVX512-NEXT:    vpcmpltb (%rdi), %xmm2, %k1
105; AVX512-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
106; AVX512-NEXT:    retq
107  %load = load <16 x i8>, <16 x i8>* %p
108  %cmp = icmp sgt <16 x i8> %load, zeroinitializer
109  %sel = select <16 x i1> %cmp, <16 x i16> %x, <16 x i16> %y
110  ret <16 x i16> %sel
111}
112
113define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
114; AVX1-LABEL: slt_zero:
115; AVX1:       # %bb.0:
116; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm2
117; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
118; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
119; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
120; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
121; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
122; AVX1-NEXT:    retq
123;
124; AVX2-LABEL: slt_zero:
125; AVX2:       # %bb.0:
126; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm2
127; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
128; AVX2-NEXT:    retq
129;
130; AVX512-LABEL: slt_zero:
131; AVX512:       # %bb.0:
132; AVX512-NEXT:    vpmovsxbw (%rdi), %xmm2
133; AVX512-NEXT:    vpmovw2m %xmm2, %k1
134; AVX512-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
135; AVX512-NEXT:    retq
136  %load = load <8 x i8>, <8 x i8>* %p
137  %cmp = icmp slt <8 x i8> %load, zeroinitializer
138  %sel = select <8 x i1> %cmp, <8 x i32> %x, <8 x i32> %y
139  ret <8 x i32> %sel
140}
141
142define <4 x double> @eq_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x double> %y) {
143; AVX1-LABEL: eq_zero_fp_select:
144; AVX1:       # %bb.0:
145; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
146; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
147; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
148; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
149; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
150; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
151; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
152; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
153; AVX1-NEXT:    retq
154;
155; AVX2-LABEL: eq_zero_fp_select:
156; AVX2:       # %bb.0:
157; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
158; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
159; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm2, %ymm2
160; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
161; AVX2-NEXT:    retq
162;
163; AVX512-LABEL: eq_zero_fp_select:
164; AVX512:       # %bb.0:
165; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
166; AVX512-NEXT:    vptestnmd %xmm2, %xmm2, %k1
167; AVX512-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
168; AVX512-NEXT:    retq
169  %load = load <4 x i8>, <4 x i8>* %p
170  %cmp = icmp eq <4 x i8> %load, zeroinitializer
171  %sel = select <4 x i1> %cmp, <4 x double> %x, <4 x double> %y
172  ret <4 x double> %sel
173}
174
175define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float> %y) {
176; AVX1-LABEL: ne_zero_fp_select:
177; AVX1:       # %bb.0:
178; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
179; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
180; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
181; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
182; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
183; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
184; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
185; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
186; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
187; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
188; AVX1-NEXT:    retq
189;
190; AVX2-LABEL: ne_zero_fp_select:
191; AVX2:       # %bb.0:
192; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
193; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
194; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
195; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
196; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
197; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
198; AVX2-NEXT:    retq
199;
200; AVX512-LABEL: ne_zero_fp_select:
201; AVX512:       # %bb.0:
202; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
203; AVX512-NEXT:    vptestmw %xmm2, %xmm2, %k1
204; AVX512-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
205; AVX512-NEXT:    retq
206  %load = load <8 x i8>, <8 x i8>* %p
207  %cmp = icmp ne <8 x i8> %load, zeroinitializer
208  %sel = select <8 x i1> %cmp, <8 x float> %x, <8 x float> %y
209  ret <8 x float> %sel
210}
211
212define <4 x double> @sgt_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x double> %y) {
213; AVX1-LABEL: sgt_zero_fp_select:
214; AVX1:       # %bb.0:
215; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm2
216; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
217; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
218; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
219; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
220; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
221; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
222; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
223; AVX1-NEXT:    retq
224;
225; AVX2-LABEL: sgt_zero_fp_select:
226; AVX2:       # %bb.0:
227; AVX2-NEXT:    vpmovsxbq (%rdi), %ymm2
228; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
229; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm2
230; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
231; AVX2-NEXT:    retq
232;
233; AVX512-LABEL: sgt_zero_fp_select:
234; AVX512:       # %bb.0:
235; AVX512-NEXT:    vpmovsxbd (%rdi), %xmm2
236; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
237; AVX512-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
238; AVX512-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
239; AVX512-NEXT:    retq
240  %load = load <4 x i8>, <4 x i8>* %p
241  %cmp = icmp sgt <4 x i8> %load, zeroinitializer
242  %sel = select <4 x i1> %cmp, <4 x double> %x, <4 x double> %y
243  ret <4 x double> %sel
244}
245
246define <8 x float> @slt_zero_fp_select(<8 x i16>* %p, <8 x float> %x, <8 x float> %y) {
247; AVX1-LABEL: slt_zero_fp_select:
248; AVX1:       # %bb.0:
249; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm2
250; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm3
251; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
252; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
253; AVX1-NEXT:    retq
254;
255; AVX2-LABEL: slt_zero_fp_select:
256; AVX2:       # %bb.0:
257; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm2
258; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
259; AVX2-NEXT:    retq
260;
261; AVX512-LABEL: slt_zero_fp_select:
262; AVX512:       # %bb.0:
263; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
264; AVX512-NEXT:    vpcmpgtw (%rdi), %xmm2, %k1
265; AVX512-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
266; AVX512-NEXT:    retq
267  %load = load <8 x i16>, <8 x i16>* %p
268  %cmp = icmp slt <8 x i16> %load, zeroinitializer
269  %sel = select <8 x i1> %cmp, <8 x float> %x, <8 x float> %y
270  ret <8 x float> %sel
271}
272
273