1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=sse2    | FileCheck %s --check-prefixes=SSE,SSE2
3; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=sse4.2  | FileCheck %s --check-prefixes=SSE,SSE42
4; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx     | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
5; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2    | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
6; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
7; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512bw,avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VLBW
8
9define <1 x double> @load_v1f64_v1i64(<1 x i64> %trigger, <1 x double>* %addr, <1 x double> %dst) {
10; SSE-LABEL: load_v1f64_v1i64:
11; SSE:       ## %bb.0:
12; SSE-NEXT:    testq %rdi, %rdi
13; SSE-NEXT:    jne LBB0_2
14; SSE-NEXT:  ## %bb.1: ## %cond.load
15; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
16; SSE-NEXT:  LBB0_2: ## %else
17; SSE-NEXT:    retq
18;
19; AVX-LABEL: load_v1f64_v1i64:
20; AVX:       ## %bb.0:
21; AVX-NEXT:    testq %rdi, %rdi
22; AVX-NEXT:    jne LBB0_2
23; AVX-NEXT:  ## %bb.1: ## %cond.load
24; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
25; AVX-NEXT:  LBB0_2: ## %else
26; AVX-NEXT:    retq
27  %mask = icmp eq <1 x i64> %trigger, zeroinitializer
28  %res = call <1 x double> @llvm.masked.load.v1f64.p0v1f64(<1 x double>* %addr, i32 4, <1 x i1>%mask, <1 x double>%dst)
29  ret <1 x double> %res
30}
31
32define <2 x double> @load_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <2 x double> %dst) {
33; SSE2-LABEL: load_v2f64_v2i64:
34; SSE2:       ## %bb.0:
35; SSE2-NEXT:    pxor %xmm2, %xmm2
36; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
37; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
38; SSE2-NEXT:    pand %xmm2, %xmm0
39; SSE2-NEXT:    movd %xmm0, %eax
40; SSE2-NEXT:    testb $1, %al
41; SSE2-NEXT:    je LBB1_2
42; SSE2-NEXT:  ## %bb.1: ## %cond.load
43; SSE2-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
44; SSE2-NEXT:  LBB1_2: ## %else
45; SSE2-NEXT:    pextrw $4, %xmm0, %eax
46; SSE2-NEXT:    testb $1, %al
47; SSE2-NEXT:    je LBB1_4
48; SSE2-NEXT:  ## %bb.3: ## %cond.load1
49; SSE2-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
50; SSE2-NEXT:  LBB1_4: ## %else2
51; SSE2-NEXT:    movapd %xmm1, %xmm0
52; SSE2-NEXT:    retq
53;
54; SSE42-LABEL: load_v2f64_v2i64:
55; SSE42:       ## %bb.0:
56; SSE42-NEXT:    pxor %xmm2, %xmm2
57; SSE42-NEXT:    pcmpeqq %xmm0, %xmm2
58; SSE42-NEXT:    pextrb $0, %xmm2, %eax
59; SSE42-NEXT:    testb $1, %al
60; SSE42-NEXT:    je LBB1_2
61; SSE42-NEXT:  ## %bb.1: ## %cond.load
62; SSE42-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
63; SSE42-NEXT:  LBB1_2: ## %else
64; SSE42-NEXT:    pextrb $8, %xmm2, %eax
65; SSE42-NEXT:    testb $1, %al
66; SSE42-NEXT:    je LBB1_4
67; SSE42-NEXT:  ## %bb.3: ## %cond.load1
68; SSE42-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
69; SSE42-NEXT:  LBB1_4: ## %else2
70; SSE42-NEXT:    movapd %xmm1, %xmm0
71; SSE42-NEXT:    retq
72;
73; AVX1OR2-LABEL: load_v2f64_v2i64:
74; AVX1OR2:       ## %bb.0:
75; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
76; AVX1OR2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
77; AVX1OR2-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2
78; AVX1OR2-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
79; AVX1OR2-NEXT:    retq
80;
81; AVX512F-LABEL: load_v2f64_v2i64:
82; AVX512F:       ## %bb.0:
83; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
84; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
85; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
86; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
87; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
88; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
89; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
90; AVX512F-NEXT:    vzeroupper
91; AVX512F-NEXT:    retq
92;
93; AVX512VLBW-LABEL: load_v2f64_v2i64:
94; AVX512VLBW:       ## %bb.0:
95; AVX512VLBW-NEXT:    vptestnmq %xmm0, %xmm0, %k1
96; AVX512VLBW-NEXT:    vblendmpd (%rdi), %xmm1, %xmm0 {%k1}
97; AVX512VLBW-NEXT:    retq
98  %mask = icmp eq <2 x i64> %trigger, zeroinitializer
99  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1>%mask, <2 x double>%dst)
100  ret <2 x double> %res
101}
102
103define <4 x float> @load_v4f32_v4i32(<4 x i32> %trigger, <4 x float>* %addr, <4 x float> %dst) {
104; SSE2-LABEL: load_v4f32_v4i32:
105; SSE2:       ## %bb.0:
106; SSE2-NEXT:    pxor %xmm2, %xmm2
107; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
108; SSE2-NEXT:    movd %xmm2, %eax
109; SSE2-NEXT:    testb $1, %al
110; SSE2-NEXT:    je LBB2_2
111; SSE2-NEXT:  ## %bb.1: ## %cond.load
112; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
113; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
114; SSE2-NEXT:  LBB2_2: ## %else
115; SSE2-NEXT:    pextrw $2, %xmm2, %eax
116; SSE2-NEXT:    testb $1, %al
117; SSE2-NEXT:    je LBB2_4
118; SSE2-NEXT:  ## %bb.3: ## %cond.load1
119; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
120; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
121; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
122; SSE2-NEXT:    movaps %xmm2, %xmm1
123; SSE2-NEXT:  LBB2_4: ## %else2
124; SSE2-NEXT:    xorps %xmm2, %xmm2
125; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
126; SSE2-NEXT:    pextrw $4, %xmm0, %eax
127; SSE2-NEXT:    testb $1, %al
128; SSE2-NEXT:    je LBB2_6
129; SSE2-NEXT:  ## %bb.5: ## %cond.load4
130; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
131; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
132; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
133; SSE2-NEXT:  LBB2_6: ## %else5
134; SSE2-NEXT:    pextrw $6, %xmm0, %eax
135; SSE2-NEXT:    testb $1, %al
136; SSE2-NEXT:    je LBB2_8
137; SSE2-NEXT:  ## %bb.7: ## %cond.load7
138; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
139; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
140; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
141; SSE2-NEXT:  LBB2_8: ## %else8
142; SSE2-NEXT:    movaps %xmm1, %xmm0
143; SSE2-NEXT:    retq
144;
145; SSE42-LABEL: load_v4f32_v4i32:
146; SSE42:       ## %bb.0:
147; SSE42-NEXT:    pxor %xmm2, %xmm2
148; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
149; SSE42-NEXT:    pextrb $0, %xmm2, %eax
150; SSE42-NEXT:    testb $1, %al
151; SSE42-NEXT:    je LBB2_2
152; SSE42-NEXT:  ## %bb.1: ## %cond.load
153; SSE42-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
154; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
155; SSE42-NEXT:  LBB2_2: ## %else
156; SSE42-NEXT:    pextrb $4, %xmm2, %eax
157; SSE42-NEXT:    testb $1, %al
158; SSE42-NEXT:    je LBB2_4
159; SSE42-NEXT:  ## %bb.3: ## %cond.load1
160; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
161; SSE42-NEXT:  LBB2_4: ## %else2
162; SSE42-NEXT:    pxor %xmm2, %xmm2
163; SSE42-NEXT:    pcmpeqd %xmm2, %xmm0
164; SSE42-NEXT:    pextrb $8, %xmm0, %eax
165; SSE42-NEXT:    testb $1, %al
166; SSE42-NEXT:    je LBB2_6
167; SSE42-NEXT:  ## %bb.5: ## %cond.load4
168; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
169; SSE42-NEXT:  LBB2_6: ## %else5
170; SSE42-NEXT:    pextrb $12, %xmm0, %eax
171; SSE42-NEXT:    testb $1, %al
172; SSE42-NEXT:    je LBB2_8
173; SSE42-NEXT:  ## %bb.7: ## %cond.load7
174; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
175; SSE42-NEXT:  LBB2_8: ## %else8
176; SSE42-NEXT:    movaps %xmm1, %xmm0
177; SSE42-NEXT:    retq
178;
179; AVX1OR2-LABEL: load_v4f32_v4i32:
180; AVX1OR2:       ## %bb.0:
181; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
182; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
183; AVX1OR2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
184; AVX1OR2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
185; AVX1OR2-NEXT:    retq
186;
187; AVX512F-LABEL: load_v4f32_v4i32:
188; AVX512F:       ## %bb.0:
189; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
190; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
191; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
192; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
193; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
194; AVX512F-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
195; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
196; AVX512F-NEXT:    vzeroupper
197; AVX512F-NEXT:    retq
198;
199; AVX512VLBW-LABEL: load_v4f32_v4i32:
200; AVX512VLBW:       ## %bb.0:
201; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k1
202; AVX512VLBW-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
203; AVX512VLBW-NEXT:    retq
204  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
205  %res = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %addr, i32 4, <4 x i1>%mask, <4 x float>%dst)
206  ret <4 x float> %res
207}
208
209define <4 x i32> @load_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
210; SSE2-LABEL: load_v4i32_v4i32:
211; SSE2:       ## %bb.0:
212; SSE2-NEXT:    pxor %xmm2, %xmm2
213; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
214; SSE2-NEXT:    movd %xmm2, %eax
215; SSE2-NEXT:    testb $1, %al
216; SSE2-NEXT:    je LBB3_2
217; SSE2-NEXT:  ## %bb.1: ## %cond.load
218; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
219; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
220; SSE2-NEXT:  LBB3_2: ## %else
221; SSE2-NEXT:    pextrw $2, %xmm2, %eax
222; SSE2-NEXT:    testb $1, %al
223; SSE2-NEXT:    je LBB3_4
224; SSE2-NEXT:  ## %bb.3: ## %cond.load1
225; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
226; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
227; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
228; SSE2-NEXT:    movaps %xmm2, %xmm1
229; SSE2-NEXT:  LBB3_4: ## %else2
230; SSE2-NEXT:    xorps %xmm2, %xmm2
231; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
232; SSE2-NEXT:    pextrw $4, %xmm0, %eax
233; SSE2-NEXT:    testb $1, %al
234; SSE2-NEXT:    je LBB3_6
235; SSE2-NEXT:  ## %bb.5: ## %cond.load4
236; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
237; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
238; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
239; SSE2-NEXT:  LBB3_6: ## %else5
240; SSE2-NEXT:    pextrw $6, %xmm0, %eax
241; SSE2-NEXT:    testb $1, %al
242; SSE2-NEXT:    je LBB3_8
243; SSE2-NEXT:  ## %bb.7: ## %cond.load7
244; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
245; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
246; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
247; SSE2-NEXT:  LBB3_8: ## %else8
248; SSE2-NEXT:    movaps %xmm1, %xmm0
249; SSE2-NEXT:    retq
250;
251; SSE42-LABEL: load_v4i32_v4i32:
252; SSE42:       ## %bb.0:
253; SSE42-NEXT:    pxor %xmm2, %xmm2
254; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
255; SSE42-NEXT:    pextrb $0, %xmm2, %eax
256; SSE42-NEXT:    testb $1, %al
257; SSE42-NEXT:    je LBB3_2
258; SSE42-NEXT:  ## %bb.1: ## %cond.load
259; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm1
260; SSE42-NEXT:  LBB3_2: ## %else
261; SSE42-NEXT:    pextrb $4, %xmm2, %eax
262; SSE42-NEXT:    testb $1, %al
263; SSE42-NEXT:    je LBB3_4
264; SSE42-NEXT:  ## %bb.3: ## %cond.load1
265; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm1
266; SSE42-NEXT:  LBB3_4: ## %else2
267; SSE42-NEXT:    pxor %xmm2, %xmm2
268; SSE42-NEXT:    pcmpeqd %xmm2, %xmm0
269; SSE42-NEXT:    pextrb $8, %xmm0, %eax
270; SSE42-NEXT:    testb $1, %al
271; SSE42-NEXT:    je LBB3_6
272; SSE42-NEXT:  ## %bb.5: ## %cond.load4
273; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm1
274; SSE42-NEXT:  LBB3_6: ## %else5
275; SSE42-NEXT:    pextrb $12, %xmm0, %eax
276; SSE42-NEXT:    testb $1, %al
277; SSE42-NEXT:    je LBB3_8
278; SSE42-NEXT:  ## %bb.7: ## %cond.load7
279; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm1
280; SSE42-NEXT:  LBB3_8: ## %else8
281; SSE42-NEXT:    movdqa %xmm1, %xmm0
282; SSE42-NEXT:    retq
283;
284; AVX1-LABEL: load_v4i32_v4i32:
285; AVX1:       ## %bb.0:
286; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
287; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
288; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
289; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
290; AVX1-NEXT:    retq
291;
292; AVX2-LABEL: load_v4i32_v4i32:
293; AVX2:       ## %bb.0:
294; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
295; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
296; AVX2-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2
297; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
298; AVX2-NEXT:    retq
299;
300; AVX512F-LABEL: load_v4i32_v4i32:
301; AVX512F:       ## %bb.0:
302; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
303; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
304; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
305; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
306; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
307; AVX512F-NEXT:    vpblendmd (%rdi), %zmm1, %zmm0 {%k1}
308; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
309; AVX512F-NEXT:    vzeroupper
310; AVX512F-NEXT:    retq
311;
312; AVX512VLBW-LABEL: load_v4i32_v4i32:
313; AVX512VLBW:       ## %bb.0:
314; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k1
315; AVX512VLBW-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
316; AVX512VLBW-NEXT:    retq
317  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
318  %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1>%mask, <4 x i32>%dst)
319  ret <4 x i32> %res
320}
321
322define <4 x double> @load_v4f64_v4i32(<4 x i32> %trigger, <4 x double>* %addr, <4 x double> %dst) {
323; SSE2-LABEL: load_v4f64_v4i32:
324; SSE2:       ## %bb.0:
325; SSE2-NEXT:    pxor %xmm3, %xmm3
326; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
327; SSE2-NEXT:    movd %xmm3, %eax
328; SSE2-NEXT:    testb $1, %al
329; SSE2-NEXT:    je LBB4_2
330; SSE2-NEXT:  ## %bb.1: ## %cond.load
331; SSE2-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
332; SSE2-NEXT:  LBB4_2: ## %else
333; SSE2-NEXT:    pextrw $2, %xmm3, %eax
334; SSE2-NEXT:    testb $1, %al
335; SSE2-NEXT:    je LBB4_4
336; SSE2-NEXT:  ## %bb.3: ## %cond.load1
337; SSE2-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
338; SSE2-NEXT:  LBB4_4: ## %else2
339; SSE2-NEXT:    pxor %xmm3, %xmm3
340; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
341; SSE2-NEXT:    pextrw $4, %xmm0, %eax
342; SSE2-NEXT:    testb $1, %al
343; SSE2-NEXT:    je LBB4_6
344; SSE2-NEXT:  ## %bb.5: ## %cond.load4
345; SSE2-NEXT:    movlpd {{.*#+}} xmm2 = mem[0],xmm2[1]
346; SSE2-NEXT:  LBB4_6: ## %else5
347; SSE2-NEXT:    pextrw $6, %xmm0, %eax
348; SSE2-NEXT:    testb $1, %al
349; SSE2-NEXT:    je LBB4_8
350; SSE2-NEXT:  ## %bb.7: ## %cond.load7
351; SSE2-NEXT:    movhpd {{.*#+}} xmm2 = xmm2[0],mem[0]
352; SSE2-NEXT:  LBB4_8: ## %else8
353; SSE2-NEXT:    movapd %xmm1, %xmm0
354; SSE2-NEXT:    movapd %xmm2, %xmm1
355; SSE2-NEXT:    retq
356;
357; SSE42-LABEL: load_v4f64_v4i32:
358; SSE42:       ## %bb.0:
359; SSE42-NEXT:    pxor %xmm3, %xmm3
360; SSE42-NEXT:    pcmpeqd %xmm0, %xmm3
361; SSE42-NEXT:    pextrb $0, %xmm3, %eax
362; SSE42-NEXT:    testb $1, %al
363; SSE42-NEXT:    je LBB4_2
364; SSE42-NEXT:  ## %bb.1: ## %cond.load
365; SSE42-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
366; SSE42-NEXT:  LBB4_2: ## %else
367; SSE42-NEXT:    pextrb $4, %xmm3, %eax
368; SSE42-NEXT:    testb $1, %al
369; SSE42-NEXT:    je LBB4_4
370; SSE42-NEXT:  ## %bb.3: ## %cond.load1
371; SSE42-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
372; SSE42-NEXT:  LBB4_4: ## %else2
373; SSE42-NEXT:    pxor %xmm3, %xmm3
374; SSE42-NEXT:    pcmpeqd %xmm3, %xmm0
375; SSE42-NEXT:    pextrb $8, %xmm0, %eax
376; SSE42-NEXT:    testb $1, %al
377; SSE42-NEXT:    je LBB4_6
378; SSE42-NEXT:  ## %bb.5: ## %cond.load4
379; SSE42-NEXT:    movlpd {{.*#+}} xmm2 = mem[0],xmm2[1]
380; SSE42-NEXT:  LBB4_6: ## %else5
381; SSE42-NEXT:    pextrb $12, %xmm0, %eax
382; SSE42-NEXT:    testb $1, %al
383; SSE42-NEXT:    je LBB4_8
384; SSE42-NEXT:  ## %bb.7: ## %cond.load7
385; SSE42-NEXT:    movhpd {{.*#+}} xmm2 = xmm2[0],mem[0]
386; SSE42-NEXT:  LBB4_8: ## %else8
387; SSE42-NEXT:    movapd %xmm1, %xmm0
388; SSE42-NEXT:    movapd %xmm2, %xmm1
389; SSE42-NEXT:    retq
390;
391; AVX1-LABEL: load_v4f64_v4i32:
392; AVX1:       ## %bb.0:
393; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
394; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
395; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm2
396; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
397; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
398; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
399; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2
400; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
401; AVX1-NEXT:    retq
402;
403; AVX2-LABEL: load_v4f64_v4i32:
404; AVX2:       ## %bb.0:
405; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
406; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
407; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
408; AVX2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2
409; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
410; AVX2-NEXT:    retq
411;
412; AVX512F-LABEL: load_v4f64_v4i32:
413; AVX512F:       ## %bb.0:
414; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
415; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
416; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
417; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
418; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
419; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
420; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
421; AVX512F-NEXT:    retq
422;
423; AVX512VLBW-LABEL: load_v4f64_v4i32:
424; AVX512VLBW:       ## %bb.0:
425; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k1
426; AVX512VLBW-NEXT:    vblendmpd (%rdi), %ymm1, %ymm0 {%k1}
427; AVX512VLBW-NEXT:    retq
428  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
429  %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 32, <4 x i1>%mask, <4 x double>%dst)
430  ret <4 x double> %res
431}
432
433define <4 x double> @load_zero_v4f64_v4i32(<4 x i32> %trigger, <4 x double>* %addr, <4 x double> %dst) {
434; SSE2-LABEL: load_zero_v4f64_v4i32:
435; SSE2:       ## %bb.0:
436; SSE2-NEXT:    movdqa %xmm0, %xmm2
437; SSE2-NEXT:    pxor %xmm0, %xmm0
438; SSE2-NEXT:    movdqa %xmm2, %xmm3
439; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
440; SSE2-NEXT:    movd %xmm3, %eax
441; SSE2-NEXT:    testb $1, %al
442; SSE2-NEXT:    xorpd %xmm1, %xmm1
443; SSE2-NEXT:    je LBB5_2
444; SSE2-NEXT:  ## %bb.1: ## %cond.load
445; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
446; SSE2-NEXT:    xorpd %xmm1, %xmm1
447; SSE2-NEXT:  LBB5_2: ## %else
448; SSE2-NEXT:    pextrw $2, %xmm3, %eax
449; SSE2-NEXT:    testb $1, %al
450; SSE2-NEXT:    je LBB5_4
451; SSE2-NEXT:  ## %bb.3: ## %cond.load1
452; SSE2-NEXT:    movhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
453; SSE2-NEXT:  LBB5_4: ## %else2
454; SSE2-NEXT:    pxor %xmm3, %xmm3
455; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
456; SSE2-NEXT:    pextrw $4, %xmm2, %eax
457; SSE2-NEXT:    testb $1, %al
458; SSE2-NEXT:    je LBB5_6
459; SSE2-NEXT:  ## %bb.5: ## %cond.load4
460; SSE2-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
461; SSE2-NEXT:  LBB5_6: ## %else5
462; SSE2-NEXT:    pextrw $6, %xmm2, %eax
463; SSE2-NEXT:    testb $1, %al
464; SSE2-NEXT:    je LBB5_8
465; SSE2-NEXT:  ## %bb.7: ## %cond.load7
466; SSE2-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
467; SSE2-NEXT:  LBB5_8: ## %else8
468; SSE2-NEXT:    retq
469;
470; SSE42-LABEL: load_zero_v4f64_v4i32:
471; SSE42:       ## %bb.0:
472; SSE42-NEXT:    movdqa %xmm0, %xmm2
473; SSE42-NEXT:    pxor %xmm0, %xmm0
474; SSE42-NEXT:    movdqa %xmm2, %xmm3
475; SSE42-NEXT:    pcmpeqd %xmm0, %xmm3
476; SSE42-NEXT:    pextrb $0, %xmm3, %eax
477; SSE42-NEXT:    testb $1, %al
478; SSE42-NEXT:    xorpd %xmm1, %xmm1
479; SSE42-NEXT:    je LBB5_2
480; SSE42-NEXT:  ## %bb.1: ## %cond.load
481; SSE42-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
482; SSE42-NEXT:    xorpd %xmm1, %xmm1
483; SSE42-NEXT:  LBB5_2: ## %else
484; SSE42-NEXT:    pextrb $4, %xmm3, %eax
485; SSE42-NEXT:    testb $1, %al
486; SSE42-NEXT:    je LBB5_4
487; SSE42-NEXT:  ## %bb.3: ## %cond.load1
488; SSE42-NEXT:    movhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
489; SSE42-NEXT:  LBB5_4: ## %else2
490; SSE42-NEXT:    pxor %xmm3, %xmm3
491; SSE42-NEXT:    pcmpeqd %xmm3, %xmm2
492; SSE42-NEXT:    pextrb $8, %xmm2, %eax
493; SSE42-NEXT:    testb $1, %al
494; SSE42-NEXT:    je LBB5_6
495; SSE42-NEXT:  ## %bb.5: ## %cond.load4
496; SSE42-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
497; SSE42-NEXT:  LBB5_6: ## %else5
498; SSE42-NEXT:    pextrb $12, %xmm2, %eax
499; SSE42-NEXT:    testb $1, %al
500; SSE42-NEXT:    je LBB5_8
501; SSE42-NEXT:  ## %bb.7: ## %cond.load7
502; SSE42-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
503; SSE42-NEXT:  LBB5_8: ## %else8
504; SSE42-NEXT:    retq
505;
506; AVX1-LABEL: load_zero_v4f64_v4i32:
507; AVX1:       ## %bb.0:
508; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
509; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
510; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
511; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
512; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
513; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
514; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm0
515; AVX1-NEXT:    retq
516;
517; AVX2-LABEL: load_zero_v4f64_v4i32:
518; AVX2:       ## %bb.0:
519; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
520; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
521; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
522; AVX2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm0
523; AVX2-NEXT:    retq
524;
525; AVX512F-LABEL: load_zero_v4f64_v4i32:
526; AVX512F:       ## %bb.0:
527; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
528; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
529; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
530; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
531; AVX512F-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z}
532; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
533; AVX512F-NEXT:    retq
534;
535; AVX512VLBW-LABEL: load_zero_v4f64_v4i32:
536; AVX512VLBW:       ## %bb.0:
537; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k1
538; AVX512VLBW-NEXT:    vmovapd (%rdi), %ymm0 {%k1} {z}
539; AVX512VLBW-NEXT:    retq
540  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
541  %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 32, <4 x i1>%mask, <4 x double>zeroinitializer)
542  ret <4 x double> %res
543}
544
545define <8 x float> @load_v8f32_v8i32(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %dst) {
546; SSE2-LABEL: load_v8f32_v8i32:
547; SSE2:       ## %bb.0:
548; SSE2-NEXT:    pxor %xmm4, %xmm4
549; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
550; SSE2-NEXT:    movdqa %xmm4, %xmm5
551; SSE2-NEXT:    packssdw %xmm0, %xmm5
552; SSE2-NEXT:    movd %xmm5, %eax
553; SSE2-NEXT:    testb $1, %al
554; SSE2-NEXT:    je LBB6_2
555; SSE2-NEXT:  ## %bb.1: ## %cond.load
556; SSE2-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
557; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm5[0],xmm2[1,2,3]
558; SSE2-NEXT:  LBB6_2: ## %else
559; SSE2-NEXT:    psrlq $16, %xmm4
560; SSE2-NEXT:    movd %xmm4, %eax
561; SSE2-NEXT:    shrl $16, %eax
562; SSE2-NEXT:    testb $1, %al
563; SSE2-NEXT:    je LBB6_4
564; SSE2-NEXT:  ## %bb.3: ## %cond.load1
565; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
566; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,0],xmm2[0,0]
567; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[2,3]
568; SSE2-NEXT:    movaps %xmm4, %xmm2
569; SSE2-NEXT:  LBB6_4: ## %else2
570; SSE2-NEXT:    xorps %xmm4, %xmm4
571; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
572; SSE2-NEXT:    pextrw $4, %xmm0, %eax
573; SSE2-NEXT:    testb $1, %al
574; SSE2-NEXT:    je LBB6_6
575; SSE2-NEXT:  ## %bb.5: ## %cond.load4
576; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
577; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,0],xmm2[3,0]
578; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
579; SSE2-NEXT:  LBB6_6: ## %else5
580; SSE2-NEXT:    pextrw $6, %xmm0, %eax
581; SSE2-NEXT:    testb $1, %al
582; SSE2-NEXT:    je LBB6_8
583; SSE2-NEXT:  ## %bb.7: ## %cond.load7
584; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
585; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
586; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
587; SSE2-NEXT:  LBB6_8: ## %else8
588; SSE2-NEXT:    xorps %xmm0, %xmm0
589; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
590; SSE2-NEXT:    pextrw $0, %xmm0, %eax
591; SSE2-NEXT:    testb $1, %al
592; SSE2-NEXT:    je LBB6_10
593; SSE2-NEXT:  ## %bb.9: ## %cond.load10
594; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
595; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
596; SSE2-NEXT:  LBB6_10: ## %else11
597; SSE2-NEXT:    pextrw $2, %xmm0, %eax
598; SSE2-NEXT:    testb $1, %al
599; SSE2-NEXT:    je LBB6_12
600; SSE2-NEXT:  ## %bb.11: ## %cond.load13
601; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
602; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[0,0]
603; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[2,3]
604; SSE2-NEXT:    movaps %xmm0, %xmm3
605; SSE2-NEXT:  LBB6_12: ## %else14
606; SSE2-NEXT:    xorps %xmm0, %xmm0
607; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
608; SSE2-NEXT:    pextrw $4, %xmm1, %eax
609; SSE2-NEXT:    testb $1, %al
610; SSE2-NEXT:    je LBB6_14
611; SSE2-NEXT:  ## %bb.13: ## %cond.load16
612; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
613; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[3,0]
614; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[0,2]
615; SSE2-NEXT:  LBB6_14: ## %else17
616; SSE2-NEXT:    pextrw $6, %xmm1, %eax
617; SSE2-NEXT:    testb $1, %al
618; SSE2-NEXT:    je LBB6_16
619; SSE2-NEXT:  ## %bb.15: ## %cond.load19
620; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
621; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[2,0]
622; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,0]
623; SSE2-NEXT:  LBB6_16: ## %else20
624; SSE2-NEXT:    movaps %xmm2, %xmm0
625; SSE2-NEXT:    movaps %xmm3, %xmm1
626; SSE2-NEXT:    retq
627;
628; SSE42-LABEL: load_v8f32_v8i32:
629; SSE42:       ## %bb.0:
630; SSE42-NEXT:    pxor %xmm4, %xmm4
631; SSE42-NEXT:    pcmpeqd %xmm0, %xmm4
632; SSE42-NEXT:    pextrb $0, %xmm4, %eax
633; SSE42-NEXT:    testb $1, %al
634; SSE42-NEXT:    je LBB6_2
635; SSE42-NEXT:  ## %bb.1: ## %cond.load
636; SSE42-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
637; SSE42-NEXT:    blendps {{.*#+}} xmm2 = xmm5[0],xmm2[1,2,3]
638; SSE42-NEXT:  LBB6_2: ## %else
639; SSE42-NEXT:    pextrb $4, %xmm4, %eax
640; SSE42-NEXT:    testb $1, %al
641; SSE42-NEXT:    je LBB6_4
642; SSE42-NEXT:  ## %bb.3: ## %cond.load1
643; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
644; SSE42-NEXT:  LBB6_4: ## %else2
645; SSE42-NEXT:    pxor %xmm4, %xmm4
646; SSE42-NEXT:    pcmpeqd %xmm4, %xmm0
647; SSE42-NEXT:    pextrb $8, %xmm0, %eax
648; SSE42-NEXT:    testb $1, %al
649; SSE42-NEXT:    je LBB6_6
650; SSE42-NEXT:  ## %bb.5: ## %cond.load4
651; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
652; SSE42-NEXT:  LBB6_6: ## %else5
653; SSE42-NEXT:    pextrb $12, %xmm0, %eax
654; SSE42-NEXT:    testb $1, %al
655; SSE42-NEXT:    je LBB6_8
656; SSE42-NEXT:  ## %bb.7: ## %cond.load7
657; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
658; SSE42-NEXT:  LBB6_8: ## %else8
659; SSE42-NEXT:    pxor %xmm0, %xmm0
660; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
661; SSE42-NEXT:    pextrb $0, %xmm0, %eax
662; SSE42-NEXT:    testb $1, %al
663; SSE42-NEXT:    je LBB6_10
664; SSE42-NEXT:  ## %bb.9: ## %cond.load10
665; SSE42-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
666; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3,4,5,6,7]
667; SSE42-NEXT:  LBB6_10: ## %else11
668; SSE42-NEXT:    pextrb $4, %xmm0, %eax
669; SSE42-NEXT:    testb $1, %al
670; SSE42-NEXT:    je LBB6_12
671; SSE42-NEXT:  ## %bb.11: ## %cond.load13
672; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
673; SSE42-NEXT:  LBB6_12: ## %else14
674; SSE42-NEXT:    pxor %xmm0, %xmm0
675; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
676; SSE42-NEXT:    pextrb $8, %xmm1, %eax
677; SSE42-NEXT:    testb $1, %al
678; SSE42-NEXT:    je LBB6_14
679; SSE42-NEXT:  ## %bb.13: ## %cond.load16
680; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
681; SSE42-NEXT:  LBB6_14: ## %else17
682; SSE42-NEXT:    pextrb $12, %xmm1, %eax
683; SSE42-NEXT:    testb $1, %al
684; SSE42-NEXT:    je LBB6_16
685; SSE42-NEXT:  ## %bb.15: ## %cond.load19
686; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
687; SSE42-NEXT:  LBB6_16: ## %else20
688; SSE42-NEXT:    movaps %xmm2, %xmm0
689; SSE42-NEXT:    movaps %xmm3, %xmm1
690; SSE42-NEXT:    retq
691;
692; AVX1-LABEL: load_v8f32_v8i32:
693; AVX1:       ## %bb.0:
694; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
695; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
696; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
697; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
698; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
699; AVX1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2
700; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
701; AVX1-NEXT:    retq
702;
703; AVX2-LABEL: load_v8f32_v8i32:
704; AVX2:       ## %bb.0:
705; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
706; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm0
707; AVX2-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2
708; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
709; AVX2-NEXT:    retq
710;
711; AVX512F-LABEL: load_v8f32_v8i32:
712; AVX512F:       ## %bb.0:
713; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
714; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
715; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
716; AVX512F-NEXT:    kshiftlw $8, %k0, %k0
717; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
718; AVX512F-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
719; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
720; AVX512F-NEXT:    retq
721;
722; AVX512VLBW-LABEL: load_v8f32_v8i32:
723; AVX512VLBW:       ## %bb.0:
724; AVX512VLBW-NEXT:    vptestnmd %ymm0, %ymm0, %k1
725; AVX512VLBW-NEXT:    vblendmps (%rdi), %ymm1, %ymm0 {%k1}
726; AVX512VLBW-NEXT:    retq
727  %mask = icmp eq <8 x i32> %trigger, zeroinitializer
728  %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 32, <8 x i1>%mask, <8 x float>%dst)
729  ret <8 x float> %res
730}
731
732define <8 x i32> @load_v8i32_v8i1(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %dst) {
733; SSE2-LABEL: load_v8i32_v8i1:
734; SSE2:       ## %bb.0:
735; SSE2-NEXT:    movd %xmm0, %eax
736; SSE2-NEXT:    testb $1, %al
737; SSE2-NEXT:    je LBB7_2
738; SSE2-NEXT:  ## %bb.1: ## %cond.load
739; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
740; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
741; SSE2-NEXT:  LBB7_2: ## %else
742; SSE2-NEXT:    shrl $16, %eax
743; SSE2-NEXT:    testb $1, %al
744; SSE2-NEXT:    je LBB7_4
745; SSE2-NEXT:  ## %bb.3: ## %cond.load1
746; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
747; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
748; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
749; SSE2-NEXT:    movaps %xmm3, %xmm1
750; SSE2-NEXT:  LBB7_4: ## %else2
751; SSE2-NEXT:    pextrw $2, %xmm0, %eax
752; SSE2-NEXT:    testb $1, %al
753; SSE2-NEXT:    je LBB7_6
754; SSE2-NEXT:  ## %bb.5: ## %cond.load4
755; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
756; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
757; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
758; SSE2-NEXT:  LBB7_6: ## %else5
759; SSE2-NEXT:    pextrw $3, %xmm0, %eax
760; SSE2-NEXT:    testb $1, %al
761; SSE2-NEXT:    je LBB7_8
762; SSE2-NEXT:  ## %bb.7: ## %cond.load7
763; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
764; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[2,0]
765; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
766; SSE2-NEXT:  LBB7_8: ## %else8
767; SSE2-NEXT:    pextrw $4, %xmm0, %eax
768; SSE2-NEXT:    testb $1, %al
769; SSE2-NEXT:    je LBB7_10
770; SSE2-NEXT:  ## %bb.9: ## %cond.load10
771; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
772; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
773; SSE2-NEXT:  LBB7_10: ## %else11
774; SSE2-NEXT:    pextrw $5, %xmm0, %eax
775; SSE2-NEXT:    testb $1, %al
776; SSE2-NEXT:    je LBB7_12
777; SSE2-NEXT:  ## %bb.11: ## %cond.load13
778; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
779; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
780; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,3]
781; SSE2-NEXT:    movaps %xmm3, %xmm2
782; SSE2-NEXT:  LBB7_12: ## %else14
783; SSE2-NEXT:    pextrw $6, %xmm0, %eax
784; SSE2-NEXT:    testb $1, %al
785; SSE2-NEXT:    je LBB7_14
786; SSE2-NEXT:  ## %bb.13: ## %cond.load16
787; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
788; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[3,0]
789; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0,2]
790; SSE2-NEXT:  LBB7_14: ## %else17
791; SSE2-NEXT:    pextrw $7, %xmm0, %eax
792; SSE2-NEXT:    testb $1, %al
793; SSE2-NEXT:    je LBB7_16
794; SSE2-NEXT:  ## %bb.15: ## %cond.load19
795; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
796; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
797; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
798; SSE2-NEXT:  LBB7_16: ## %else20
799; SSE2-NEXT:    movaps %xmm1, %xmm0
800; SSE2-NEXT:    movaps %xmm2, %xmm1
801; SSE2-NEXT:    retq
802;
803; SSE42-LABEL: load_v8i32_v8i1:
804; SSE42:       ## %bb.0:
805; SSE42-NEXT:    pextrb $0, %xmm0, %eax
806; SSE42-NEXT:    testb $1, %al
807; SSE42-NEXT:    je LBB7_2
808; SSE42-NEXT:  ## %bb.1: ## %cond.load
809; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm1
810; SSE42-NEXT:  LBB7_2: ## %else
811; SSE42-NEXT:    pextrb $2, %xmm0, %eax
812; SSE42-NEXT:    testb $1, %al
813; SSE42-NEXT:    je LBB7_4
814; SSE42-NEXT:  ## %bb.3: ## %cond.load1
815; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm1
816; SSE42-NEXT:  LBB7_4: ## %else2
817; SSE42-NEXT:    pextrb $4, %xmm0, %eax
818; SSE42-NEXT:    testb $1, %al
819; SSE42-NEXT:    je LBB7_6
820; SSE42-NEXT:  ## %bb.5: ## %cond.load4
821; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm1
822; SSE42-NEXT:  LBB7_6: ## %else5
823; SSE42-NEXT:    pextrb $6, %xmm0, %eax
824; SSE42-NEXT:    testb $1, %al
825; SSE42-NEXT:    je LBB7_8
826; SSE42-NEXT:  ## %bb.7: ## %cond.load7
827; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm1
828; SSE42-NEXT:  LBB7_8: ## %else8
829; SSE42-NEXT:    pextrb $8, %xmm0, %eax
830; SSE42-NEXT:    testb $1, %al
831; SSE42-NEXT:    je LBB7_10
832; SSE42-NEXT:  ## %bb.9: ## %cond.load10
833; SSE42-NEXT:    pinsrd $0, 16(%rdi), %xmm2
834; SSE42-NEXT:  LBB7_10: ## %else11
835; SSE42-NEXT:    pextrb $10, %xmm0, %eax
836; SSE42-NEXT:    testb $1, %al
837; SSE42-NEXT:    je LBB7_12
838; SSE42-NEXT:  ## %bb.11: ## %cond.load13
839; SSE42-NEXT:    pinsrd $1, 20(%rdi), %xmm2
840; SSE42-NEXT:  LBB7_12: ## %else14
841; SSE42-NEXT:    pextrb $12, %xmm0, %eax
842; SSE42-NEXT:    testb $1, %al
843; SSE42-NEXT:    je LBB7_14
844; SSE42-NEXT:  ## %bb.13: ## %cond.load16
845; SSE42-NEXT:    pinsrd $2, 24(%rdi), %xmm2
846; SSE42-NEXT:  LBB7_14: ## %else17
847; SSE42-NEXT:    pextrb $14, %xmm0, %eax
848; SSE42-NEXT:    testb $1, %al
849; SSE42-NEXT:    je LBB7_16
850; SSE42-NEXT:  ## %bb.15: ## %cond.load19
851; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm2
852; SSE42-NEXT:  LBB7_16: ## %else20
853; SSE42-NEXT:    movdqa %xmm1, %xmm0
854; SSE42-NEXT:    movdqa %xmm2, %xmm1
855; SSE42-NEXT:    retq
856;
857; AVX1-LABEL: load_v8i32_v8i1:
858; AVX1:       ## %bb.0:
859; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
860; AVX1-NEXT:    vpslld $31, %xmm2, %xmm2
861; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
862; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
863; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
864; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
865; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
866; AVX1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2
867; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
868; AVX1-NEXT:    retq
869;
870; AVX2-LABEL: load_v8i32_v8i1:
871; AVX2:       ## %bb.0:
872; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
873; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
874; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
875; AVX2-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2
876; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
877; AVX2-NEXT:    retq
878;
879; AVX512F-LABEL: load_v8i32_v8i1:
880; AVX512F:       ## %bb.0:
881; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
882; AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
883; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
884; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
885; AVX512F-NEXT:    vpblendmd (%rdi), %zmm1, %zmm0 {%k1}
886; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
887; AVX512F-NEXT:    retq
888;
889; AVX512VLBW-LABEL: load_v8i32_v8i1:
890; AVX512VLBW:       ## %bb.0:
891; AVX512VLBW-NEXT:    vpsllw $15, %xmm0, %xmm0
892; AVX512VLBW-NEXT:    vpmovw2m %xmm0, %k1
893; AVX512VLBW-NEXT:    vpblendmd (%rdi), %ymm1, %ymm0 {%k1}
894; AVX512VLBW-NEXT:    retq
895  %res = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* %addr, i32 4, <8 x i1>%mask, <8 x i32>%dst)
896  ret <8 x i32> %res
897}
898
899define <8 x float> @load_zero_v8f32_v8i1(<8 x i1> %mask, <8 x float>* %addr) {
900; SSE2-LABEL: load_zero_v8f32_v8i1:
901; SSE2:       ## %bb.0:
902; SSE2-NEXT:    movdqa %xmm0, %xmm2
903; SSE2-NEXT:    movd %xmm0, %eax
904; SSE2-NEXT:    pxor %xmm0, %xmm0
905; SSE2-NEXT:    testb $1, %al
906; SSE2-NEXT:    xorps %xmm1, %xmm1
907; SSE2-NEXT:    je LBB8_2
908; SSE2-NEXT:  ## %bb.1: ## %cond.load
909; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
910; SSE2-NEXT:    xorps %xmm1, %xmm1
911; SSE2-NEXT:  LBB8_2: ## %else
912; SSE2-NEXT:    shrl $16, %eax
913; SSE2-NEXT:    testb $1, %al
914; SSE2-NEXT:    je LBB8_4
915; SSE2-NEXT:  ## %bb.3: ## %cond.load1
916; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
917; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[0,0]
918; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
919; SSE2-NEXT:    movaps %xmm3, %xmm0
920; SSE2-NEXT:  LBB8_4: ## %else2
921; SSE2-NEXT:    pextrw $2, %xmm2, %eax
922; SSE2-NEXT:    testb $1, %al
923; SSE2-NEXT:    je LBB8_6
924; SSE2-NEXT:  ## %bb.5: ## %cond.load4
925; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
926; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
927; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
928; SSE2-NEXT:  LBB8_6: ## %else5
929; SSE2-NEXT:    pextrw $3, %xmm2, %eax
930; SSE2-NEXT:    testb $1, %al
931; SSE2-NEXT:    je LBB8_8
932; SSE2-NEXT:  ## %bb.7: ## %cond.load7
933; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
934; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[2,0]
935; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
936; SSE2-NEXT:  LBB8_8: ## %else8
937; SSE2-NEXT:    pextrw $4, %xmm2, %eax
938; SSE2-NEXT:    testb $1, %al
939; SSE2-NEXT:    je LBB8_10
940; SSE2-NEXT:  ## %bb.9: ## %cond.load10
941; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
942; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
943; SSE2-NEXT:  LBB8_10: ## %else11
944; SSE2-NEXT:    pextrw $5, %xmm2, %eax
945; SSE2-NEXT:    testb $1, %al
946; SSE2-NEXT:    je LBB8_12
947; SSE2-NEXT:  ## %bb.11: ## %cond.load13
948; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
949; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
950; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
951; SSE2-NEXT:    movaps %xmm3, %xmm1
952; SSE2-NEXT:  LBB8_12: ## %else14
953; SSE2-NEXT:    pextrw $6, %xmm2, %eax
954; SSE2-NEXT:    testb $1, %al
955; SSE2-NEXT:    je LBB8_14
956; SSE2-NEXT:  ## %bb.13: ## %cond.load16
957; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
958; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
959; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
960; SSE2-NEXT:  LBB8_14: ## %else17
961; SSE2-NEXT:    pextrw $7, %xmm2, %eax
962; SSE2-NEXT:    testb $1, %al
963; SSE2-NEXT:    je LBB8_16
964; SSE2-NEXT:  ## %bb.15: ## %cond.load19
965; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
966; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
967; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
968; SSE2-NEXT:  LBB8_16: ## %else20
969; SSE2-NEXT:    retq
970;
971; SSE42-LABEL: load_zero_v8f32_v8i1:
972; SSE42:       ## %bb.0:
973; SSE42-NEXT:    movdqa %xmm0, %xmm2
974; SSE42-NEXT:    pextrb $0, %xmm0, %eax
975; SSE42-NEXT:    pxor %xmm0, %xmm0
976; SSE42-NEXT:    testb $1, %al
977; SSE42-NEXT:    xorps %xmm1, %xmm1
978; SSE42-NEXT:    je LBB8_2
979; SSE42-NEXT:  ## %bb.1: ## %cond.load
980; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
981; SSE42-NEXT:    xorps %xmm1, %xmm1
982; SSE42-NEXT:  LBB8_2: ## %else
983; SSE42-NEXT:    pextrb $2, %xmm2, %eax
984; SSE42-NEXT:    testb $1, %al
985; SSE42-NEXT:    je LBB8_4
986; SSE42-NEXT:  ## %bb.3: ## %cond.load1
987; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
988; SSE42-NEXT:  LBB8_4: ## %else2
989; SSE42-NEXT:    pextrb $4, %xmm2, %eax
990; SSE42-NEXT:    testb $1, %al
991; SSE42-NEXT:    je LBB8_6
992; SSE42-NEXT:  ## %bb.5: ## %cond.load4
993; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
994; SSE42-NEXT:  LBB8_6: ## %else5
995; SSE42-NEXT:    pextrb $6, %xmm2, %eax
996; SSE42-NEXT:    testb $1, %al
997; SSE42-NEXT:    je LBB8_8
998; SSE42-NEXT:  ## %bb.7: ## %cond.load7
999; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
1000; SSE42-NEXT:  LBB8_8: ## %else8
1001; SSE42-NEXT:    pextrb $8, %xmm2, %eax
1002; SSE42-NEXT:    testb $1, %al
1003; SSE42-NEXT:    je LBB8_10
1004; SSE42-NEXT:  ## %bb.9: ## %cond.load10
1005; SSE42-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1006; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
1007; SSE42-NEXT:  LBB8_10: ## %else11
1008; SSE42-NEXT:    pextrb $10, %xmm2, %eax
1009; SSE42-NEXT:    testb $1, %al
1010; SSE42-NEXT:    je LBB8_12
1011; SSE42-NEXT:  ## %bb.11: ## %cond.load13
1012; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
1013; SSE42-NEXT:  LBB8_12: ## %else14
1014; SSE42-NEXT:    pextrb $12, %xmm2, %eax
1015; SSE42-NEXT:    testb $1, %al
1016; SSE42-NEXT:    je LBB8_14
1017; SSE42-NEXT:  ## %bb.13: ## %cond.load16
1018; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
1019; SSE42-NEXT:  LBB8_14: ## %else17
1020; SSE42-NEXT:    pextrb $14, %xmm2, %eax
1021; SSE42-NEXT:    testb $1, %al
1022; SSE42-NEXT:    je LBB8_16
1023; SSE42-NEXT:  ## %bb.15: ## %cond.load19
1024; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
1025; SSE42-NEXT:  LBB8_16: ## %else20
1026; SSE42-NEXT:    retq
1027;
1028; AVX1-LABEL: load_zero_v8f32_v8i1:
1029; AVX1:       ## %bb.0:
1030; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1031; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
1032; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
1033; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1034; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
1035; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
1036; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1037; AVX1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm0
1038; AVX1-NEXT:    retq
1039;
1040; AVX2-LABEL: load_zero_v8f32_v8i1:
1041; AVX2:       ## %bb.0:
1042; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1043; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
1044; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
1045; AVX2-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm0
1046; AVX2-NEXT:    retq
1047;
1048; AVX512F-LABEL: load_zero_v8f32_v8i1:
1049; AVX512F:       ## %bb.0:
1050; AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
1051; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
1052; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
1053; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z}
1054; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1055; AVX512F-NEXT:    retq
1056;
1057; AVX512VLBW-LABEL: load_zero_v8f32_v8i1:
1058; AVX512VLBW:       ## %bb.0:
1059; AVX512VLBW-NEXT:    vpsllw $15, %xmm0, %xmm0
1060; AVX512VLBW-NEXT:    vpmovw2m %xmm0, %k1
1061; AVX512VLBW-NEXT:    vmovaps (%rdi), %ymm0 {%k1} {z}
1062; AVX512VLBW-NEXT:    retq
1063  %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 32, <8 x i1> %mask, <8 x float> zeroinitializer)
1064  ret <8 x float> %res
1065}
1066
1067define <8 x i32> @load_zero_v8i32_v8i1(<8 x i1> %mask, <8 x i32>* %addr) {
1068; SSE2-LABEL: load_zero_v8i32_v8i1:
1069; SSE2:       ## %bb.0:
1070; SSE2-NEXT:    movdqa %xmm0, %xmm2
1071; SSE2-NEXT:    movd %xmm0, %eax
1072; SSE2-NEXT:    pxor %xmm0, %xmm0
1073; SSE2-NEXT:    testb $1, %al
1074; SSE2-NEXT:    xorps %xmm1, %xmm1
1075; SSE2-NEXT:    je LBB9_2
1076; SSE2-NEXT:  ## %bb.1: ## %cond.load
1077; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1078; SSE2-NEXT:    xorps %xmm1, %xmm1
1079; SSE2-NEXT:  LBB9_2: ## %else
1080; SSE2-NEXT:    shrl $16, %eax
1081; SSE2-NEXT:    testb $1, %al
1082; SSE2-NEXT:    je LBB9_4
1083; SSE2-NEXT:  ## %bb.3: ## %cond.load1
1084; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1085; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[0,0]
1086; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
1087; SSE2-NEXT:    movaps %xmm3, %xmm0
1088; SSE2-NEXT:  LBB9_4: ## %else2
1089; SSE2-NEXT:    pextrw $2, %xmm2, %eax
1090; SSE2-NEXT:    testb $1, %al
1091; SSE2-NEXT:    je LBB9_6
1092; SSE2-NEXT:  ## %bb.5: ## %cond.load4
1093; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1094; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
1095; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
1096; SSE2-NEXT:  LBB9_6: ## %else5
1097; SSE2-NEXT:    pextrw $3, %xmm2, %eax
1098; SSE2-NEXT:    testb $1, %al
1099; SSE2-NEXT:    je LBB9_8
1100; SSE2-NEXT:  ## %bb.7: ## %cond.load7
1101; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1102; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[2,0]
1103; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
1104; SSE2-NEXT:  LBB9_8: ## %else8
1105; SSE2-NEXT:    pextrw $4, %xmm2, %eax
1106; SSE2-NEXT:    testb $1, %al
1107; SSE2-NEXT:    je LBB9_10
1108; SSE2-NEXT:  ## %bb.9: ## %cond.load10
1109; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1110; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
1111; SSE2-NEXT:  LBB9_10: ## %else11
1112; SSE2-NEXT:    pextrw $5, %xmm2, %eax
1113; SSE2-NEXT:    testb $1, %al
1114; SSE2-NEXT:    je LBB9_12
1115; SSE2-NEXT:  ## %bb.11: ## %cond.load13
1116; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1117; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
1118; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
1119; SSE2-NEXT:    movaps %xmm3, %xmm1
1120; SSE2-NEXT:  LBB9_12: ## %else14
1121; SSE2-NEXT:    pextrw $6, %xmm2, %eax
1122; SSE2-NEXT:    testb $1, %al
1123; SSE2-NEXT:    je LBB9_14
1124; SSE2-NEXT:  ## %bb.13: ## %cond.load16
1125; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1126; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
1127; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
1128; SSE2-NEXT:  LBB9_14: ## %else17
1129; SSE2-NEXT:    pextrw $7, %xmm2, %eax
1130; SSE2-NEXT:    testb $1, %al
1131; SSE2-NEXT:    je LBB9_16
1132; SSE2-NEXT:  ## %bb.15: ## %cond.load19
1133; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1134; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
1135; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
1136; SSE2-NEXT:  LBB9_16: ## %else20
1137; SSE2-NEXT:    retq
1138;
1139; SSE42-LABEL: load_zero_v8i32_v8i1:
1140; SSE42:       ## %bb.0:
1141; SSE42-NEXT:    movdqa %xmm0, %xmm2
1142; SSE42-NEXT:    pextrb $0, %xmm0, %eax
1143; SSE42-NEXT:    pxor %xmm0, %xmm0
1144; SSE42-NEXT:    testb $1, %al
1145; SSE42-NEXT:    pxor %xmm1, %xmm1
1146; SSE42-NEXT:    je LBB9_2
1147; SSE42-NEXT:  ## %bb.1: ## %cond.load
1148; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1149; SSE42-NEXT:    pxor %xmm1, %xmm1
1150; SSE42-NEXT:  LBB9_2: ## %else
1151; SSE42-NEXT:    pextrb $2, %xmm2, %eax
1152; SSE42-NEXT:    testb $1, %al
1153; SSE42-NEXT:    je LBB9_4
1154; SSE42-NEXT:  ## %bb.3: ## %cond.load1
1155; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm0
1156; SSE42-NEXT:  LBB9_4: ## %else2
1157; SSE42-NEXT:    pextrb $4, %xmm2, %eax
1158; SSE42-NEXT:    testb $1, %al
1159; SSE42-NEXT:    je LBB9_6
1160; SSE42-NEXT:  ## %bb.5: ## %cond.load4
1161; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm0
1162; SSE42-NEXT:  LBB9_6: ## %else5
1163; SSE42-NEXT:    pextrb $6, %xmm2, %eax
1164; SSE42-NEXT:    testb $1, %al
1165; SSE42-NEXT:    je LBB9_8
1166; SSE42-NEXT:  ## %bb.7: ## %cond.load7
1167; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm0
1168; SSE42-NEXT:  LBB9_8: ## %else8
1169; SSE42-NEXT:    pextrb $8, %xmm2, %eax
1170; SSE42-NEXT:    testb $1, %al
1171; SSE42-NEXT:    je LBB9_10
1172; SSE42-NEXT:  ## %bb.9: ## %cond.load10
1173; SSE42-NEXT:    pinsrd $0, 16(%rdi), %xmm1
1174; SSE42-NEXT:  LBB9_10: ## %else11
1175; SSE42-NEXT:    pextrb $10, %xmm2, %eax
1176; SSE42-NEXT:    testb $1, %al
1177; SSE42-NEXT:    je LBB9_12
1178; SSE42-NEXT:  ## %bb.11: ## %cond.load13
1179; SSE42-NEXT:    pinsrd $1, 20(%rdi), %xmm1
1180; SSE42-NEXT:  LBB9_12: ## %else14
1181; SSE42-NEXT:    pextrb $12, %xmm2, %eax
1182; SSE42-NEXT:    testb $1, %al
1183; SSE42-NEXT:    je LBB9_14
1184; SSE42-NEXT:  ## %bb.13: ## %cond.load16
1185; SSE42-NEXT:    pinsrd $2, 24(%rdi), %xmm1
1186; SSE42-NEXT:  LBB9_14: ## %else17
1187; SSE42-NEXT:    pextrb $14, %xmm2, %eax
1188; SSE42-NEXT:    testb $1, %al
1189; SSE42-NEXT:    je LBB9_16
1190; SSE42-NEXT:  ## %bb.15: ## %cond.load19
1191; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm1
1192; SSE42-NEXT:  LBB9_16: ## %else20
1193; SSE42-NEXT:    retq
1194;
1195; AVX1-LABEL: load_zero_v8i32_v8i1:
1196; AVX1:       ## %bb.0:
1197; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1198; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
1199; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
1200; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
1201; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
1202; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
1203; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
1204; AVX1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm0
1205; AVX1-NEXT:    retq
1206;
1207; AVX2-LABEL: load_zero_v8i32_v8i1:
1208; AVX2:       ## %bb.0:
1209; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1210; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
1211; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
1212; AVX2-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm0
1213; AVX2-NEXT:    retq
1214;
1215; AVX512F-LABEL: load_zero_v8i32_v8i1:
1216; AVX512F:       ## %bb.0:
1217; AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
1218; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
1219; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k1
1220; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} {z}
1221; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1222; AVX512F-NEXT:    retq
1223;
1224; AVX512VLBW-LABEL: load_zero_v8i32_v8i1:
1225; AVX512VLBW:       ## %bb.0:
1226; AVX512VLBW-NEXT:    vpsllw $15, %xmm0, %xmm0
1227; AVX512VLBW-NEXT:    vpmovw2m %xmm0, %k1
1228; AVX512VLBW-NEXT:    vmovdqu32 (%rdi), %ymm0 {%k1} {z}
1229; AVX512VLBW-NEXT:    retq
1230  %res = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* %addr, i32 4, <8 x i1> %mask, <8 x i32> zeroinitializer)
1231  ret <8 x i32> %res
1232}
1233
1234define <2 x float> @load_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %dst) {
1235; SSE2-LABEL: load_v2f32_v2i32:
1236; SSE2:       ## %bb.0:
1237; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1238; SSE2-NEXT:    pxor %xmm2, %xmm2
1239; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
1240; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
1241; SSE2-NEXT:    pand %xmm2, %xmm0
1242; SSE2-NEXT:    movd %xmm0, %eax
1243; SSE2-NEXT:    testb $1, %al
1244; SSE2-NEXT:    je LBB10_2
1245; SSE2-NEXT:  ## %bb.1: ## %cond.load
1246; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1247; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
1248; SSE2-NEXT:  LBB10_2: ## %else
1249; SSE2-NEXT:    pextrw $4, %xmm0, %eax
1250; SSE2-NEXT:    testb $1, %al
1251; SSE2-NEXT:    je LBB10_4
1252; SSE2-NEXT:  ## %bb.3: ## %cond.load1
1253; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1254; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
1255; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
1256; SSE2-NEXT:    movaps %xmm0, %xmm1
1257; SSE2-NEXT:  LBB10_4: ## %else2
1258; SSE2-NEXT:    movaps %xmm1, %xmm0
1259; SSE2-NEXT:    retq
1260;
1261; SSE42-LABEL: load_v2f32_v2i32:
1262; SSE42:       ## %bb.0:
1263; SSE42-NEXT:    pxor %xmm2, %xmm2
1264; SSE42-NEXT:    movdqa %xmm0, %xmm3
1265; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1266; SSE42-NEXT:    pcmpeqq %xmm2, %xmm3
1267; SSE42-NEXT:    pextrb $0, %xmm3, %eax
1268; SSE42-NEXT:    testb $1, %al
1269; SSE42-NEXT:    je LBB10_2
1270; SSE42-NEXT:  ## %bb.1: ## %cond.load
1271; SSE42-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
1272; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3,4,5,6,7]
1273; SSE42-NEXT:  LBB10_2: ## %else
1274; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
1275; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
1276; SSE42-NEXT:    pextrb $8, %xmm0, %eax
1277; SSE42-NEXT:    testb $1, %al
1278; SSE42-NEXT:    je LBB10_4
1279; SSE42-NEXT:  ## %bb.3: ## %cond.load1
1280; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
1281; SSE42-NEXT:  LBB10_4: ## %else2
1282; SSE42-NEXT:    movaps %xmm1, %xmm0
1283; SSE42-NEXT:    retq
1284;
1285; AVX1-LABEL: load_v2f32_v2i32:
1286; AVX1:       ## %bb.0:
1287; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1288; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
1289; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
1290; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1291; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
1292; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
1293; AVX1-NEXT:    retq
1294;
1295; AVX2-LABEL: load_v2f32_v2i32:
1296; AVX2:       ## %bb.0:
1297; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1298; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1299; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
1300; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1301; AVX2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
1302; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
1303; AVX2-NEXT:    retq
1304;
1305; AVX512F-LABEL: load_v2f32_v2i32:
1306; AVX512F:       ## %bb.0:
1307; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
1308; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1309; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1310; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
1311; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
1312; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
1313; AVX512F-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
1314; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
1315; AVX512F-NEXT:    vzeroupper
1316; AVX512F-NEXT:    retq
1317;
1318; AVX512VLBW-LABEL: load_v2f32_v2i32:
1319; AVX512VLBW:       ## %bb.0:
1320; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1321; AVX512VLBW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1322; AVX512VLBW-NEXT:    vptestnmq %xmm0, %xmm0, %k1
1323; AVX512VLBW-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
1324; AVX512VLBW-NEXT:    retq
1325  %mask = icmp eq <2 x i32> %trigger, zeroinitializer
1326  %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>%dst)
1327  ret <2 x float> %res
1328}
1329
1330define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
1331; SSE2-LABEL: load_v2i32_v2i32:
1332; SSE2:       ## %bb.0:
1333; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1334; SSE2-NEXT:    pxor %xmm2, %xmm2
1335; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
1336; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
1337; SSE2-NEXT:    pand %xmm2, %xmm0
1338; SSE2-NEXT:    movd %xmm0, %eax
1339; SSE2-NEXT:    testb $1, %al
1340; SSE2-NEXT:    je LBB11_2
1341; SSE2-NEXT:  ## %bb.1: ## %cond.load
1342; SSE2-NEXT:    movl (%rdi), %eax
1343; SSE2-NEXT:    movq %rax, %xmm2
1344; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
1345; SSE2-NEXT:  LBB11_2: ## %else
1346; SSE2-NEXT:    pextrw $4, %xmm0, %eax
1347; SSE2-NEXT:    testb $1, %al
1348; SSE2-NEXT:    je LBB11_4
1349; SSE2-NEXT:  ## %bb.3: ## %cond.load1
1350; SSE2-NEXT:    movl 4(%rdi), %eax
1351; SSE2-NEXT:    movq %rax, %xmm0
1352; SSE2-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1353; SSE2-NEXT:  LBB11_4: ## %else2
1354; SSE2-NEXT:    movapd %xmm1, %xmm0
1355; SSE2-NEXT:    retq
1356;
1357; SSE42-LABEL: load_v2i32_v2i32:
1358; SSE42:       ## %bb.0:
1359; SSE42-NEXT:    pxor %xmm2, %xmm2
1360; SSE42-NEXT:    movdqa %xmm0, %xmm3
1361; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
1362; SSE42-NEXT:    pcmpeqq %xmm2, %xmm3
1363; SSE42-NEXT:    pextrb $0, %xmm3, %eax
1364; SSE42-NEXT:    testb $1, %al
1365; SSE42-NEXT:    je LBB11_2
1366; SSE42-NEXT:  ## %bb.1: ## %cond.load
1367; SSE42-NEXT:    movl (%rdi), %eax
1368; SSE42-NEXT:    pinsrq $0, %rax, %xmm1
1369; SSE42-NEXT:  LBB11_2: ## %else
1370; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
1371; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
1372; SSE42-NEXT:    pextrb $8, %xmm0, %eax
1373; SSE42-NEXT:    testb $1, %al
1374; SSE42-NEXT:    je LBB11_4
1375; SSE42-NEXT:  ## %bb.3: ## %cond.load1
1376; SSE42-NEXT:    movl 4(%rdi), %eax
1377; SSE42-NEXT:    pinsrq $1, %rax, %xmm1
1378; SSE42-NEXT:  LBB11_4: ## %else2
1379; SSE42-NEXT:    movdqa %xmm1, %xmm0
1380; SSE42-NEXT:    retq
1381;
1382; AVX1-LABEL: load_v2i32_v2i32:
1383; AVX1:       ## %bb.0:
1384; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1385; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
1386; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
1387; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1388; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
1389; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
1390; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
1391; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
1392; AVX1-NEXT:    retq
1393;
1394; AVX2-LABEL: load_v2i32_v2i32:
1395; AVX2:       ## %bb.0:
1396; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1397; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1398; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
1399; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1400; AVX2-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2
1401; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
1402; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
1403; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
1404; AVX2-NEXT:    retq
1405;
1406; AVX512F-LABEL: load_v2i32_v2i32:
1407; AVX512F:       ## %bb.0:
1408; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1409; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1410; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
1411; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1412; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
1413; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
1414; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1}
1415; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
1416; AVX512F-NEXT:    vzeroupper
1417; AVX512F-NEXT:    retq
1418;
1419; AVX512VLBW-LABEL: load_v2i32_v2i32:
1420; AVX512VLBW:       ## %bb.0:
1421; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
1422; AVX512VLBW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
1423; AVX512VLBW-NEXT:    vptestnmq %xmm0, %xmm0, %k1
1424; AVX512VLBW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1425; AVX512VLBW-NEXT:    vmovdqu32 (%rdi), %xmm0 {%k1}
1426; AVX512VLBW-NEXT:    vpmovsxdq %xmm0, %xmm0
1427; AVX512VLBW-NEXT:    retq
1428  %mask = icmp eq <2 x i32> %trigger, zeroinitializer
1429  %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1>%mask, <2 x i32>%dst)
1430  ret <2 x i32> %res
1431}
1432
1433define <2 x float> @load_undef_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr) {
1434; SSE2-LABEL: load_undef_v2f32_v2i32:
1435; SSE2:       ## %bb.0:
1436; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
1437; SSE2-NEXT:    pxor %xmm2, %xmm2
1438; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
1439; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
1440; SSE2-NEXT:    pand %xmm2, %xmm1
1441; SSE2-NEXT:    movd %xmm1, %eax
1442; SSE2-NEXT:    testb $1, %al
1443; SSE2-NEXT:    ## implicit-def: $xmm0
1444; SSE2-NEXT:    je LBB12_2
1445; SSE2-NEXT:  ## %bb.1: ## %cond.load
1446; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1447; SSE2-NEXT:  LBB12_2: ## %else
1448; SSE2-NEXT:    pextrw $4, %xmm1, %eax
1449; SSE2-NEXT:    testb $1, %al
1450; SSE2-NEXT:    je LBB12_4
1451; SSE2-NEXT:  ## %bb.3: ## %cond.load1
1452; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1453; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
1454; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
1455; SSE2-NEXT:    movaps %xmm1, %xmm0
1456; SSE2-NEXT:  LBB12_4: ## %else2
1457; SSE2-NEXT:    retq
1458;
1459; SSE42-LABEL: load_undef_v2f32_v2i32:
1460; SSE42:       ## %bb.0:
1461; SSE42-NEXT:    movdqa %xmm0, %xmm1
1462; SSE42-NEXT:    pxor %xmm2, %xmm2
1463; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
1464; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
1465; SSE42-NEXT:    pextrb $0, %xmm0, %eax
1466; SSE42-NEXT:    testb $1, %al
1467; SSE42-NEXT:    ## implicit-def: $xmm0
1468; SSE42-NEXT:    je LBB12_2
1469; SSE42-NEXT:  ## %bb.1: ## %cond.load
1470; SSE42-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1471; SSE42-NEXT:  LBB12_2: ## %else
1472; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
1473; SSE42-NEXT:    pcmpeqq %xmm2, %xmm1
1474; SSE42-NEXT:    pextrb $8, %xmm1, %eax
1475; SSE42-NEXT:    testb $1, %al
1476; SSE42-NEXT:    je LBB12_4
1477; SSE42-NEXT:  ## %bb.3: ## %cond.load1
1478; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
1479; SSE42-NEXT:  LBB12_4: ## %else2
1480; SSE42-NEXT:    retq
1481;
1482; AVX1-LABEL: load_undef_v2f32_v2i32:
1483; AVX1:       ## %bb.0:
1484; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1485; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
1486; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
1487; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1488; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm0
1489; AVX1-NEXT:    retq
1490;
1491; AVX2-LABEL: load_undef_v2f32_v2i32:
1492; AVX2:       ## %bb.0:
1493; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1494; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
1495; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
1496; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
1497; AVX2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm0
1498; AVX2-NEXT:    retq
1499;
1500; AVX512F-LABEL: load_undef_v2f32_v2i32:
1501; AVX512F:       ## %bb.0:
1502; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1503; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
1504; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
1505; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
1506; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
1507; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z}
1508; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
1509; AVX512F-NEXT:    vzeroupper
1510; AVX512F-NEXT:    retq
1511;
1512; AVX512VLBW-LABEL: load_undef_v2f32_v2i32:
1513; AVX512VLBW:       ## %bb.0:
1514; AVX512VLBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
1515; AVX512VLBW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
1516; AVX512VLBW-NEXT:    vptestnmq %xmm0, %xmm0, %k1
1517; AVX512VLBW-NEXT:    vmovups (%rdi), %xmm0 {%k1} {z}
1518; AVX512VLBW-NEXT:    retq
1519  %mask = icmp eq <2 x i32> %trigger, zeroinitializer
1520  %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>undef)
1521  ret <2 x float> %res
1522}
1523
1524define <4 x float> @load_all_v4f32_v4i32(<4 x i32> %trigger, <4 x float>* %addr) {
1525; SSE-LABEL: load_all_v4f32_v4i32:
1526; SSE:       ## %bb.0:
1527; SSE-NEXT:    movups (%rdi), %xmm0
1528; SSE-NEXT:    retq
1529;
1530; AVX1OR2-LABEL: load_all_v4f32_v4i32:
1531; AVX1OR2:       ## %bb.0:
1532; AVX1OR2-NEXT:    vmovups (%rdi), %xmm0
1533; AVX1OR2-NEXT:    retq
1534;
1535; AVX512F-LABEL: load_all_v4f32_v4i32:
1536; AVX512F:       ## %bb.0:
1537; AVX512F-NEXT:    movw $15, %ax
1538; AVX512F-NEXT:    kmovw %eax, %k1
1539; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z}
1540; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
1541; AVX512F-NEXT:    vzeroupper
1542; AVX512F-NEXT:    retq
1543;
1544; AVX512VLBW-LABEL: load_all_v4f32_v4i32:
1545; AVX512VLBW:       ## %bb.0:
1546; AVX512VLBW-NEXT:    kxnorw %k0, %k0, %k1
1547; AVX512VLBW-NEXT:    vmovups (%rdi), %xmm0 {%k1} {z}
1548; AVX512VLBW-NEXT:    retq
1549  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
1550  %res = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %addr, i32 4, <4 x i1><i1 true, i1 true, i1 true, i1 true>, <4 x float>undef)
1551  ret <4 x float> %res
1552}
1553
1554;;; Loads with Constant Masks - these should be optimized to use something other than a variable blend.
1555
1556; 128-bit FP vectors are supported with AVX.
1557
1558define <4 x float> @mload_constmask_v4f32(<4 x float>* %addr, <4 x float> %dst) {
1559; SSE2-LABEL: mload_constmask_v4f32:
1560; SSE2:       ## %bb.0:
1561; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1562; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1563; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
1564; SSE2-NEXT:    movaps %xmm0, %xmm1
1565; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
1566; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1567; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
1568; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
1569; SSE2-NEXT:    retq
1570;
1571; SSE42-LABEL: mload_constmask_v4f32:
1572; SSE42:       ## %bb.0:
1573; SSE42-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1574; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
1575; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
1576; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
1577; SSE42-NEXT:    retq
1578;
1579; AVX1OR2-LABEL: mload_constmask_v4f32:
1580; AVX1OR2:       ## %bb.0:
1581; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = mem[0],xmm0[1],mem[2,3]
1582; AVX1OR2-NEXT:    retq
1583;
1584; AVX512F-LABEL: mload_constmask_v4f32:
1585; AVX512F:       ## %bb.0:
1586; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
1587; AVX512F-NEXT:    movw $13, %ax
1588; AVX512F-NEXT:    kmovw %eax, %k1
1589; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1}
1590; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
1591; AVX512F-NEXT:    vzeroupper
1592; AVX512F-NEXT:    retq
1593;
1594; AVX512VLBW-LABEL: mload_constmask_v4f32:
1595; AVX512VLBW:       ## %bb.0:
1596; AVX512VLBW-NEXT:    movb $13, %al
1597; AVX512VLBW-NEXT:    kmovd %eax, %k1
1598; AVX512VLBW-NEXT:    vmovups (%rdi), %xmm0 {%k1}
1599; AVX512VLBW-NEXT:    retq
1600  %res = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %addr, i32 4, <4 x i1> <i1 1, i1 0, i1 1, i1 1>, <4 x float> %dst)
1601  ret <4 x float> %res
1602}
1603
1604define <2 x double> @mload_constmask_v2f64(<2 x double>* %addr, <2 x double> %dst) {
1605; SSE-LABEL: mload_constmask_v2f64:
1606; SSE:       ## %bb.0:
1607; SSE-NEXT:    movhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
1608; SSE-NEXT:    retq
1609;
1610; AVX-LABEL: mload_constmask_v2f64:
1611; AVX:       ## %bb.0:
1612; AVX-NEXT:    vmovhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
1613; AVX-NEXT:    retq
1614  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> <i1 0, i1 1>, <2 x double> %dst)
1615  ret <2 x double> %res
1616}
1617
1618; 128-bit integer vectors are supported with AVX2.
1619
1620define <4 x i32> @mload_constmask_v4i32(<4 x i32>* %addr, <4 x i32> %dst) {
1621; SSE2-LABEL: mload_constmask_v4i32:
1622; SSE2:       ## %bb.0:
1623; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1624; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
1625; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1626; SSE2-NEXT:    movaps %xmm1, %xmm2
1627; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[0,2]
1628; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1629; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
1630; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,0]
1631; SSE2-NEXT:    movaps %xmm1, %xmm0
1632; SSE2-NEXT:    retq
1633;
1634; SSE42-LABEL: mload_constmask_v4i32:
1635; SSE42:       ## %bb.0:
1636; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm0
1637; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm0
1638; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm0
1639; SSE42-NEXT:    retq
1640;
1641; AVX1-LABEL: mload_constmask_v4i32:
1642; AVX1:       ## %bb.0:
1643; AVX1-NEXT:    vmovaps {{.*#+}} xmm1 = [0,4294967295,4294967295,4294967295]
1644; AVX1-NEXT:    vmaskmovps (%rdi), %xmm1, %xmm1
1645; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
1646; AVX1-NEXT:    retq
1647;
1648; AVX2-LABEL: mload_constmask_v4i32:
1649; AVX2:       ## %bb.0:
1650; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,4294967295,4294967295,4294967295]
1651; AVX2-NEXT:    vpmaskmovd (%rdi), %xmm1, %xmm1
1652; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
1653; AVX2-NEXT:    retq
1654;
1655; AVX512F-LABEL: mload_constmask_v4i32:
1656; AVX512F:       ## %bb.0:
1657; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
1658; AVX512F-NEXT:    movw $14, %ax
1659; AVX512F-NEXT:    kmovw %eax, %k1
1660; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1}
1661; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
1662; AVX512F-NEXT:    vzeroupper
1663; AVX512F-NEXT:    retq
1664;
1665; AVX512VLBW-LABEL: mload_constmask_v4i32:
1666; AVX512VLBW:       ## %bb.0:
1667; AVX512VLBW-NEXT:    movb $14, %al
1668; AVX512VLBW-NEXT:    kmovd %eax, %k1
1669; AVX512VLBW-NEXT:    vmovdqu32 (%rdi), %xmm0 {%k1}
1670; AVX512VLBW-NEXT:    retq
1671  %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> <i1 0, i1 1, i1 1, i1 1>, <4 x i32> %dst)
1672  ret <4 x i32> %res
1673}
1674
1675define <2 x i64> @mload_constmask_v2i64(<2 x i64>* %addr, <2 x i64> %dst) {
1676; SSE2-LABEL: mload_constmask_v2i64:
1677; SSE2:       ## %bb.0:
1678; SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
1679; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1680; SSE2-NEXT:    retq
1681;
1682; SSE42-LABEL: mload_constmask_v2i64:
1683; SSE42:       ## %bb.0:
1684; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm0
1685; SSE42-NEXT:    retq
1686;
1687; AVX-LABEL: mload_constmask_v2i64:
1688; AVX:       ## %bb.0:
1689; AVX-NEXT:    vpinsrq $1, 8(%rdi), %xmm0, %xmm0
1690; AVX-NEXT:    retq
1691  %res = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %addr, i32 4, <2 x i1> <i1 0, i1 1>, <2 x i64> %dst)
1692  ret <2 x i64> %res
1693}
1694
1695; 256-bit FP vectors are supported with AVX.
1696
1697define <8 x float> @mload_constmask_v8f32(<8 x float>* %addr, <8 x float> %dst) {
1698; SSE2-LABEL: mload_constmask_v8f32:
1699; SSE2:       ## %bb.0:
1700; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1701; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1702; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[0,0]
1703; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1704; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
1705; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[0,2]
1706; SSE2-NEXT:    movaps %xmm2, %xmm0
1707; SSE2-NEXT:    retq
1708;
1709; SSE42-LABEL: mload_constmask_v8f32:
1710; SSE42:       ## %bb.0:
1711; SSE42-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1712; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
1713; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
1714; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
1715; SSE42-NEXT:    retq
1716;
1717; AVX1OR2-LABEL: mload_constmask_v8f32:
1718; AVX1OR2:       ## %bb.0:
1719; AVX1OR2-NEXT:    vmovaps {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,0,0,0,0,0]
1720; AVX1OR2-NEXT:    vmaskmovps (%rdi), %ymm1, %ymm1
1721; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
1722; AVX1OR2-NEXT:    retq
1723;
1724; AVX512F-LABEL: mload_constmask_v8f32:
1725; AVX512F:       ## %bb.0:
1726; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
1727; AVX512F-NEXT:    movw $7, %ax
1728; AVX512F-NEXT:    kmovw %eax, %k1
1729; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1}
1730; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1731; AVX512F-NEXT:    retq
1732;
1733; AVX512VLBW-LABEL: mload_constmask_v8f32:
1734; AVX512VLBW:       ## %bb.0:
1735; AVX512VLBW-NEXT:    movb $7, %al
1736; AVX512VLBW-NEXT:    kmovd %eax, %k1
1737; AVX512VLBW-NEXT:    vmovups (%rdi), %ymm0 {%k1}
1738; AVX512VLBW-NEXT:    retq
1739  %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> <i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0>, <8 x float> %dst)
1740  ret <8 x float> %res
1741}
1742
1743define <4 x double> @mload_constmask_v4f64(<4 x double>* %addr, <4 x double> %dst) {
1744; SSE-LABEL: mload_constmask_v4f64:
1745; SSE:       ## %bb.0:
1746; SSE-NEXT:    movups (%rdi), %xmm0
1747; SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
1748; SSE-NEXT:    retq
1749;
1750; AVX1OR2-LABEL: mload_constmask_v4f64:
1751; AVX1OR2:       ## %bb.0:
1752; AVX1OR2-NEXT:    vmovapd {{.*#+}} ymm1 = [18446744073709551615,18446744073709551615,18446744073709551615,0]
1753; AVX1OR2-NEXT:    vmaskmovpd (%rdi), %ymm1, %ymm1
1754; AVX1OR2-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3]
1755; AVX1OR2-NEXT:    retq
1756;
1757; AVX512F-LABEL: mload_constmask_v4f64:
1758; AVX512F:       ## %bb.0:
1759; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
1760; AVX512F-NEXT:    movb $7, %al
1761; AVX512F-NEXT:    kmovw %eax, %k1
1762; AVX512F-NEXT:    vmovupd (%rdi), %zmm0 {%k1}
1763; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1764; AVX512F-NEXT:    retq
1765;
1766; AVX512VLBW-LABEL: mload_constmask_v4f64:
1767; AVX512VLBW:       ## %bb.0:
1768; AVX512VLBW-NEXT:    movb $7, %al
1769; AVX512VLBW-NEXT:    kmovd %eax, %k1
1770; AVX512VLBW-NEXT:    vmovupd (%rdi), %ymm0 {%k1}
1771; AVX512VLBW-NEXT:    retq
1772  %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 4, <4 x i1> <i1 1, i1 1, i1 1, i1 0>, <4 x double> %dst)
1773  ret <4 x double> %res
1774}
1775
1776; 256-bit integer vectors are supported with AVX2.
1777
1778define <8 x i32> @mload_constmask_v8i32(<8 x i32>* %addr, <8 x i32> %dst) {
1779; SSE2-LABEL: mload_constmask_v8i32:
1780; SSE2:       ## %bb.0:
1781; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1782; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
1783; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[0,0]
1784; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
1785; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
1786; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[0,2]
1787; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
1788; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
1789; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
1790; SSE2-NEXT:    movaps %xmm2, %xmm0
1791; SSE2-NEXT:    retq
1792;
1793; SSE42-LABEL: mload_constmask_v8i32:
1794; SSE42:       ## %bb.0:
1795; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm0
1796; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm0
1797; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm0
1798; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm1
1799; SSE42-NEXT:    retq
1800;
1801; AVX1OR2-LABEL: mload_constmask_v8i32:
1802; AVX1OR2:       ## %bb.0:
1803; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2],ymm0[3,4,5,6],mem[7]
1804; AVX1OR2-NEXT:    retq
1805;
1806; AVX512F-LABEL: mload_constmask_v8i32:
1807; AVX512F:       ## %bb.0:
1808; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
1809; AVX512F-NEXT:    movw $135, %ax
1810; AVX512F-NEXT:    kmovw %eax, %k1
1811; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1}
1812; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1813; AVX512F-NEXT:    retq
1814;
1815; AVX512VLBW-LABEL: mload_constmask_v8i32:
1816; AVX512VLBW:       ## %bb.0:
1817; AVX512VLBW-NEXT:    movb $-121, %al
1818; AVX512VLBW-NEXT:    kmovd %eax, %k1
1819; AVX512VLBW-NEXT:    vmovdqu32 (%rdi), %ymm0 {%k1}
1820; AVX512VLBW-NEXT:    retq
1821  %res = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* %addr, i32 4, <8 x i1> <i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1>, <8 x i32> %dst)
1822  ret <8 x i32> %res
1823}
1824
1825define <4 x i64> @mload_constmask_v4i64(<4 x i64>* %addr, <4 x i64> %dst) {
1826; SSE2-LABEL: mload_constmask_v4i64:
1827; SSE2:       ## %bb.0:
1828; SSE2-NEXT:    movlpd {{.*#+}} xmm0 = mem[0],xmm0[1]
1829; SSE2-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
1830; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1831; SSE2-NEXT:    retq
1832;
1833; SSE42-LABEL: mload_constmask_v4i64:
1834; SSE42:       ## %bb.0:
1835; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm0
1836; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm1
1837; SSE42-NEXT:    retq
1838;
1839; AVX1OR2-LABEL: mload_constmask_v4i64:
1840; AVX1OR2:       ## %bb.0:
1841; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2,3,4,5],mem[6,7]
1842; AVX1OR2-NEXT:    retq
1843;
1844; AVX512F-LABEL: mload_constmask_v4i64:
1845; AVX512F:       ## %bb.0:
1846; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
1847; AVX512F-NEXT:    movb $9, %al
1848; AVX512F-NEXT:    kmovw %eax, %k1
1849; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1}
1850; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1851; AVX512F-NEXT:    retq
1852;
1853; AVX512VLBW-LABEL: mload_constmask_v4i64:
1854; AVX512VLBW:       ## %bb.0:
1855; AVX512VLBW-NEXT:    movb $9, %al
1856; AVX512VLBW-NEXT:    kmovd %eax, %k1
1857; AVX512VLBW-NEXT:    vmovdqu64 (%rdi), %ymm0 {%k1}
1858; AVX512VLBW-NEXT:    retq
1859  %res = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* %addr, i32 4, <4 x i1> <i1 1, i1 0, i1 0, i1 1>, <4 x i64> %dst)
1860  ret <4 x i64> %res
1861}
1862
1863; 512-bit FP vectors are supported with AVX512.
1864
1865define <8 x double> @mload_constmask_v8f64(<8 x double>* %addr, <8 x double> %dst) {
1866; SSE-LABEL: mload_constmask_v8f64:
1867; SSE:       ## %bb.0:
1868; SSE-NEXT:    movups (%rdi), %xmm0
1869; SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
1870; SSE-NEXT:    movhpd {{.*#+}} xmm3 = xmm3[0],mem[0]
1871; SSE-NEXT:    retq
1872;
1873; AVX1OR2-LABEL: mload_constmask_v8f64:
1874; AVX1OR2:       ## %bb.0:
1875; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6,7]
1876; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
1877; AVX1OR2-NEXT:    retq
1878;
1879; AVX512F-LABEL: mload_constmask_v8f64:
1880; AVX512F:       ## %bb.0:
1881; AVX512F-NEXT:    movb $-121, %al
1882; AVX512F-NEXT:    kmovw %eax, %k1
1883; AVX512F-NEXT:    vmovupd (%rdi), %zmm0 {%k1}
1884; AVX512F-NEXT:    retq
1885;
1886; AVX512VLBW-LABEL: mload_constmask_v8f64:
1887; AVX512VLBW:       ## %bb.0:
1888; AVX512VLBW-NEXT:    movb $-121, %al
1889; AVX512VLBW-NEXT:    kmovd %eax, %k1
1890; AVX512VLBW-NEXT:    vmovupd (%rdi), %zmm0 {%k1}
1891; AVX512VLBW-NEXT:    retq
1892  %res = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* %addr, i32 4, <8 x i1> <i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1>, <8 x double> %dst)
1893  ret <8 x double> %res
1894}
1895
1896; If the pass-through operand is undef, no blend is needed.
1897
1898define <4 x double> @mload_constmask_v4f64_undef_passthrough(<4 x double>* %addr) {
1899; SSE-LABEL: mload_constmask_v4f64_undef_passthrough:
1900; SSE:       ## %bb.0:
1901; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
1902; SSE-NEXT:    movups (%rdi), %xmm0
1903; SSE-NEXT:    retq
1904;
1905; AVX1OR2-LABEL: mload_constmask_v4f64_undef_passthrough:
1906; AVX1OR2:       ## %bb.0:
1907; AVX1OR2-NEXT:    vmovapd {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,18446744073709551615,0]
1908; AVX1OR2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm0
1909; AVX1OR2-NEXT:    retq
1910;
1911; AVX512F-LABEL: mload_constmask_v4f64_undef_passthrough:
1912; AVX512F:       ## %bb.0:
1913; AVX512F-NEXT:    movb $7, %al
1914; AVX512F-NEXT:    kmovw %eax, %k1
1915; AVX512F-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z}
1916; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1917; AVX512F-NEXT:    retq
1918;
1919; AVX512VLBW-LABEL: mload_constmask_v4f64_undef_passthrough:
1920; AVX512VLBW:       ## %bb.0:
1921; AVX512VLBW-NEXT:    movb $7, %al
1922; AVX512VLBW-NEXT:    kmovd %eax, %k1
1923; AVX512VLBW-NEXT:    vmovupd (%rdi), %ymm0 {%k1} {z}
1924; AVX512VLBW-NEXT:    retq
1925  %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 4, <4 x i1> <i1 1, i1 1, i1 1, i1 0>, <4 x double> undef)
1926  ret <4 x double> %res
1927}
1928
1929define <4 x i64> @mload_constmask_v4i64_undef_passthrough(<4 x i64>* %addr) {
1930; SSE-LABEL: mload_constmask_v4i64_undef_passthrough:
1931; SSE:       ## %bb.0:
1932; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
1933; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
1934; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
1935; SSE-NEXT:    retq
1936;
1937; AVX1-LABEL: mload_constmask_v4i64_undef_passthrough:
1938; AVX1:       ## %bb.0:
1939; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [0,18446744073709551615,18446744073709551615,0]
1940; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm0
1941; AVX1-NEXT:    retq
1942;
1943; AVX2-LABEL: mload_constmask_v4i64_undef_passthrough:
1944; AVX2:       ## %bb.0:
1945; AVX2-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,18446744073709551615,18446744073709551615,0]
1946; AVX2-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm0
1947; AVX2-NEXT:    retq
1948;
1949; AVX512F-LABEL: mload_constmask_v4i64_undef_passthrough:
1950; AVX512F:       ## %bb.0:
1951; AVX512F-NEXT:    movb $6, %al
1952; AVX512F-NEXT:    kmovw %eax, %k1
1953; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} {z}
1954; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
1955; AVX512F-NEXT:    retq
1956;
1957; AVX512VLBW-LABEL: mload_constmask_v4i64_undef_passthrough:
1958; AVX512VLBW:       ## %bb.0:
1959; AVX512VLBW-NEXT:    movb $6, %al
1960; AVX512VLBW-NEXT:    kmovd %eax, %k1
1961; AVX512VLBW-NEXT:    vmovdqu64 (%rdi), %ymm0 {%k1} {z}
1962; AVX512VLBW-NEXT:    retq
1963  %res = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* %addr, i32 4, <4 x i1> <i1 0, i1 1, i1 1, i1 0>, <4 x i64> undef)
1964  ret <4 x i64> %res
1965}
1966
1967;  When only one element of the mask is set, reduce to a scalar load.
1968
1969define <4 x i32> @load_one_mask_bit_set1(<4 x i32>* %addr, <4 x i32> %val) {
1970; SSE2-LABEL: load_one_mask_bit_set1:
1971; SSE2:       ## %bb.0:
1972; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1973; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
1974; SSE2-NEXT:    retq
1975;
1976; SSE42-LABEL: load_one_mask_bit_set1:
1977; SSE42:       ## %bb.0:
1978; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm0
1979; SSE42-NEXT:    retq
1980;
1981; AVX-LABEL: load_one_mask_bit_set1:
1982; AVX:       ## %bb.0:
1983; AVX-NEXT:    vpinsrd $0, (%rdi), %xmm0, %xmm0
1984; AVX-NEXT:    retq
1985  %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1><i1 true, i1 false, i1 false, i1 false>, <4 x i32> %val)
1986  ret <4 x i32> %res
1987}
1988
1989; Choose a different element to show that the correct address offset is produced.
1990
1991define <4 x float> @load_one_mask_bit_set2(<4 x float>* %addr, <4 x float> %val) {
1992; SSE2-LABEL: load_one_mask_bit_set2:
1993; SSE2:       ## %bb.0:
1994; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
1995; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[3,0]
1996; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
1997; SSE2-NEXT:    retq
1998;
1999; SSE42-LABEL: load_one_mask_bit_set2:
2000; SSE42:       ## %bb.0:
2001; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
2002; SSE42-NEXT:    retq
2003;
2004; AVX-LABEL: load_one_mask_bit_set2:
2005; AVX:       ## %bb.0:
2006; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
2007; AVX-NEXT:    retq
2008  %res = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %addr, i32 4, <4 x i1><i1 false, i1 false, i1 true, i1 false>, <4 x float> %val)
2009  ret <4 x float> %res
2010}
2011
2012; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.
2013
2014define <4 x i64> @load_one_mask_bit_set3(<4 x i64>* %addr, <4 x i64> %val) {
2015; SSE2-LABEL: load_one_mask_bit_set3:
2016; SSE2:       ## %bb.0:
2017; SSE2-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
2018; SSE2-NEXT:    retq
2019;
2020; SSE42-LABEL: load_one_mask_bit_set3:
2021; SSE42:       ## %bb.0:
2022; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm1
2023; SSE42-NEXT:    retq
2024;
2025; AVX1-LABEL: load_one_mask_bit_set3:
2026; AVX1:       ## %bb.0:
2027; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
2028; AVX1-NEXT:    vpinsrq $0, 16(%rdi), %xmm1, %xmm1
2029; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
2030; AVX1-NEXT:    retq
2031;
2032; AVX2-LABEL: load_one_mask_bit_set3:
2033; AVX2:       ## %bb.0:
2034; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
2035; AVX2-NEXT:    vpinsrq $0, 16(%rdi), %xmm1, %xmm1
2036; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
2037; AVX2-NEXT:    retq
2038;
2039; AVX512-LABEL: load_one_mask_bit_set3:
2040; AVX512:       ## %bb.0:
2041; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
2042; AVX512-NEXT:    vpinsrq $0, 16(%rdi), %xmm1, %xmm1
2043; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
2044; AVX512-NEXT:    retq
2045  %res = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* %addr, i32 4, <4 x i1><i1 false, i1 false, i1 true, i1 false>, <4 x i64> %val)
2046  ret <4 x i64> %res
2047}
2048
2049; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.
2050
2051define <4 x double> @load_one_mask_bit_set4(<4 x double>* %addr, <4 x double> %val) {
2052; SSE-LABEL: load_one_mask_bit_set4:
2053; SSE:       ## %bb.0:
2054; SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
2055; SSE-NEXT:    retq
2056;
2057; AVX-LABEL: load_one_mask_bit_set4:
2058; AVX:       ## %bb.0:
2059; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
2060; AVX-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
2061; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
2062; AVX-NEXT:    retq
2063  %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 4, <4 x i1><i1 false, i1 false, i1 false, i1 true>, <4 x double> %val)
2064  ret <4 x double> %res
2065}
2066
2067; Try a 512-bit vector to make sure AVX doesn't die and AVX512 works as expected.
2068
2069define <8 x double> @load_one_mask_bit_set5(<8 x double>* %addr, <8 x double> %val) {
2070; SSE-LABEL: load_one_mask_bit_set5:
2071; SSE:       ## %bb.0:
2072; SSE-NEXT:    movhpd {{.*#+}} xmm3 = xmm3[0],mem[0]
2073; SSE-NEXT:    retq
2074;
2075; AVX1OR2-LABEL: load_one_mask_bit_set5:
2076; AVX1OR2:       ## %bb.0:
2077; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm2
2078; AVX1OR2-NEXT:    vmovhpd {{.*#+}} xmm2 = xmm2[0],mem[0]
2079; AVX1OR2-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
2080; AVX1OR2-NEXT:    retq
2081;
2082; AVX512-LABEL: load_one_mask_bit_set5:
2083; AVX512:       ## %bb.0:
2084; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
2085; AVX512-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
2086; AVX512-NEXT:    vinsertf32x4 $3, %xmm1, %zmm0, %zmm0
2087; AVX512-NEXT:    retq
2088  %res = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x double> %val)
2089  ret <8 x double> %res
2090}
2091
2092define i32 @pr38986(i1 %c, i32* %p) {
2093; SSE-LABEL: pr38986:
2094; SSE:       ## %bb.0:
2095; SSE-NEXT:    testb $1, %dil
2096; SSE-NEXT:    ## implicit-def: $eax
2097; SSE-NEXT:    je LBB30_2
2098; SSE-NEXT:  ## %bb.1: ## %cond.load
2099; SSE-NEXT:    movl (%rsi), %eax
2100; SSE-NEXT:  LBB30_2: ## %else
2101; SSE-NEXT:    retq
2102;
2103; AVX-LABEL: pr38986:
2104; AVX:       ## %bb.0:
2105; AVX-NEXT:    testb $1, %dil
2106; AVX-NEXT:    ## implicit-def: $eax
2107; AVX-NEXT:    je LBB30_2
2108; AVX-NEXT:  ## %bb.1: ## %cond.load
2109; AVX-NEXT:    movl (%rsi), %eax
2110; AVX-NEXT:  LBB30_2: ## %else
2111; AVX-NEXT:    retq
2112 %vc = insertelement <1 x i1> undef, i1 %c, i32 0
2113 %vp = bitcast i32* %p to <1 x i32>*
2114 %L = call <1 x i32> @llvm.masked.load.v1i32.p0v1i32 (<1 x i32>* %vp, i32 4, <1 x i1> %vc, <1 x i32> undef)
2115 %ret = bitcast <1 x i32> %L to i32
2116 ret i32 %ret
2117}
2118declare <1 x i32>  @llvm.masked.load.v1i32.p0v1i32 (<1 x i32>*, i32, <1 x i1>, <1 x i32>)
2119
2120
2121declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
2122declare <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>*, i32, <2 x i1>, <2 x i32>)
2123declare <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>*, i32, <4 x i1>, <4 x i64>)
2124declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)
2125declare <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>*, i32, <8 x i1>, <8 x float>)
2126declare <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>)
2127declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
2128declare <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>*, i32, <2 x i1>, <2 x float>)
2129declare <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>*, i32, <8 x i1>, <8 x double>)
2130declare <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>*, i32, <4 x i1>, <4 x double>)
2131declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>*, i32, <2 x i1>, <2 x double>)
2132declare <1 x double> @llvm.masked.load.v1f64.p0v1f64(<1 x double>*, i32, <1 x i1>, <1 x double>)
2133