1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=ALL,SSE,SSE41
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=ALL,AVX,AVX1
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX,AVX2
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefixes=ALL,XOP
6
7; PR37428 - https://bugs.llvm.org/show_bug.cgi?id=37428
8; This is a larger-than-usual regression test to verify that several backend
9; transforms are working together. We want to hoist the expansion of non-uniform
10; vector shifts out of a loop if we do not have real vector shift instructions.
11; See test/Transforms/CodeGenPrepare/X86/vec-shift.ll for the 1st step in that
12; sequence.
13
14define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1) nounwind {
15; SSE-LABEL: vector_variable_shift_left_loop:
16; SSE:       # %bb.0: # %entry
17; SSE-NEXT:    testl %edx, %edx
18; SSE-NEXT:    jle .LBB0_9
19; SSE-NEXT:  # %bb.1: # %for.body.preheader
20; SSE-NEXT:    movl %ecx, %r9d
21; SSE-NEXT:    movl %edx, %eax
22; SSE-NEXT:    cmpl $31, %edx
23; SSE-NEXT:    ja .LBB0_3
24; SSE-NEXT:  # %bb.2:
25; SSE-NEXT:    xorl %edx, %edx
26; SSE-NEXT:    jmp .LBB0_6
27; SSE-NEXT:  .LBB0_3: # %vector.ph
28; SSE-NEXT:    movl %eax, %edx
29; SSE-NEXT:    andl $-32, %edx
30; SSE-NEXT:    movd %r9d, %xmm0
31; SSE-NEXT:    movd %r8d, %xmm1
32; SSE-NEXT:    xorl %ecx, %ecx
33; SSE-NEXT:    pmovzxdq {{.*#+}} xmm14 = xmm1[0],zero,xmm1[1],zero
34; SSE-NEXT:    pmovzxdq {{.*#+}} xmm15 = xmm0[0],zero,xmm0[1],zero
35; SSE-NEXT:    .p2align 4, 0x90
36; SSE-NEXT:  .LBB0_4: # %vector.body
37; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
38; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
39; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
40; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
41; SSE-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
42; SSE-NEXT:    pxor %xmm1, %xmm1
43; SSE-NEXT:    pcmpeqb %xmm1, %xmm0
44; SSE-NEXT:    pmovsxbd %xmm0, %xmm7
45; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
46; SSE-NEXT:    pmovsxbd %xmm0, %xmm0
47; SSE-NEXT:    pcmpeqb %xmm1, %xmm3
48; SSE-NEXT:    pmovsxbd %xmm3, %xmm13
49; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
50; SSE-NEXT:    pmovsxbd %xmm3, %xmm6
51; SSE-NEXT:    pcmpeqb %xmm1, %xmm4
52; SSE-NEXT:    pmovsxbd %xmm4, %xmm11
53; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
54; SSE-NEXT:    pmovsxbd %xmm3, %xmm2
55; SSE-NEXT:    pcmpeqb %xmm1, %xmm5
56; SSE-NEXT:    pmovsxbd %xmm5, %xmm8
57; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,2,3]
58; SSE-NEXT:    pmovsxbd %xmm3, %xmm9
59; SSE-NEXT:    movdqu 16(%rdi,%rcx,4), %xmm3
60; SSE-NEXT:    movdqa %xmm3, %xmm4
61; SSE-NEXT:    pslld %xmm15, %xmm4
62; SSE-NEXT:    pslld %xmm14, %xmm3
63; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm3
64; SSE-NEXT:    movdqu (%rdi,%rcx,4), %xmm10
65; SSE-NEXT:    movdqa %xmm10, %xmm5
66; SSE-NEXT:    pslld %xmm15, %xmm5
67; SSE-NEXT:    pslld %xmm14, %xmm10
68; SSE-NEXT:    movdqa %xmm7, %xmm0
69; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm10
70; SSE-NEXT:    movdqu 48(%rdi,%rcx,4), %xmm12
71; SSE-NEXT:    movdqa %xmm12, %xmm5
72; SSE-NEXT:    pslld %xmm15, %xmm5
73; SSE-NEXT:    pslld %xmm14, %xmm12
74; SSE-NEXT:    movdqa %xmm6, %xmm0
75; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm12
76; SSE-NEXT:    movdqu 32(%rdi,%rcx,4), %xmm6
77; SSE-NEXT:    movdqa %xmm6, %xmm5
78; SSE-NEXT:    pslld %xmm15, %xmm5
79; SSE-NEXT:    pslld %xmm14, %xmm6
80; SSE-NEXT:    movdqa %xmm13, %xmm0
81; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm6
82; SSE-NEXT:    movdqu 80(%rdi,%rcx,4), %xmm1
83; SSE-NEXT:    movdqa %xmm1, %xmm5
84; SSE-NEXT:    pslld %xmm15, %xmm5
85; SSE-NEXT:    pslld %xmm14, %xmm1
86; SSE-NEXT:    movdqa %xmm2, %xmm0
87; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm1
88; SSE-NEXT:    movdqu 64(%rdi,%rcx,4), %xmm5
89; SSE-NEXT:    movdqa %xmm5, %xmm2
90; SSE-NEXT:    pslld %xmm15, %xmm2
91; SSE-NEXT:    pslld %xmm14, %xmm5
92; SSE-NEXT:    movdqa %xmm11, %xmm0
93; SSE-NEXT:    blendvps %xmm0, %xmm2, %xmm5
94; SSE-NEXT:    movdqu 112(%rdi,%rcx,4), %xmm2
95; SSE-NEXT:    movdqa %xmm2, %xmm4
96; SSE-NEXT:    pslld %xmm15, %xmm4
97; SSE-NEXT:    pslld %xmm14, %xmm2
98; SSE-NEXT:    movdqa %xmm9, %xmm0
99; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm2
100; SSE-NEXT:    movdqu 96(%rdi,%rcx,4), %xmm4
101; SSE-NEXT:    movdqa %xmm4, %xmm7
102; SSE-NEXT:    pslld %xmm15, %xmm7
103; SSE-NEXT:    pslld %xmm14, %xmm4
104; SSE-NEXT:    movdqa %xmm8, %xmm0
105; SSE-NEXT:    blendvps %xmm0, %xmm7, %xmm4
106; SSE-NEXT:    movups %xmm10, (%rdi,%rcx,4)
107; SSE-NEXT:    movups %xmm3, 16(%rdi,%rcx,4)
108; SSE-NEXT:    movups %xmm6, 32(%rdi,%rcx,4)
109; SSE-NEXT:    movups %xmm12, 48(%rdi,%rcx,4)
110; SSE-NEXT:    movups %xmm5, 64(%rdi,%rcx,4)
111; SSE-NEXT:    movups %xmm1, 80(%rdi,%rcx,4)
112; SSE-NEXT:    movups %xmm4, 96(%rdi,%rcx,4)
113; SSE-NEXT:    movups %xmm2, 112(%rdi,%rcx,4)
114; SSE-NEXT:    addq $32, %rcx
115; SSE-NEXT:    cmpq %rcx, %rdx
116; SSE-NEXT:    jne .LBB0_4
117; SSE-NEXT:  # %bb.5: # %middle.block
118; SSE-NEXT:    cmpq %rax, %rdx
119; SSE-NEXT:    jne .LBB0_6
120; SSE-NEXT:  .LBB0_9: # %for.cond.cleanup
121; SSE-NEXT:    retq
122; SSE-NEXT:    .p2align 4, 0x90
123; SSE-NEXT:  .LBB0_8: # %for.body
124; SSE-NEXT:    # in Loop: Header=BB0_6 Depth=1
125; SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
126; SSE-NEXT:    shll %cl, (%rdi,%rdx,4)
127; SSE-NEXT:    incq %rdx
128; SSE-NEXT:    cmpq %rdx, %rax
129; SSE-NEXT:    je .LBB0_9
130; SSE-NEXT:  .LBB0_6: # %for.body
131; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
132; SSE-NEXT:    cmpb $0, (%rsi,%rdx)
133; SSE-NEXT:    movl %r9d, %ecx
134; SSE-NEXT:    je .LBB0_8
135; SSE-NEXT:  # %bb.7: # %for.body
136; SSE-NEXT:    # in Loop: Header=BB0_6 Depth=1
137; SSE-NEXT:    movl %r8d, %ecx
138; SSE-NEXT:    jmp .LBB0_8
139;
140; AVX1-LABEL: vector_variable_shift_left_loop:
141; AVX1:       # %bb.0: # %entry
142; AVX1-NEXT:    subq $24, %rsp
143; AVX1-NEXT:    testl %edx, %edx
144; AVX1-NEXT:    jle .LBB0_9
145; AVX1-NEXT:  # %bb.1: # %for.body.preheader
146; AVX1-NEXT:    movl %ecx, %r9d
147; AVX1-NEXT:    movl %edx, %eax
148; AVX1-NEXT:    cmpl $31, %edx
149; AVX1-NEXT:    ja .LBB0_3
150; AVX1-NEXT:  # %bb.2:
151; AVX1-NEXT:    xorl %edx, %edx
152; AVX1-NEXT:    jmp .LBB0_6
153; AVX1-NEXT:  .LBB0_3: # %vector.ph
154; AVX1-NEXT:    movl %eax, %edx
155; AVX1-NEXT:    andl $-32, %edx
156; AVX1-NEXT:    vmovd %r9d, %xmm0
157; AVX1-NEXT:    vmovd %r8d, %xmm1
158; AVX1-NEXT:    xorl %ecx, %ecx
159; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
160; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
161; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
162; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
163; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
164; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
165; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm0[0],zero,xmm0[1],zero
166; AVX1-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
167; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm1[0],zero,xmm1[1],zero
168; AVX1-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
169; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm0[0],zero,xmm0[1],zero
170; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
171; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
172; AVX1-NEXT:    .p2align 4, 0x90
173; AVX1-NEXT:  .LBB0_4: # %vector.body
174; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
175; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
176; AVX1-NEXT:    # xmm1 = mem[0],zero,mem[1],zero
177; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
178; AVX1-NEXT:    # xmm2 = mem[0],zero,mem[1],zero
179; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
180; AVX1-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
181; AVX1-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
182; AVX1-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
183; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm11, %xmm3
184; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm7
185; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
186; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
187; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm11, %xmm4
188; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm8
189; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
190; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
191; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm11, %xmm5
192; AVX1-NEXT:    vmovdqu (%rdi,%rcx,4), %xmm9
193; AVX1-NEXT:    vpslld %xmm2, %xmm9, %xmm10
194; AVX1-NEXT:    vpslld %xmm1, %xmm9, %xmm0
195; AVX1-NEXT:    vblendvps %xmm7, %xmm10, %xmm0, %xmm9
196; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm7
197; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
198; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
199; AVX1-NEXT:    vpcmpeqb %xmm6, %xmm11, %xmm6
200; AVX1-NEXT:    vmovdqu 16(%rdi,%rcx,4), %xmm0
201; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
202; AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
203; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
204; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
205; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm6
206; AVX1-NEXT:    vblendvps %xmm3, %xmm2, %xmm0, %xmm10
207; AVX1-NEXT:    vmovdqu 32(%rdi,%rcx,4), %xmm2
208; AVX1-NEXT:    vpslld %xmm15, %xmm2, %xmm3
209; AVX1-NEXT:    vpslld %xmm14, %xmm2, %xmm2
210; AVX1-NEXT:    vblendvps %xmm8, %xmm3, %xmm2, %xmm8
211; AVX1-NEXT:    vmovdqu 48(%rdi,%rcx,4), %xmm3
212; AVX1-NEXT:    vpslld %xmm15, %xmm3, %xmm0
213; AVX1-NEXT:    vpslld %xmm14, %xmm3, %xmm3
214; AVX1-NEXT:    vblendvps %xmm4, %xmm0, %xmm3, %xmm0
215; AVX1-NEXT:    vmovdqu 64(%rdi,%rcx,4), %xmm3
216; AVX1-NEXT:    vpslld %xmm13, %xmm3, %xmm4
217; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
218; AVX1-NEXT:    vpslld %xmm2, %xmm3, %xmm3
219; AVX1-NEXT:    vblendvps %xmm7, %xmm4, %xmm3, %xmm3
220; AVX1-NEXT:    vmovdqu 80(%rdi,%rcx,4), %xmm4
221; AVX1-NEXT:    vpslld %xmm13, %xmm4, %xmm7
222; AVX1-NEXT:    vpslld %xmm2, %xmm4, %xmm4
223; AVX1-NEXT:    vblendvps %xmm5, %xmm7, %xmm4, %xmm4
224; AVX1-NEXT:    vmovdqu 96(%rdi,%rcx,4), %xmm5
225; AVX1-NEXT:    vpslld %xmm12, %xmm5, %xmm7
226; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
227; AVX1-NEXT:    vpslld %xmm2, %xmm5, %xmm5
228; AVX1-NEXT:    vblendvps %xmm1, %xmm7, %xmm5, %xmm1
229; AVX1-NEXT:    vmovdqu 112(%rdi,%rcx,4), %xmm5
230; AVX1-NEXT:    vpslld %xmm12, %xmm5, %xmm7
231; AVX1-NEXT:    vpslld %xmm2, %xmm5, %xmm5
232; AVX1-NEXT:    vblendvps %xmm6, %xmm7, %xmm5, %xmm5
233; AVX1-NEXT:    vmovups %xmm9, (%rdi,%rcx,4)
234; AVX1-NEXT:    vmovups %xmm10, 16(%rdi,%rcx,4)
235; AVX1-NEXT:    vmovups %xmm8, 32(%rdi,%rcx,4)
236; AVX1-NEXT:    vmovups %xmm0, 48(%rdi,%rcx,4)
237; AVX1-NEXT:    vmovups %xmm3, 64(%rdi,%rcx,4)
238; AVX1-NEXT:    vmovups %xmm4, 80(%rdi,%rcx,4)
239; AVX1-NEXT:    vmovups %xmm1, 96(%rdi,%rcx,4)
240; AVX1-NEXT:    vmovups %xmm5, 112(%rdi,%rcx,4)
241; AVX1-NEXT:    addq $32, %rcx
242; AVX1-NEXT:    cmpq %rcx, %rdx
243; AVX1-NEXT:    jne .LBB0_4
244; AVX1-NEXT:  # %bb.5: # %middle.block
245; AVX1-NEXT:    cmpq %rax, %rdx
246; AVX1-NEXT:    jne .LBB0_6
247; AVX1-NEXT:  .LBB0_9: # %for.cond.cleanup
248; AVX1-NEXT:    addq $24, %rsp
249; AVX1-NEXT:    vzeroupper
250; AVX1-NEXT:    retq
251; AVX1-NEXT:    .p2align 4, 0x90
252; AVX1-NEXT:  .LBB0_8: # %for.body
253; AVX1-NEXT:    # in Loop: Header=BB0_6 Depth=1
254; AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
255; AVX1-NEXT:    shll %cl, (%rdi,%rdx,4)
256; AVX1-NEXT:    incq %rdx
257; AVX1-NEXT:    cmpq %rdx, %rax
258; AVX1-NEXT:    je .LBB0_9
259; AVX1-NEXT:  .LBB0_6: # %for.body
260; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
261; AVX1-NEXT:    cmpb $0, (%rsi,%rdx)
262; AVX1-NEXT:    movl %r9d, %ecx
263; AVX1-NEXT:    je .LBB0_8
264; AVX1-NEXT:  # %bb.7: # %for.body
265; AVX1-NEXT:    # in Loop: Header=BB0_6 Depth=1
266; AVX1-NEXT:    movl %r8d, %ecx
267; AVX1-NEXT:    jmp .LBB0_8
268;
269; AVX2-LABEL: vector_variable_shift_left_loop:
270; AVX2:       # %bb.0: # %entry
271; AVX2-NEXT:    testl %edx, %edx
272; AVX2-NEXT:    jle .LBB0_9
273; AVX2-NEXT:  # %bb.1: # %for.body.preheader
274; AVX2-NEXT:    movl %ecx, %r9d
275; AVX2-NEXT:    movl %edx, %eax
276; AVX2-NEXT:    cmpl $31, %edx
277; AVX2-NEXT:    ja .LBB0_3
278; AVX2-NEXT:  # %bb.2:
279; AVX2-NEXT:    xorl %edx, %edx
280; AVX2-NEXT:    jmp .LBB0_6
281; AVX2-NEXT:  .LBB0_3: # %vector.ph
282; AVX2-NEXT:    movl %eax, %edx
283; AVX2-NEXT:    andl $-32, %edx
284; AVX2-NEXT:    vmovd %r9d, %xmm0
285; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
286; AVX2-NEXT:    vmovd %r8d, %xmm1
287; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
288; AVX2-NEXT:    xorl %ecx, %ecx
289; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
290; AVX2-NEXT:    .p2align 4, 0x90
291; AVX2-NEXT:  .LBB0_4: # %vector.body
292; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
293; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
294; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
295; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
296; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
297; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm3, %ymm3
298; AVX2-NEXT:    vblendvps %ymm3, %ymm0, %ymm1, %ymm3
299; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm4, %ymm4
300; AVX2-NEXT:    vblendvps %ymm4, %ymm0, %ymm1, %ymm4
301; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm5
302; AVX2-NEXT:    vblendvps %ymm5, %ymm0, %ymm1, %ymm5
303; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm6, %ymm6
304; AVX2-NEXT:    vblendvps %ymm6, %ymm0, %ymm1, %ymm6
305; AVX2-NEXT:    vmovdqu (%rdi,%rcx,4), %ymm7
306; AVX2-NEXT:    vpsllvd %ymm3, %ymm7, %ymm3
307; AVX2-NEXT:    vmovdqu 32(%rdi,%rcx,4), %ymm7
308; AVX2-NEXT:    vpsllvd %ymm4, %ymm7, %ymm4
309; AVX2-NEXT:    vmovdqu 64(%rdi,%rcx,4), %ymm7
310; AVX2-NEXT:    vpsllvd %ymm5, %ymm7, %ymm5
311; AVX2-NEXT:    vmovdqu 96(%rdi,%rcx,4), %ymm7
312; AVX2-NEXT:    vpsllvd %ymm6, %ymm7, %ymm6
313; AVX2-NEXT:    vmovdqu %ymm3, (%rdi,%rcx,4)
314; AVX2-NEXT:    vmovdqu %ymm4, 32(%rdi,%rcx,4)
315; AVX2-NEXT:    vmovdqu %ymm5, 64(%rdi,%rcx,4)
316; AVX2-NEXT:    vmovdqu %ymm6, 96(%rdi,%rcx,4)
317; AVX2-NEXT:    addq $32, %rcx
318; AVX2-NEXT:    cmpq %rcx, %rdx
319; AVX2-NEXT:    jne .LBB0_4
320; AVX2-NEXT:  # %bb.5: # %middle.block
321; AVX2-NEXT:    cmpq %rax, %rdx
322; AVX2-NEXT:    jne .LBB0_6
323; AVX2-NEXT:  .LBB0_9: # %for.cond.cleanup
324; AVX2-NEXT:    vzeroupper
325; AVX2-NEXT:    retq
326; AVX2-NEXT:    .p2align 4, 0x90
327; AVX2-NEXT:  .LBB0_8: # %for.body
328; AVX2-NEXT:    # in Loop: Header=BB0_6 Depth=1
329; AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
330; AVX2-NEXT:    shll %cl, (%rdi,%rdx,4)
331; AVX2-NEXT:    incq %rdx
332; AVX2-NEXT:    cmpq %rdx, %rax
333; AVX2-NEXT:    je .LBB0_9
334; AVX2-NEXT:  .LBB0_6: # %for.body
335; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
336; AVX2-NEXT:    cmpb $0, (%rsi,%rdx)
337; AVX2-NEXT:    movl %r9d, %ecx
338; AVX2-NEXT:    je .LBB0_8
339; AVX2-NEXT:  # %bb.7: # %for.body
340; AVX2-NEXT:    # in Loop: Header=BB0_6 Depth=1
341; AVX2-NEXT:    movl %r8d, %ecx
342; AVX2-NEXT:    jmp .LBB0_8
343;
344; XOP-LABEL: vector_variable_shift_left_loop:
345; XOP:       # %bb.0: # %entry
346; XOP-NEXT:    testl %edx, %edx
347; XOP-NEXT:    jle .LBB0_9
348; XOP-NEXT:  # %bb.1: # %for.body.preheader
349; XOP-NEXT:    movl %ecx, %r9d
350; XOP-NEXT:    movl %edx, %eax
351; XOP-NEXT:    cmpl $31, %edx
352; XOP-NEXT:    ja .LBB0_3
353; XOP-NEXT:  # %bb.2:
354; XOP-NEXT:    xorl %edx, %edx
355; XOP-NEXT:    jmp .LBB0_6
356; XOP-NEXT:  .LBB0_3: # %vector.ph
357; XOP-NEXT:    movl %eax, %edx
358; XOP-NEXT:    andl $-32, %edx
359; XOP-NEXT:    vmovd %r9d, %xmm0
360; XOP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
361; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm9
362; XOP-NEXT:    vmovd %r8d, %xmm1
363; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
364; XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm14
365; XOP-NEXT:    xorl %ecx, %ecx
366; XOP-NEXT:    vpxor %xmm8, %xmm8, %xmm8
367; XOP-NEXT:    vextractf128 $1, %ymm9, %xmm15
368; XOP-NEXT:    vextractf128 $1, %ymm14, %xmm4
369; XOP-NEXT:    .p2align 4, 0x90
370; XOP-NEXT:  .LBB0_4: # %vector.body
371; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
372; XOP-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
373; XOP-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
374; XOP-NEXT:    vmovq {{.*#+}} xmm7 = mem[0],zero
375; XOP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
376; XOP-NEXT:    vpcomeqb %xmm8, %xmm5, %xmm5
377; XOP-NEXT:    vpmovsxbd %xmm5, %xmm0
378; XOP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
379; XOP-NEXT:    vpmovsxbd %xmm5, %xmm5
380; XOP-NEXT:    vpcomeqb %xmm8, %xmm6, %xmm6
381; XOP-NEXT:    vpmovsxbd %xmm6, %xmm10
382; XOP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
383; XOP-NEXT:    vpmovsxbd %xmm6, %xmm6
384; XOP-NEXT:    vpcomeqb %xmm8, %xmm7, %xmm7
385; XOP-NEXT:    vpmovsxbd %xmm7, %xmm11
386; XOP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,3]
387; XOP-NEXT:    vpmovsxbd %xmm7, %xmm7
388; XOP-NEXT:    vpcomeqb %xmm8, %xmm2, %xmm2
389; XOP-NEXT:    vpmovsxbd %xmm2, %xmm12
390; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
391; XOP-NEXT:    vpmovsxbd %xmm2, %xmm2
392; XOP-NEXT:    vblendvps %xmm5, %xmm15, %xmm4, %xmm5
393; XOP-NEXT:    vpshld %xmm5, 16(%rdi,%rcx,4), %xmm13
394; XOP-NEXT:    vblendvps %xmm0, %xmm9, %xmm14, %xmm0
395; XOP-NEXT:    vpshld %xmm0, (%rdi,%rcx,4), %xmm0
396; XOP-NEXT:    vblendvps %xmm6, %xmm15, %xmm4, %xmm6
397; XOP-NEXT:    vpshld %xmm6, 48(%rdi,%rcx,4), %xmm6
398; XOP-NEXT:    vblendvps %xmm10, %xmm9, %xmm14, %xmm5
399; XOP-NEXT:    vpshld %xmm5, 32(%rdi,%rcx,4), %xmm5
400; XOP-NEXT:    vblendvps %xmm7, %xmm15, %xmm4, %xmm7
401; XOP-NEXT:    vpshld %xmm7, 80(%rdi,%rcx,4), %xmm7
402; XOP-NEXT:    vblendvps %xmm11, %xmm9, %xmm14, %xmm1
403; XOP-NEXT:    vpshld %xmm1, 64(%rdi,%rcx,4), %xmm1
404; XOP-NEXT:    vblendvps %xmm2, %xmm15, %xmm4, %xmm2
405; XOP-NEXT:    vpshld %xmm2, 112(%rdi,%rcx,4), %xmm2
406; XOP-NEXT:    vblendvps %xmm12, %xmm9, %xmm14, %xmm3
407; XOP-NEXT:    vpshld %xmm3, 96(%rdi,%rcx,4), %xmm3
408; XOP-NEXT:    vmovdqu %xmm0, (%rdi,%rcx,4)
409; XOP-NEXT:    vmovdqu %xmm13, 16(%rdi,%rcx,4)
410; XOP-NEXT:    vmovdqu %xmm5, 32(%rdi,%rcx,4)
411; XOP-NEXT:    vmovdqu %xmm6, 48(%rdi,%rcx,4)
412; XOP-NEXT:    vmovdqu %xmm1, 64(%rdi,%rcx,4)
413; XOP-NEXT:    vmovdqu %xmm7, 80(%rdi,%rcx,4)
414; XOP-NEXT:    vmovdqu %xmm3, 96(%rdi,%rcx,4)
415; XOP-NEXT:    vmovdqu %xmm2, 112(%rdi,%rcx,4)
416; XOP-NEXT:    addq $32, %rcx
417; XOP-NEXT:    cmpq %rcx, %rdx
418; XOP-NEXT:    jne .LBB0_4
419; XOP-NEXT:  # %bb.5: # %middle.block
420; XOP-NEXT:    cmpq %rax, %rdx
421; XOP-NEXT:    jne .LBB0_6
422; XOP-NEXT:  .LBB0_9: # %for.cond.cleanup
423; XOP-NEXT:    vzeroupper
424; XOP-NEXT:    retq
425; XOP-NEXT:    .p2align 4, 0x90
426; XOP-NEXT:  .LBB0_8: # %for.body
427; XOP-NEXT:    # in Loop: Header=BB0_6 Depth=1
428; XOP-NEXT:    # kill: def $cl killed $cl killed $ecx
429; XOP-NEXT:    shll %cl, (%rdi,%rdx,4)
430; XOP-NEXT:    incq %rdx
431; XOP-NEXT:    cmpq %rdx, %rax
432; XOP-NEXT:    je .LBB0_9
433; XOP-NEXT:  .LBB0_6: # %for.body
434; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
435; XOP-NEXT:    cmpb $0, (%rsi,%rdx)
436; XOP-NEXT:    movl %r9d, %ecx
437; XOP-NEXT:    je .LBB0_8
438; XOP-NEXT:  # %bb.7: # %for.body
439; XOP-NEXT:    # in Loop: Header=BB0_6 Depth=1
440; XOP-NEXT:    movl %r8d, %ecx
441; XOP-NEXT:    jmp .LBB0_8
442entry:
443  %cmp12 = icmp sgt i32 %count, 0
444  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
445
446for.body.preheader:
447  %wide.trip.count = zext i32 %count to i64
448  %min.iters.check = icmp ult i32 %count, 32
449  br i1 %min.iters.check, label %for.body.preheader40, label %vector.ph
450
451for.body.preheader40:
452  %indvars.iv.ph = phi i64 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
453  br label %for.body
454
455vector.ph:
456  %n.vec = and i64 %wide.trip.count, 4294967264
457  %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %amt0, i32 0
458  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer
459  %broadcast.splatinsert22 = insertelement <8 x i32> undef, i32 %amt1, i32 0
460  %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> undef, <8 x i32> zeroinitializer
461  %broadcast.splatinsert24 = insertelement <8 x i32> undef, i32 %amt0, i32 0
462  %broadcast.splat25 = shufflevector <8 x i32> %broadcast.splatinsert24, <8 x i32> undef, <8 x i32> zeroinitializer
463  %broadcast.splatinsert26 = insertelement <8 x i32> undef, i32 %amt1, i32 0
464  %broadcast.splat27 = shufflevector <8 x i32> %broadcast.splatinsert26, <8 x i32> undef, <8 x i32> zeroinitializer
465  %broadcast.splatinsert28 = insertelement <8 x i32> undef, i32 %amt0, i32 0
466  %broadcast.splat29 = shufflevector <8 x i32> %broadcast.splatinsert28, <8 x i32> undef, <8 x i32> zeroinitializer
467  %broadcast.splatinsert30 = insertelement <8 x i32> undef, i32 %amt1, i32 0
468  %broadcast.splat31 = shufflevector <8 x i32> %broadcast.splatinsert30, <8 x i32> undef, <8 x i32> zeroinitializer
469  %broadcast.splatinsert32 = insertelement <8 x i32> undef, i32 %amt0, i32 0
470  %broadcast.splat33 = shufflevector <8 x i32> %broadcast.splatinsert32, <8 x i32> undef, <8 x i32> zeroinitializer
471  %broadcast.splatinsert34 = insertelement <8 x i32> undef, i32 %amt1, i32 0
472  %broadcast.splat35 = shufflevector <8 x i32> %broadcast.splatinsert34, <8 x i32> undef, <8 x i32> zeroinitializer
473  br label %vector.body
474
475vector.body:
476  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
477  %0 = getelementptr inbounds i8, i8* %control, i64 %index
478  %1 = bitcast i8* %0 to <8 x i8>*
479  %wide.load = load <8 x i8>, <8 x i8>* %1, align 1
480  %2 = getelementptr inbounds i8, i8* %0, i64 8
481  %3 = bitcast i8* %2 to <8 x i8>*
482  %wide.load17 = load <8 x i8>, <8 x i8>* %3, align 1
483  %4 = getelementptr inbounds i8, i8* %0, i64 16
484  %5 = bitcast i8* %4 to <8 x i8>*
485  %wide.load18 = load <8 x i8>, <8 x i8>* %5, align 1
486  %6 = getelementptr inbounds i8, i8* %0, i64 24
487  %7 = bitcast i8* %6 to <8 x i8>*
488  %wide.load19 = load <8 x i8>, <8 x i8>* %7, align 1
489  %8 = icmp eq <8 x i8> %wide.load, zeroinitializer
490  %9 = icmp eq <8 x i8> %wide.load17, zeroinitializer
491  %10 = icmp eq <8 x i8> %wide.load18, zeroinitializer
492  %11 = icmp eq <8 x i8> %wide.load19, zeroinitializer
493  %12 = select <8 x i1> %8, <8 x i32> %broadcast.splat21, <8 x i32> %broadcast.splat23
494  %13 = select <8 x i1> %9, <8 x i32> %broadcast.splat25, <8 x i32> %broadcast.splat27
495  %14 = select <8 x i1> %10, <8 x i32> %broadcast.splat29, <8 x i32> %broadcast.splat31
496  %15 = select <8 x i1> %11, <8 x i32> %broadcast.splat33, <8 x i32> %broadcast.splat35
497  %16 = getelementptr inbounds i32, i32* %arr, i64 %index
498  %17 = bitcast i32* %16 to <8 x i32>*
499  %wide.load36 = load <8 x i32>, <8 x i32>* %17, align 4
500  %18 = getelementptr inbounds i32, i32* %16, i64 8
501  %19 = bitcast i32* %18 to <8 x i32>*
502  %wide.load37 = load <8 x i32>, <8 x i32>* %19, align 4
503  %20 = getelementptr inbounds i32, i32* %16, i64 16
504  %21 = bitcast i32* %20 to <8 x i32>*
505  %wide.load38 = load <8 x i32>, <8 x i32>* %21, align 4
506  %22 = getelementptr inbounds i32, i32* %16, i64 24
507  %23 = bitcast i32* %22 to <8 x i32>*
508  %wide.load39 = load <8 x i32>, <8 x i32>* %23, align 4
509  %24 = shl <8 x i32> %wide.load36, %12
510  %25 = shl <8 x i32> %wide.load37, %13
511  %26 = shl <8 x i32> %wide.load38, %14
512  %27 = shl <8 x i32> %wide.load39, %15
513  %28 = bitcast i32* %16 to <8 x i32>*
514  store <8 x i32> %24, <8 x i32>* %28, align 4
515  %29 = bitcast i32* %18 to <8 x i32>*
516  store <8 x i32> %25, <8 x i32>* %29, align 4
517  %30 = bitcast i32* %20 to <8 x i32>*
518  store <8 x i32> %26, <8 x i32>* %30, align 4
519  %31 = bitcast i32* %22 to <8 x i32>*
520  store <8 x i32> %27, <8 x i32>* %31, align 4
521  %index.next = add i64 %index, 32
522  %32 = icmp eq i64 %index.next, %n.vec
523  br i1 %32, label %middle.block, label %vector.body
524
525middle.block:
526  %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
527  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader40
528
529for.cond.cleanup:
530  ret void
531
532for.body:
533  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader40 ]
534  %arrayidx = getelementptr inbounds i8, i8* %control, i64 %indvars.iv
535  %33 = load i8, i8* %arrayidx, align 1
536  %tobool = icmp eq i8 %33, 0
537  %cond = select i1 %tobool, i32 %amt0, i32 %amt1
538  %arrayidx2 = getelementptr inbounds i32, i32* %arr, i64 %indvars.iv
539  %34 = load i32, i32* %arrayidx2, align 4
540  %shl = shl i32 %34, %cond
541  store i32 %shl, i32* %arrayidx2, align 4
542  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
543  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
544  br i1 %exitcond, label %for.cond.cleanup, label %for.body
545}
546
547define void @vector_variable_shift_left_loop_simpler(i32* nocapture %arr, i8* nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1, i32 %x) nounwind {
548; SSE-LABEL: vector_variable_shift_left_loop_simpler:
549; SSE:       # %bb.0: # %entry
550; SSE-NEXT:    testl %edx, %edx
551; SSE-NEXT:    jle .LBB1_3
552; SSE-NEXT:  # %bb.1: # %vector.ph
553; SSE-NEXT:    movl %edx, %eax
554; SSE-NEXT:    andl $-4, %eax
555; SSE-NEXT:    movd %ecx, %xmm0
556; SSE-NEXT:    movd %r8d, %xmm2
557; SSE-NEXT:    movd %r9d, %xmm3
558; SSE-NEXT:    xorl %ecx, %ecx
559; SSE-NEXT:    pslld $23, %xmm0
560; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
561; SSE-NEXT:    paddd %xmm4, %xmm0
562; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
563; SSE-NEXT:    pmulld %xmm3, %xmm0
564; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
565; SSE-NEXT:    pslld $23, %xmm2
566; SSE-NEXT:    paddd %xmm4, %xmm2
567; SSE-NEXT:    cvttps2dq %xmm2, %xmm0
568; SSE-NEXT:    pmulld %xmm3, %xmm0
569; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
570; SSE-NEXT:    pxor %xmm3, %xmm3
571; SSE-NEXT:    .p2align 4, 0x90
572; SSE-NEXT:  .LBB1_2: # %vector.body
573; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
574; SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
575; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
576; SSE-NEXT:    movdqa %xmm2, %xmm4
577; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm4
578; SSE-NEXT:    movups %xmm4, (%rdi,%rcx,4)
579; SSE-NEXT:    addq $4, %rcx
580; SSE-NEXT:    cmpq %rcx, %rax
581; SSE-NEXT:    jne .LBB1_2
582; SSE-NEXT:  .LBB1_3: # %exit
583; SSE-NEXT:    retq
584;
585; AVX1-LABEL: vector_variable_shift_left_loop_simpler:
586; AVX1:       # %bb.0: # %entry
587; AVX1-NEXT:    testl %edx, %edx
588; AVX1-NEXT:    jle .LBB1_3
589; AVX1-NEXT:  # %bb.1: # %vector.ph
590; AVX1-NEXT:    movl %edx, %eax
591; AVX1-NEXT:    andl $-4, %eax
592; AVX1-NEXT:    vmovd %ecx, %xmm0
593; AVX1-NEXT:    vmovd %r8d, %xmm1
594; AVX1-NEXT:    vmovd %r9d, %xmm2
595; AVX1-NEXT:    xorl %ecx, %ecx
596; AVX1-NEXT:    vpslld $23, %xmm0, %xmm0
597; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
598; AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
599; AVX1-NEXT:    vcvttps2dq %xmm0, %xmm0
600; AVX1-NEXT:    vpmulld %xmm0, %xmm2, %xmm0
601; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
602; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
603; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
604; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
605; AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
606; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
607; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
608; AVX1-NEXT:    .p2align 4, 0x90
609; AVX1-NEXT:  .LBB1_2: # %vector.body
610; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
611; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
612; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm3
613; AVX1-NEXT:    vblendvps %xmm3, %xmm0, %xmm1, %xmm3
614; AVX1-NEXT:    vmovups %xmm3, (%rdi,%rcx,4)
615; AVX1-NEXT:    addq $4, %rcx
616; AVX1-NEXT:    cmpq %rcx, %rax
617; AVX1-NEXT:    jne .LBB1_2
618; AVX1-NEXT:  .LBB1_3: # %exit
619; AVX1-NEXT:    retq
620;
621; AVX2-LABEL: vector_variable_shift_left_loop_simpler:
622; AVX2:       # %bb.0: # %entry
623; AVX2-NEXT:    testl %edx, %edx
624; AVX2-NEXT:    jle .LBB1_3
625; AVX2-NEXT:  # %bb.1: # %vector.ph
626; AVX2-NEXT:    movl %edx, %eax
627; AVX2-NEXT:    andl $-4, %eax
628; AVX2-NEXT:    vmovd %ecx, %xmm0
629; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
630; AVX2-NEXT:    vmovd %r8d, %xmm1
631; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
632; AVX2-NEXT:    vmovd %r9d, %xmm2
633; AVX2-NEXT:    vpbroadcastd %xmm2, %xmm2
634; AVX2-NEXT:    xorl %ecx, %ecx
635; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
636; AVX2-NEXT:    .p2align 4, 0x90
637; AVX2-NEXT:  .LBB1_2: # %vector.body
638; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
639; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
640; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
641; AVX2-NEXT:    vblendvps %xmm4, %xmm0, %xmm1, %xmm4
642; AVX2-NEXT:    vpsllvd %xmm4, %xmm2, %xmm4
643; AVX2-NEXT:    vmovdqu %xmm4, (%rdi,%rcx,4)
644; AVX2-NEXT:    addq $4, %rcx
645; AVX2-NEXT:    cmpq %rcx, %rax
646; AVX2-NEXT:    jne .LBB1_2
647; AVX2-NEXT:  .LBB1_3: # %exit
648; AVX2-NEXT:    retq
649;
650; XOP-LABEL: vector_variable_shift_left_loop_simpler:
651; XOP:       # %bb.0: # %entry
652; XOP-NEXT:    testl %edx, %edx
653; XOP-NEXT:    jle .LBB1_3
654; XOP-NEXT:  # %bb.1: # %vector.ph
655; XOP-NEXT:    movl %edx, %eax
656; XOP-NEXT:    andl $-4, %eax
657; XOP-NEXT:    vmovd %ecx, %xmm0
658; XOP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
659; XOP-NEXT:    vmovd %r8d, %xmm1
660; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
661; XOP-NEXT:    vmovd %r9d, %xmm2
662; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
663; XOP-NEXT:    xorl %ecx, %ecx
664; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
665; XOP-NEXT:    .p2align 4, 0x90
666; XOP-NEXT:  .LBB1_2: # %vector.body
667; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
668; XOP-NEXT:    vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
669; XOP-NEXT:    vpcomeqd %xmm3, %xmm4, %xmm4
670; XOP-NEXT:    vblendvps %xmm4, %xmm0, %xmm1, %xmm4
671; XOP-NEXT:    vpshld %xmm4, %xmm2, %xmm4
672; XOP-NEXT:    vmovdqu %xmm4, (%rdi,%rcx,4)
673; XOP-NEXT:    addq $4, %rcx
674; XOP-NEXT:    cmpq %rcx, %rax
675; XOP-NEXT:    jne .LBB1_2
676; XOP-NEXT:  .LBB1_3: # %exit
677; XOP-NEXT:    retq
678entry:
679  %cmp16 = icmp sgt i32 %count, 0
680  %wide.trip.count = zext i32 %count to i64
681  br i1 %cmp16, label %vector.ph, label %exit
682
683vector.ph:
684  %n.vec = and i64 %wide.trip.count, 4294967292
685  %splatinsert18 = insertelement <4 x i32> undef, i32 %amt0, i32 0
686  %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer
687  %splatinsert20 = insertelement <4 x i32> undef, i32 %amt1, i32 0
688  %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
689  %splatinsert22 = insertelement <4 x i32> undef, i32 %x, i32 0
690  %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
691  br label %vector.body
692
693vector.body:
694  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
695  %0 = getelementptr inbounds i8, i8* %control, i64 %index
696  %1 = bitcast i8* %0 to <4 x i8>*
697  %wide.load = load <4 x i8>, <4 x i8>* %1, align 1
698  %2 = icmp eq <4 x i8> %wide.load, zeroinitializer
699  %3 = select <4 x i1> %2, <4 x i32> %splat1, <4 x i32> %splat2
700  %4 = shl <4 x i32> %splat3, %3
701  %5 = getelementptr inbounds i32, i32* %arr, i64 %index
702  %6 = bitcast i32* %5 to <4 x i32>*
703  store <4 x i32> %4, <4 x i32>* %6, align 4
704  %index.next = add i64 %index, 4
705  %7 = icmp eq i64 %index.next, %n.vec
706  br i1 %7, label %exit, label %vector.body
707
708exit:
709  ret void
710}
711