1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
5
6; PR37428 - https://bugs.llvm.org/show_bug.cgi?id=37428
7; This is a larger-than-usual regression test to verify that several backend
8; transforms are working together. We want to hoist the expansion of non-uniform
9; vector shifts out of a loop if we do not have real vector shift instructions.
10; See test/Transforms/CodeGenPrepare/X86/vec-shift.ll for the 1st step in that
11; sequence.
12
13define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1) nounwind {
14; SSE-LABEL: vector_variable_shift_left_loop:
15; SSE:       # %bb.0: # %entry
16; SSE-NEXT:    testl %edx, %edx
17; SSE-NEXT:    jle .LBB0_9
18; SSE-NEXT:  # %bb.1: # %for.body.preheader
19; SSE-NEXT:    movl %ecx, %r9d
20; SSE-NEXT:    movl %edx, %eax
21; SSE-NEXT:    cmpl $31, %edx
22; SSE-NEXT:    ja .LBB0_3
23; SSE-NEXT:  # %bb.2:
24; SSE-NEXT:    xorl %edx, %edx
25; SSE-NEXT:    jmp .LBB0_6
26; SSE-NEXT:  .LBB0_3: # %vector.ph
27; SSE-NEXT:    movl %eax, %edx
28; SSE-NEXT:    andl $-32, %edx
29; SSE-NEXT:    movd %r9d, %xmm0
30; SSE-NEXT:    movd %r8d, %xmm1
31; SSE-NEXT:    xorl %ecx, %ecx
32; SSE-NEXT:    pmovzxdq {{.*#+}} xmm13 = xmm1[0],zero,xmm1[1],zero
33; SSE-NEXT:    pmovzxdq {{.*#+}} xmm14 = xmm0[0],zero,xmm0[1],zero
34; SSE-NEXT:    .p2align 4, 0x90
35; SSE-NEXT:  .LBB0_4: # %vector.body
36; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
37; SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
38; SSE-NEXT:    pmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
39; SSE-NEXT:    pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
40; SSE-NEXT:    pmovzxbw {{.*#+}} xmm15 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
41; SSE-NEXT:    pxor %xmm1, %xmm1
42; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
43; SSE-NEXT:    pmovzxwd {{.*#+}} xmm12 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
44; SSE-NEXT:    pslld $24, %xmm12
45; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
46; SSE-NEXT:    pslld $24, %xmm0
47; SSE-NEXT:    pcmpeqw %xmm1, %xmm3
48; SSE-NEXT:    pmovzxwd {{.*#+}} xmm11 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
49; SSE-NEXT:    pslld $24, %xmm11
50; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
51; SSE-NEXT:    pslld $24, %xmm3
52; SSE-NEXT:    pcmpeqw %xmm1, %xmm2
53; SSE-NEXT:    pmovzxwd {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
54; SSE-NEXT:    pslld $24, %xmm9
55; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
56; SSE-NEXT:    pslld $24, %xmm2
57; SSE-NEXT:    pcmpeqw %xmm1, %xmm15
58; SSE-NEXT:    pmovzxwd {{.*#+}} xmm8 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
59; SSE-NEXT:    pslld $24, %xmm8
60; SSE-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
61; SSE-NEXT:    pslld $24, %xmm15
62; SSE-NEXT:    movdqu 16(%rdi,%rcx,4), %xmm6
63; SSE-NEXT:    movdqa %xmm6, %xmm4
64; SSE-NEXT:    pslld %xmm14, %xmm4
65; SSE-NEXT:    pslld %xmm13, %xmm6
66; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm6
67; SSE-NEXT:    movdqu (%rdi,%rcx,4), %xmm10
68; SSE-NEXT:    movdqa %xmm10, %xmm4
69; SSE-NEXT:    pslld %xmm14, %xmm4
70; SSE-NEXT:    pslld %xmm13, %xmm10
71; SSE-NEXT:    movdqa %xmm12, %xmm0
72; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm10
73; SSE-NEXT:    movdqu 48(%rdi,%rcx,4), %xmm12
74; SSE-NEXT:    movdqa %xmm12, %xmm5
75; SSE-NEXT:    pslld %xmm14, %xmm5
76; SSE-NEXT:    pslld %xmm13, %xmm12
77; SSE-NEXT:    movdqa %xmm3, %xmm0
78; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm12
79; SSE-NEXT:    movdqu 32(%rdi,%rcx,4), %xmm3
80; SSE-NEXT:    movdqa %xmm3, %xmm5
81; SSE-NEXT:    pslld %xmm14, %xmm5
82; SSE-NEXT:    pslld %xmm13, %xmm3
83; SSE-NEXT:    movdqa %xmm11, %xmm0
84; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm3
85; SSE-NEXT:    movdqu 80(%rdi,%rcx,4), %xmm1
86; SSE-NEXT:    movdqa %xmm1, %xmm5
87; SSE-NEXT:    pslld %xmm14, %xmm5
88; SSE-NEXT:    pslld %xmm13, %xmm1
89; SSE-NEXT:    movdqa %xmm2, %xmm0
90; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm1
91; SSE-NEXT:    movdqu 64(%rdi,%rcx,4), %xmm2
92; SSE-NEXT:    movdqa %xmm2, %xmm5
93; SSE-NEXT:    pslld %xmm14, %xmm5
94; SSE-NEXT:    pslld %xmm13, %xmm2
95; SSE-NEXT:    movdqa %xmm9, %xmm0
96; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm2
97; SSE-NEXT:    movdqu 112(%rdi,%rcx,4), %xmm5
98; SSE-NEXT:    movdqa %xmm5, %xmm7
99; SSE-NEXT:    pslld %xmm14, %xmm7
100; SSE-NEXT:    pslld %xmm13, %xmm5
101; SSE-NEXT:    movdqa %xmm15, %xmm0
102; SSE-NEXT:    blendvps %xmm0, %xmm7, %xmm5
103; SSE-NEXT:    movdqu 96(%rdi,%rcx,4), %xmm7
104; SSE-NEXT:    movdqa %xmm7, %xmm4
105; SSE-NEXT:    pslld %xmm14, %xmm4
106; SSE-NEXT:    pslld %xmm13, %xmm7
107; SSE-NEXT:    movdqa %xmm8, %xmm0
108; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm7
109; SSE-NEXT:    movups %xmm10, (%rdi,%rcx,4)
110; SSE-NEXT:    movups %xmm6, 16(%rdi,%rcx,4)
111; SSE-NEXT:    movups %xmm3, 32(%rdi,%rcx,4)
112; SSE-NEXT:    movups %xmm12, 48(%rdi,%rcx,4)
113; SSE-NEXT:    movups %xmm2, 64(%rdi,%rcx,4)
114; SSE-NEXT:    movups %xmm1, 80(%rdi,%rcx,4)
115; SSE-NEXT:    movups %xmm7, 96(%rdi,%rcx,4)
116; SSE-NEXT:    movups %xmm5, 112(%rdi,%rcx,4)
117; SSE-NEXT:    addq $32, %rcx
118; SSE-NEXT:    cmpq %rcx, %rdx
119; SSE-NEXT:    jne .LBB0_4
120; SSE-NEXT:  # %bb.5: # %middle.block
121; SSE-NEXT:    cmpq %rax, %rdx
122; SSE-NEXT:    jne .LBB0_6
123; SSE-NEXT:  .LBB0_9: # %for.cond.cleanup
124; SSE-NEXT:    retq
125; SSE-NEXT:    .p2align 4, 0x90
126; SSE-NEXT:  .LBB0_8: # %for.body
127; SSE-NEXT:    # in Loop: Header=BB0_6 Depth=1
128; SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
129; SSE-NEXT:    shll %cl, (%rdi,%rdx,4)
130; SSE-NEXT:    incq %rdx
131; SSE-NEXT:    cmpq %rdx, %rax
132; SSE-NEXT:    je .LBB0_9
133; SSE-NEXT:  .LBB0_6: # %for.body
134; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
135; SSE-NEXT:    cmpb $0, (%rsi,%rdx)
136; SSE-NEXT:    movl %r9d, %ecx
137; SSE-NEXT:    je .LBB0_8
138; SSE-NEXT:  # %bb.7: # %for.body
139; SSE-NEXT:    # in Loop: Header=BB0_6 Depth=1
140; SSE-NEXT:    movl %r8d, %ecx
141; SSE-NEXT:    jmp .LBB0_8
142;
143; AVX1-LABEL: vector_variable_shift_left_loop:
144; AVX1:       # %bb.0: # %entry
145; AVX1-NEXT:    subq $24, %rsp
146; AVX1-NEXT:    testl %edx, %edx
147; AVX1-NEXT:    jle .LBB0_9
148; AVX1-NEXT:  # %bb.1: # %for.body.preheader
149; AVX1-NEXT:    movl %ecx, %r9d
150; AVX1-NEXT:    movl %edx, %eax
151; AVX1-NEXT:    cmpl $31, %edx
152; AVX1-NEXT:    ja .LBB0_3
153; AVX1-NEXT:  # %bb.2:
154; AVX1-NEXT:    xorl %edx, %edx
155; AVX1-NEXT:    jmp .LBB0_6
156; AVX1-NEXT:  .LBB0_3: # %vector.ph
157; AVX1-NEXT:    movl %eax, %edx
158; AVX1-NEXT:    andl $-32, %edx
159; AVX1-NEXT:    vmovd %r9d, %xmm0
160; AVX1-NEXT:    vmovd %r8d, %xmm1
161; AVX1-NEXT:    xorl %ecx, %ecx
162; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
163; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
164; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
165; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
166; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
167; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
168; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm0[0],zero,xmm0[1],zero
169; AVX1-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
170; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm1[0],zero,xmm1[1],zero
171; AVX1-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
172; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm0[0],zero,xmm0[1],zero
173; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
174; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
175; AVX1-NEXT:    .p2align 4, 0x90
176; AVX1-NEXT:  .LBB0_4: # %vector.body
177; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
178; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
179; AVX1-NEXT:    # xmm1 = mem[0],zero,mem[1],zero
180; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
181; AVX1-NEXT:    # xmm2 = mem[0],zero,mem[1],zero
182; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
183; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
184; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
185; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
186; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm3, %xmm3
187; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm7
188; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
189; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
190; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm4, %xmm4
191; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm8
192; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
193; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm4
194; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm5, %xmm5
195; AVX1-NEXT:    vmovdqu (%rdi,%rcx,4), %xmm9
196; AVX1-NEXT:    vpslld %xmm2, %xmm9, %xmm10
197; AVX1-NEXT:    vpslld %xmm1, %xmm9, %xmm0
198; AVX1-NEXT:    vblendvps %xmm7, %xmm10, %xmm0, %xmm9
199; AVX1-NEXT:    vpmovsxwd %xmm5, %xmm7
200; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
201; AVX1-NEXT:    vpmovsxwd %xmm5, %xmm5
202; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm6, %xmm6
203; AVX1-NEXT:    vmovdqu 16(%rdi,%rcx,4), %xmm0
204; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
205; AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
206; AVX1-NEXT:    vpmovsxwd %xmm6, %xmm1
207; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
208; AVX1-NEXT:    vpmovsxwd %xmm6, %xmm6
209; AVX1-NEXT:    vblendvps %xmm3, %xmm2, %xmm0, %xmm10
210; AVX1-NEXT:    vmovdqu 32(%rdi,%rcx,4), %xmm2
211; AVX1-NEXT:    vpslld %xmm15, %xmm2, %xmm3
212; AVX1-NEXT:    vpslld %xmm14, %xmm2, %xmm2
213; AVX1-NEXT:    vblendvps %xmm8, %xmm3, %xmm2, %xmm8
214; AVX1-NEXT:    vmovdqu 48(%rdi,%rcx,4), %xmm3
215; AVX1-NEXT:    vpslld %xmm15, %xmm3, %xmm0
216; AVX1-NEXT:    vpslld %xmm14, %xmm3, %xmm3
217; AVX1-NEXT:    vblendvps %xmm4, %xmm0, %xmm3, %xmm0
218; AVX1-NEXT:    vmovdqu 64(%rdi,%rcx,4), %xmm3
219; AVX1-NEXT:    vpslld %xmm13, %xmm3, %xmm4
220; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
221; AVX1-NEXT:    vpslld %xmm2, %xmm3, %xmm3
222; AVX1-NEXT:    vblendvps %xmm7, %xmm4, %xmm3, %xmm3
223; AVX1-NEXT:    vmovdqu 80(%rdi,%rcx,4), %xmm4
224; AVX1-NEXT:    vpslld %xmm13, %xmm4, %xmm7
225; AVX1-NEXT:    vpslld %xmm2, %xmm4, %xmm4
226; AVX1-NEXT:    vblendvps %xmm5, %xmm7, %xmm4, %xmm4
227; AVX1-NEXT:    vmovdqu 96(%rdi,%rcx,4), %xmm5
228; AVX1-NEXT:    vpslld %xmm12, %xmm5, %xmm7
229; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
230; AVX1-NEXT:    vpslld %xmm2, %xmm5, %xmm5
231; AVX1-NEXT:    vblendvps %xmm1, %xmm7, %xmm5, %xmm1
232; AVX1-NEXT:    vmovdqu 112(%rdi,%rcx,4), %xmm5
233; AVX1-NEXT:    vpslld %xmm12, %xmm5, %xmm7
234; AVX1-NEXT:    vpslld %xmm2, %xmm5, %xmm5
235; AVX1-NEXT:    vblendvps %xmm6, %xmm7, %xmm5, %xmm5
236; AVX1-NEXT:    vmovups %xmm9, (%rdi,%rcx,4)
237; AVX1-NEXT:    vmovups %xmm10, 16(%rdi,%rcx,4)
238; AVX1-NEXT:    vmovups %xmm8, 32(%rdi,%rcx,4)
239; AVX1-NEXT:    vmovups %xmm0, 48(%rdi,%rcx,4)
240; AVX1-NEXT:    vmovups %xmm3, 64(%rdi,%rcx,4)
241; AVX1-NEXT:    vmovups %xmm4, 80(%rdi,%rcx,4)
242; AVX1-NEXT:    vmovups %xmm1, 96(%rdi,%rcx,4)
243; AVX1-NEXT:    vmovups %xmm5, 112(%rdi,%rcx,4)
244; AVX1-NEXT:    addq $32, %rcx
245; AVX1-NEXT:    cmpq %rcx, %rdx
246; AVX1-NEXT:    jne .LBB0_4
247; AVX1-NEXT:  # %bb.5: # %middle.block
248; AVX1-NEXT:    cmpq %rax, %rdx
249; AVX1-NEXT:    jne .LBB0_6
250; AVX1-NEXT:  .LBB0_9: # %for.cond.cleanup
251; AVX1-NEXT:    addq $24, %rsp
252; AVX1-NEXT:    vzeroupper
253; AVX1-NEXT:    retq
254; AVX1-NEXT:    .p2align 4, 0x90
255; AVX1-NEXT:  .LBB0_8: # %for.body
256; AVX1-NEXT:    # in Loop: Header=BB0_6 Depth=1
257; AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
258; AVX1-NEXT:    shll %cl, (%rdi,%rdx,4)
259; AVX1-NEXT:    incq %rdx
260; AVX1-NEXT:    cmpq %rdx, %rax
261; AVX1-NEXT:    je .LBB0_9
262; AVX1-NEXT:  .LBB0_6: # %for.body
263; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
264; AVX1-NEXT:    cmpb $0, (%rsi,%rdx)
265; AVX1-NEXT:    movl %r9d, %ecx
266; AVX1-NEXT:    je .LBB0_8
267; AVX1-NEXT:  # %bb.7: # %for.body
268; AVX1-NEXT:    # in Loop: Header=BB0_6 Depth=1
269; AVX1-NEXT:    movl %r8d, %ecx
270; AVX1-NEXT:    jmp .LBB0_8
271;
272; AVX2-LABEL: vector_variable_shift_left_loop:
273; AVX2:       # %bb.0: # %entry
274; AVX2-NEXT:    testl %edx, %edx
275; AVX2-NEXT:    jle .LBB0_9
276; AVX2-NEXT:  # %bb.1: # %for.body.preheader
277; AVX2-NEXT:    movl %ecx, %r9d
278; AVX2-NEXT:    movl %edx, %eax
279; AVX2-NEXT:    cmpl $31, %edx
280; AVX2-NEXT:    ja .LBB0_3
281; AVX2-NEXT:  # %bb.2:
282; AVX2-NEXT:    xorl %edx, %edx
283; AVX2-NEXT:    jmp .LBB0_6
284; AVX2-NEXT:  .LBB0_3: # %vector.ph
285; AVX2-NEXT:    movl %eax, %edx
286; AVX2-NEXT:    andl $-32, %edx
287; AVX2-NEXT:    vmovd %r9d, %xmm0
288; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
289; AVX2-NEXT:    vmovd %r8d, %xmm1
290; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
291; AVX2-NEXT:    xorl %ecx, %ecx
292; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
293; AVX2-NEXT:    .p2align 4, 0x90
294; AVX2-NEXT:  .LBB0_4: # %vector.body
295; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
296; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
297; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
298; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
299; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
300; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm3, %ymm3
301; AVX2-NEXT:    vblendvps %ymm3, %ymm0, %ymm1, %ymm3
302; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm4, %ymm4
303; AVX2-NEXT:    vblendvps %ymm4, %ymm0, %ymm1, %ymm4
304; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm5
305; AVX2-NEXT:    vblendvps %ymm5, %ymm0, %ymm1, %ymm5
306; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm6, %ymm6
307; AVX2-NEXT:    vblendvps %ymm6, %ymm0, %ymm1, %ymm6
308; AVX2-NEXT:    vmovdqu (%rdi,%rcx,4), %ymm7
309; AVX2-NEXT:    vpsllvd %ymm3, %ymm7, %ymm3
310; AVX2-NEXT:    vmovdqu 32(%rdi,%rcx,4), %ymm7
311; AVX2-NEXT:    vpsllvd %ymm4, %ymm7, %ymm4
312; AVX2-NEXT:    vmovdqu 64(%rdi,%rcx,4), %ymm7
313; AVX2-NEXT:    vpsllvd %ymm5, %ymm7, %ymm5
314; AVX2-NEXT:    vmovdqu 96(%rdi,%rcx,4), %ymm7
315; AVX2-NEXT:    vpsllvd %ymm6, %ymm7, %ymm6
316; AVX2-NEXT:    vmovdqu %ymm3, (%rdi,%rcx,4)
317; AVX2-NEXT:    vmovdqu %ymm4, 32(%rdi,%rcx,4)
318; AVX2-NEXT:    vmovdqu %ymm5, 64(%rdi,%rcx,4)
319; AVX2-NEXT:    vmovdqu %ymm6, 96(%rdi,%rcx,4)
320; AVX2-NEXT:    addq $32, %rcx
321; AVX2-NEXT:    cmpq %rcx, %rdx
322; AVX2-NEXT:    jne .LBB0_4
323; AVX2-NEXT:  # %bb.5: # %middle.block
324; AVX2-NEXT:    cmpq %rax, %rdx
325; AVX2-NEXT:    jne .LBB0_6
326; AVX2-NEXT:  .LBB0_9: # %for.cond.cleanup
327; AVX2-NEXT:    vzeroupper
328; AVX2-NEXT:    retq
329; AVX2-NEXT:    .p2align 4, 0x90
330; AVX2-NEXT:  .LBB0_8: # %for.body
331; AVX2-NEXT:    # in Loop: Header=BB0_6 Depth=1
332; AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
333; AVX2-NEXT:    shll %cl, (%rdi,%rdx,4)
334; AVX2-NEXT:    incq %rdx
335; AVX2-NEXT:    cmpq %rdx, %rax
336; AVX2-NEXT:    je .LBB0_9
337; AVX2-NEXT:  .LBB0_6: # %for.body
338; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
339; AVX2-NEXT:    cmpb $0, (%rsi,%rdx)
340; AVX2-NEXT:    movl %r9d, %ecx
341; AVX2-NEXT:    je .LBB0_8
342; AVX2-NEXT:  # %bb.7: # %for.body
343; AVX2-NEXT:    # in Loop: Header=BB0_6 Depth=1
344; AVX2-NEXT:    movl %r8d, %ecx
345; AVX2-NEXT:    jmp .LBB0_8
346entry:
347  %cmp12 = icmp sgt i32 %count, 0
348  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
349
350for.body.preheader:
351  %wide.trip.count = zext i32 %count to i64
352  %min.iters.check = icmp ult i32 %count, 32
353  br i1 %min.iters.check, label %for.body.preheader40, label %vector.ph
354
355for.body.preheader40:
356  %indvars.iv.ph = phi i64 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
357  br label %for.body
358
359vector.ph:
360  %n.vec = and i64 %wide.trip.count, 4294967264
361  %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %amt0, i32 0
362  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer
363  %broadcast.splatinsert22 = insertelement <8 x i32> undef, i32 %amt1, i32 0
364  %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> undef, <8 x i32> zeroinitializer
365  %broadcast.splatinsert24 = insertelement <8 x i32> undef, i32 %amt0, i32 0
366  %broadcast.splat25 = shufflevector <8 x i32> %broadcast.splatinsert24, <8 x i32> undef, <8 x i32> zeroinitializer
367  %broadcast.splatinsert26 = insertelement <8 x i32> undef, i32 %amt1, i32 0
368  %broadcast.splat27 = shufflevector <8 x i32> %broadcast.splatinsert26, <8 x i32> undef, <8 x i32> zeroinitializer
369  %broadcast.splatinsert28 = insertelement <8 x i32> undef, i32 %amt0, i32 0
370  %broadcast.splat29 = shufflevector <8 x i32> %broadcast.splatinsert28, <8 x i32> undef, <8 x i32> zeroinitializer
371  %broadcast.splatinsert30 = insertelement <8 x i32> undef, i32 %amt1, i32 0
372  %broadcast.splat31 = shufflevector <8 x i32> %broadcast.splatinsert30, <8 x i32> undef, <8 x i32> zeroinitializer
373  %broadcast.splatinsert32 = insertelement <8 x i32> undef, i32 %amt0, i32 0
374  %broadcast.splat33 = shufflevector <8 x i32> %broadcast.splatinsert32, <8 x i32> undef, <8 x i32> zeroinitializer
375  %broadcast.splatinsert34 = insertelement <8 x i32> undef, i32 %amt1, i32 0
376  %broadcast.splat35 = shufflevector <8 x i32> %broadcast.splatinsert34, <8 x i32> undef, <8 x i32> zeroinitializer
377  br label %vector.body
378
379vector.body:
380  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
381  %0 = getelementptr inbounds i8, i8* %control, i64 %index
382  %1 = bitcast i8* %0 to <8 x i8>*
383  %wide.load = load <8 x i8>, <8 x i8>* %1, align 1
384  %2 = getelementptr inbounds i8, i8* %0, i64 8
385  %3 = bitcast i8* %2 to <8 x i8>*
386  %wide.load17 = load <8 x i8>, <8 x i8>* %3, align 1
387  %4 = getelementptr inbounds i8, i8* %0, i64 16
388  %5 = bitcast i8* %4 to <8 x i8>*
389  %wide.load18 = load <8 x i8>, <8 x i8>* %5, align 1
390  %6 = getelementptr inbounds i8, i8* %0, i64 24
391  %7 = bitcast i8* %6 to <8 x i8>*
392  %wide.load19 = load <8 x i8>, <8 x i8>* %7, align 1
393  %8 = icmp eq <8 x i8> %wide.load, zeroinitializer
394  %9 = icmp eq <8 x i8> %wide.load17, zeroinitializer
395  %10 = icmp eq <8 x i8> %wide.load18, zeroinitializer
396  %11 = icmp eq <8 x i8> %wide.load19, zeroinitializer
397  %12 = select <8 x i1> %8, <8 x i32> %broadcast.splat21, <8 x i32> %broadcast.splat23
398  %13 = select <8 x i1> %9, <8 x i32> %broadcast.splat25, <8 x i32> %broadcast.splat27
399  %14 = select <8 x i1> %10, <8 x i32> %broadcast.splat29, <8 x i32> %broadcast.splat31
400  %15 = select <8 x i1> %11, <8 x i32> %broadcast.splat33, <8 x i32> %broadcast.splat35
401  %16 = getelementptr inbounds i32, i32* %arr, i64 %index
402  %17 = bitcast i32* %16 to <8 x i32>*
403  %wide.load36 = load <8 x i32>, <8 x i32>* %17, align 4
404  %18 = getelementptr inbounds i32, i32* %16, i64 8
405  %19 = bitcast i32* %18 to <8 x i32>*
406  %wide.load37 = load <8 x i32>, <8 x i32>* %19, align 4
407  %20 = getelementptr inbounds i32, i32* %16, i64 16
408  %21 = bitcast i32* %20 to <8 x i32>*
409  %wide.load38 = load <8 x i32>, <8 x i32>* %21, align 4
410  %22 = getelementptr inbounds i32, i32* %16, i64 24
411  %23 = bitcast i32* %22 to <8 x i32>*
412  %wide.load39 = load <8 x i32>, <8 x i32>* %23, align 4
413  %24 = shl <8 x i32> %wide.load36, %12
414  %25 = shl <8 x i32> %wide.load37, %13
415  %26 = shl <8 x i32> %wide.load38, %14
416  %27 = shl <8 x i32> %wide.load39, %15
417  %28 = bitcast i32* %16 to <8 x i32>*
418  store <8 x i32> %24, <8 x i32>* %28, align 4
419  %29 = bitcast i32* %18 to <8 x i32>*
420  store <8 x i32> %25, <8 x i32>* %29, align 4
421  %30 = bitcast i32* %20 to <8 x i32>*
422  store <8 x i32> %26, <8 x i32>* %30, align 4
423  %31 = bitcast i32* %22 to <8 x i32>*
424  store <8 x i32> %27, <8 x i32>* %31, align 4
425  %index.next = add i64 %index, 32
426  %32 = icmp eq i64 %index.next, %n.vec
427  br i1 %32, label %middle.block, label %vector.body
428
429middle.block:
430  %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
431  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader40
432
433for.cond.cleanup:
434  ret void
435
436for.body:
437  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader40 ]
438  %arrayidx = getelementptr inbounds i8, i8* %control, i64 %indvars.iv
439  %33 = load i8, i8* %arrayidx, align 1
440  %tobool = icmp eq i8 %33, 0
441  %cond = select i1 %tobool, i32 %amt0, i32 %amt1
442  %arrayidx2 = getelementptr inbounds i32, i32* %arr, i64 %indvars.iv
443  %34 = load i32, i32* %arrayidx2, align 4
444  %shl = shl i32 %34, %cond
445  store i32 %shl, i32* %arrayidx2, align 4
446  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
447  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
448  br i1 %exitcond, label %for.cond.cleanup, label %for.body
449}
450
451define void @vector_variable_shift_left_loop_simpler(i32* nocapture %arr, i8* nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1, i32 %x) nounwind {
452; SSE-LABEL: vector_variable_shift_left_loop_simpler:
453; SSE:       # %bb.0: # %entry
454; SSE-NEXT:    testl %edx, %edx
455; SSE-NEXT:    jle .LBB1_3
456; SSE-NEXT:  # %bb.1: # %vector.ph
457; SSE-NEXT:    movl %edx, %eax
458; SSE-NEXT:    andl $-4, %eax
459; SSE-NEXT:    movd %ecx, %xmm0
460; SSE-NEXT:    movd %r8d, %xmm2
461; SSE-NEXT:    movd %r9d, %xmm3
462; SSE-NEXT:    xorl %ecx, %ecx
463; SSE-NEXT:    pslld $23, %xmm0
464; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
465; SSE-NEXT:    paddd %xmm4, %xmm0
466; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
467; SSE-NEXT:    pmulld %xmm3, %xmm0
468; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
469; SSE-NEXT:    pslld $23, %xmm2
470; SSE-NEXT:    paddd %xmm4, %xmm2
471; SSE-NEXT:    cvttps2dq %xmm2, %xmm0
472; SSE-NEXT:    pmulld %xmm3, %xmm0
473; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
474; SSE-NEXT:    pxor %xmm3, %xmm3
475; SSE-NEXT:    .p2align 4, 0x90
476; SSE-NEXT:  .LBB1_2: # %vector.body
477; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
478; SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
479; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
480; SSE-NEXT:    movdqa %xmm2, %xmm4
481; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm4
482; SSE-NEXT:    movups %xmm4, (%rdi,%rcx,4)
483; SSE-NEXT:    addq $4, %rcx
484; SSE-NEXT:    cmpq %rcx, %rax
485; SSE-NEXT:    jne .LBB1_2
486; SSE-NEXT:  .LBB1_3: # %exit
487; SSE-NEXT:    retq
488;
489; AVX1-LABEL: vector_variable_shift_left_loop_simpler:
490; AVX1:       # %bb.0: # %entry
491; AVX1-NEXT:    testl %edx, %edx
492; AVX1-NEXT:    jle .LBB1_3
493; AVX1-NEXT:  # %bb.1: # %vector.ph
494; AVX1-NEXT:    movl %edx, %eax
495; AVX1-NEXT:    andl $-4, %eax
496; AVX1-NEXT:    vmovd %ecx, %xmm0
497; AVX1-NEXT:    vmovd %r8d, %xmm1
498; AVX1-NEXT:    vmovd %r9d, %xmm2
499; AVX1-NEXT:    xorl %ecx, %ecx
500; AVX1-NEXT:    vpslld $23, %xmm0, %xmm0
501; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
502; AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
503; AVX1-NEXT:    vcvttps2dq %xmm0, %xmm0
504; AVX1-NEXT:    vpmulld %xmm0, %xmm2, %xmm0
505; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
506; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
507; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
508; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
509; AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
510; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
511; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
512; AVX1-NEXT:    .p2align 4, 0x90
513; AVX1-NEXT:  .LBB1_2: # %vector.body
514; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
515; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
516; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm3
517; AVX1-NEXT:    vblendvps %xmm3, %xmm0, %xmm1, %xmm3
518; AVX1-NEXT:    vmovups %xmm3, (%rdi,%rcx,4)
519; AVX1-NEXT:    addq $4, %rcx
520; AVX1-NEXT:    cmpq %rcx, %rax
521; AVX1-NEXT:    jne .LBB1_2
522; AVX1-NEXT:  .LBB1_3: # %exit
523; AVX1-NEXT:    retq
524;
525; AVX2-LABEL: vector_variable_shift_left_loop_simpler:
526; AVX2:       # %bb.0: # %entry
527; AVX2-NEXT:    testl %edx, %edx
528; AVX2-NEXT:    jle .LBB1_3
529; AVX2-NEXT:  # %bb.1: # %vector.ph
530; AVX2-NEXT:    movl %edx, %eax
531; AVX2-NEXT:    andl $-4, %eax
532; AVX2-NEXT:    vmovd %ecx, %xmm0
533; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
534; AVX2-NEXT:    vmovd %r8d, %xmm1
535; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
536; AVX2-NEXT:    vmovd %r9d, %xmm2
537; AVX2-NEXT:    vpbroadcastd %xmm2, %xmm2
538; AVX2-NEXT:    xorl %ecx, %ecx
539; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
540; AVX2-NEXT:    .p2align 4, 0x90
541; AVX2-NEXT:  .LBB1_2: # %vector.body
542; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
543; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
544; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
545; AVX2-NEXT:    vblendvps %xmm4, %xmm0, %xmm1, %xmm4
546; AVX2-NEXT:    vpsllvd %xmm4, %xmm2, %xmm4
547; AVX2-NEXT:    vmovdqu %xmm4, (%rdi,%rcx,4)
548; AVX2-NEXT:    addq $4, %rcx
549; AVX2-NEXT:    cmpq %rcx, %rax
550; AVX2-NEXT:    jne .LBB1_2
551; AVX2-NEXT:  .LBB1_3: # %exit
552; AVX2-NEXT:    retq
553entry:
554  %cmp16 = icmp sgt i32 %count, 0
555  %wide.trip.count = zext i32 %count to i64
556  br i1 %cmp16, label %vector.ph, label %exit
557
558vector.ph:
559  %n.vec = and i64 %wide.trip.count, 4294967292
560  %splatinsert18 = insertelement <4 x i32> undef, i32 %amt0, i32 0
561  %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer
562  %splatinsert20 = insertelement <4 x i32> undef, i32 %amt1, i32 0
563  %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
564  %splatinsert22 = insertelement <4 x i32> undef, i32 %x, i32 0
565  %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
566  br label %vector.body
567
568vector.body:
569  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
570  %0 = getelementptr inbounds i8, i8* %control, i64 %index
571  %1 = bitcast i8* %0 to <4 x i8>*
572  %wide.load = load <4 x i8>, <4 x i8>* %1, align 1
573  %2 = icmp eq <4 x i8> %wide.load, zeroinitializer
574  %3 = select <4 x i1> %2, <4 x i32> %splat1, <4 x i32> %splat2
575  %4 = shl <4 x i32> %splat3, %3
576  %5 = getelementptr inbounds i32, i32* %arr, i64 %index
577  %6 = bitcast i32* %5 to <4 x i32>*
578  store <4 x i32> %4, <4 x i32>* %6, align 4
579  %index.next = add i64 %index, 4
580  %7 = icmp eq i64 %index.next, %n.vec
581  br i1 %7, label %exit, label %vector.body
582
583exit:
584  ret void
585}
586