1; RUN: opt < %s -licm -loop-vectorize -force-vector-width=4 -dce -instcombine -licm -S | FileCheck %s
2
3; First licm pass is to hoist/sink invariant stores if possible. Today LICM does
4; not hoist/sink the invariant stores. Even if that changes, we should still
5; vectorize this loop in case licm is not run.
6
7; The next licm pass after vectorization is to hoist/sink loop invariant
8; instructions.
9target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
10
11; all tests check that it is legal to vectorize the stores to invariant
12; address.
13
14
15; CHECK-LABEL: inv_val_store_to_inv_address_with_reduction(
16; memory check is found.conflict = b[max(n-1,1)] > a && (i8* a)+1 > (i8* b)
17; CHECK: vector.memcheck:
18; CHECK:    found.conflict
19
20; CHECK-LABEL: vector.body:
21; CHECK:         %vec.phi = phi <4 x i32>  [ zeroinitializer, %vector.ph ], [ [[ADD:%[a-zA-Z0-9.]+]], %vector.body ]
22; CHECK:         %wide.load = load <4 x i32>
23; CHECK:         [[ADD]] = add <4 x i32> %vec.phi, %wide.load
24; CHECK-NEXT:    store i32 %ntrunc, i32* %a
25; CHECK-NEXT:    %index.next = add i64 %index, 4
26; CHECK-NEXT:    icmp eq i64 %index.next, %n.vec
27; CHECK-NEXT:    br i1
28
29; CHECK-LABEL: middle.block:
30; CHECK:         %rdx.shuf = shufflevector <4 x i32>
31define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b) {
32entry:
33  %ntrunc = trunc i64 %n to i32
34  br label %for.body
35
36for.body:                                         ; preds = %for.body, %entry
37  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
38  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
39  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
40  %tmp2 = load i32, i32* %tmp1, align 8
41  %tmp3 = add i32 %tmp0, %tmp2
42  store i32 %ntrunc, i32* %a
43  %i.next = add nuw nsw i64 %i, 1
44  %cond = icmp slt i64 %i.next, %n
45  br i1 %cond, label %for.body, label %for.end
46
47for.end:                                          ; preds = %for.body
48  %tmp4 = phi i32 [ %tmp3, %for.body ]
49  ret i32 %tmp4
50}
51
52; CHECK-LABEL: inv_val_store_to_inv_address(
53; CHECK-LABEL: vector.body:
54; CHECK:         store i32 %ntrunc, i32* %a
55; CHECK:         store <4 x i32>
56; CHECK-NEXT:    %index.next = add i64 %index, 4
57; CHECK-NEXT:    icmp eq i64 %index.next, %n.vec
58; CHECK-NEXT:    br i1
59define void @inv_val_store_to_inv_address(i32* %a, i64 %n, i32* %b) {
60entry:
61  %ntrunc = trunc i64 %n to i32
62  br label %for.body
63
64for.body:                                         ; preds = %for.body, %entry
65  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
66  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
67  %tmp2 = load i32, i32* %tmp1, align 8
68  store i32 %ntrunc, i32* %a
69  store i32 %ntrunc, i32* %tmp1
70  %i.next = add nuw nsw i64 %i, 1
71  %cond = icmp slt i64 %i.next, %n
72  br i1 %cond, label %for.body, label %for.end
73
74for.end:                                          ; preds = %for.body
75  ret void
76}
77
78
79; Both of these tests below are handled as predicated stores.
80
81; Conditional store
82; if (b[i] == k) a = ntrunc
83; TODO: We can be better with the code gen for the first test and we can have
84; just one scalar store if vector.or.reduce(vector_cmp(b[i] == k)) is 1.
85
86; CHECK-LABEL:inv_val_store_to_inv_address_conditional(
87; CHECK-LABEL: vector.body:
88; CHECK:           %wide.load = load <4 x i32>, <4 x i32>*
89; CHECK:           [[CMP:%[a-zA-Z0-9.]+]] = icmp eq <4 x i32> %wide.load, %{{.*}}
90; CHECK:           store <4 x i32>
91; CHECK-NEXT:      [[EE:%[a-zA-Z0-9.]+]] =  extractelement <4 x i1> [[CMP]], i32 0
92; CHECK-NEXT:      br i1 [[EE]], label %pred.store.if, label %pred.store.continue
93
94; CHECK-LABEL: pred.store.if:
95; CHECK-NEXT:      store i32 %ntrunc, i32* %a
96; CHECK-NEXT:      br label %pred.store.continue
97
98; CHECK-LABEL: pred.store.continue:
99; CHECK-NEXT:      [[EE1:%[a-zA-Z0-9.]+]] =  extractelement <4 x i1> [[CMP]], i32 1
100define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
101entry:
102  %ntrunc = trunc i64 %n to i32
103  br label %for.body
104
105for.body:                                         ; preds = %for.body, %entry
106  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
107  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
108  %tmp2 = load i32, i32* %tmp1, align 8
109  %cmp = icmp eq i32 %tmp2, %k
110  store i32 %ntrunc, i32* %tmp1
111  br i1 %cmp, label %cond_store, label %latch
112
113cond_store:
114  store i32 %ntrunc, i32* %a
115  br label %latch
116
117latch:
118  %i.next = add nuw nsw i64 %i, 1
119  %cond = icmp slt i64 %i.next, %n
120  br i1 %cond, label %for.body, label %for.end
121
122for.end:                                          ; preds = %for.body
123  ret void
124}
125
126; if (b[i] == k)
127;    a = ntrunc
128; else a = k;
129; TODO: We could vectorize this once we support multiple uniform stores to the
130; same address.
131; CHECK-LABEL:inv_val_store_to_inv_address_conditional_diff_values(
132; CHECK-NOT:           load <4 x i32>
133define void @inv_val_store_to_inv_address_conditional_diff_values(i32* %a, i64 %n, i32* %b, i32 %k) {
134entry:
135  %ntrunc = trunc i64 %n to i32
136  br label %for.body
137
138for.body:                                         ; preds = %for.body, %entry
139  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
140  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
141  %tmp2 = load i32, i32* %tmp1, align 8
142  %cmp = icmp eq i32 %tmp2, %k
143  store i32 %ntrunc, i32* %tmp1
144  br i1 %cmp, label %cond_store, label %cond_store_k
145
146cond_store:
147  store i32 %ntrunc, i32* %a
148  br label %latch
149
150cond_store_k:
151  store i32 %k, i32 * %a
152  br label %latch
153
154latch:
155  %i.next = add nuw nsw i64 %i, 1
156  %cond = icmp slt i64 %i.next, %n
157  br i1 %cond, label %for.body, label %for.end
158
159for.end:                                          ; preds = %for.body
160  ret void
161}
162
163; Instcombine'd version of above test. Now the store is no longer of invariant
164; value.
165; TODO: We should be able to vectorize this loop once we support vectorizing
166; stores of variant values to invariant addresses.
167; CHECK-LABEL: inv_val_store_to_inv_address_conditional_diff_values_ic
168; CHECK-NOT:   <4 x
169define void @inv_val_store_to_inv_address_conditional_diff_values_ic(i32* %a, i64 %n, i32* %b, i32 %k) {
170entry:
171  %ntrunc = trunc i64 %n to i32
172  br label %for.body
173
174for.body:                                         ; preds = %for.body, %entry
175  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
176  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
177  %tmp2 = load i32, i32* %tmp1, align 8
178  %cmp = icmp eq i32 %tmp2, %k
179  store i32 %ntrunc, i32* %tmp1
180  br i1 %cmp, label %cond_store, label %cond_store_k
181
182cond_store:
183  br label %latch
184
185cond_store_k:
186  br label %latch
187
188latch:
189  %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ]
190  store i32 %storeval, i32* %a
191  %i.next = add nuw nsw i64 %i, 1
192  %cond = icmp slt i64 %i.next, %n
193  br i1 %cond, label %for.body, label %for.end
194
195for.end:                                          ; preds = %for.body
196  ret void
197}
198
199; invariant val stored to invariant address predicated on invariant condition
200; This is not treated as a predicated store since the block the store belongs to
201; is the latch block (which doesn't need to be predicated).
202; TODO: We should vectorize this loop once we relax the check for
203; variant/invariant values being stored to invariant address.
204; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv
205; CHECK-NOT: <4 x
206define void @inv_val_store_to_inv_address_conditional_inv(i32* %a, i64 %n, i32* %b, i32 %k) {
207entry:
208  %ntrunc = trunc i64 %n to i32
209  %cmp = icmp eq i32 %ntrunc, %k
210  br label %for.body
211
212for.body:                                         ; preds = %for.body, %entry
213  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
214  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
215  %tmp2 = load i32, i32* %tmp1, align 8
216  store i32 %ntrunc, i32* %tmp1
217  br i1 %cmp, label %cond_store, label %cond_store_k
218
219cond_store:
220  br label %latch
221
222cond_store_k:
223  br label %latch
224
225latch:
226  %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ]
227  store i32 %storeval, i32* %a
228  %i.next = add nuw nsw i64 %i, 1
229  %cond = icmp slt i64 %i.next, %n
230  br i1 %cond, label %for.body, label %for.end
231
232for.end:                                          ; preds = %for.body
233  ret void
234}
235
236; TODO: This loop can be vectorized once we support variant value being
237; stored into invariant address.
238; CHECK-LABEL: variant_val_store_to_inv_address
239; CHECK-NOT: <4 x i32>
240define i32 @variant_val_store_to_inv_address(i32* %a, i64 %n, i32* %b, i32 %k) {
241entry:
242  %ntrunc = trunc i64 %n to i32
243  %cmp = icmp eq i32 %ntrunc, %k
244  br label %for.body
245
246for.body:                                         ; preds = %for.body, %entry
247  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
248  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
249  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
250  %tmp2 = load i32, i32* %tmp1, align 8
251  store i32 %tmp2, i32* %a
252  %tmp3 = add i32 %tmp0, %tmp2
253  %i.next = add nuw nsw i64 %i, 1
254  %cond = icmp slt i64 %i.next, %n
255  br i1 %cond, label %for.body, label %for.end
256
257for.end:                                          ; preds = %for.body
258  %rdx.lcssa = phi i32 [ %tmp0, %for.body ]
259  ret i32 %rdx.lcssa
260}
261