1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v -riscv-v-vector-bits-min=256 | FileCheck %s --check-prefixes=CHECK,V
3; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zve32f -riscv-v-vector-bits-min=256 | FileCheck %s --check-prefixes=CHECK,ZVE32F
4
5%struct.foo = type { i32, i32, i32, i32 }
6
7; void gather(signed char * __restrict  A, signed char * __restrict B) {
8;   for (int i = 0; i != 1024; ++i)
9;       A[i] += B[i * 5];
10; }
11define void @gather(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
12;
13; CHECK-LABEL: gather:
14; CHECK:       # %bb.0: # %entry
15; CHECK-NEXT:    li a2, 0
16; CHECK-NEXT:    li a3, 32
17; CHECK-NEXT:    li a4, 5
18; CHECK-NEXT:    li a5, 1024
19; CHECK-NEXT:  .LBB0_1: # %vector.body
20; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
21; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
22; CHECK-NEXT:    vlse8.v v8, (a1), a4
23; CHECK-NEXT:    add a6, a0, a2
24; CHECK-NEXT:    vle8.v v9, (a6)
25; CHECK-NEXT:    vadd.vv v8, v9, v8
26; CHECK-NEXT:    vse8.v v8, (a6)
27; CHECK-NEXT:    addi a2, a2, 32
28; CHECK-NEXT:    addi a1, a1, 160
29; CHECK-NEXT:    bne a2, a5, .LBB0_1
30; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
31; CHECK-NEXT:    ret
32entry:
33  br label %vector.body
34
35vector.body:                                      ; preds = %vector.body, %entry
36  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
37  %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]
38  %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
39  %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
40  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
41  %2 = getelementptr inbounds i8, i8* %A, i64 %index
42  %3 = bitcast i8* %2 to <32 x i8>*
43  %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
44  %4 = add <32 x i8> %wide.load, %wide.masked.gather
45  %5 = bitcast i8* %2 to <32 x i8>*
46  store <32 x i8> %4, <32 x i8>* %5, align 1
47  %index.next = add nuw i64 %index, 32
48  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
49  %6 = icmp eq i64 %index.next, 1024
50  br i1 %6, label %for.cond.cleanup, label %vector.body
51
52for.cond.cleanup:                                 ; preds = %vector.body
53  ret void
54}
55
56define void @gather_masked(i8* noalias nocapture %A, i8* noalias nocapture readonly %B, <32 x i8> %maskedoff) {
57;
58; V-LABEL: gather_masked:
59; V:       # %bb.0: # %entry
60; V-NEXT:    li a2, 0
61; V-NEXT:    lui a3, 983765
62; V-NEXT:    addiw a3, a3, 873
63; V-NEXT:    vsetivli zero, 1, e32, mf2, ta, mu
64; V-NEXT:    vmv.s.x v0, a3
65; V-NEXT:    li a3, 32
66; V-NEXT:    li a4, 5
67; V-NEXT:    li a5, 1024
68; V-NEXT:  .LBB1_1: # %vector.body
69; V-NEXT:    # =>This Inner Loop Header: Depth=1
70; V-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
71; V-NEXT:    vmv1r.v v9, v8
72; V-NEXT:    vlse8.v v9, (a1), a4, v0.t
73; V-NEXT:    add a6, a0, a2
74; V-NEXT:    vle8.v v10, (a6)
75; V-NEXT:    vadd.vv v9, v10, v9
76; V-NEXT:    vse8.v v9, (a6)
77; V-NEXT:    addi a2, a2, 32
78; V-NEXT:    addi a1, a1, 160
79; V-NEXT:    bne a2, a5, .LBB1_1
80; V-NEXT:  # %bb.2: # %for.cond.cleanup
81; V-NEXT:    ret
82;
83; ZVE32F-LABEL: gather_masked:
84; ZVE32F:       # %bb.0: # %entry
85; ZVE32F-NEXT:    li a2, 0
86; ZVE32F-NEXT:    lui a3, 983765
87; ZVE32F-NEXT:    addiw a3, a3, 873
88; ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
89; ZVE32F-NEXT:    vmv.s.x v0, a3
90; ZVE32F-NEXT:    li a3, 32
91; ZVE32F-NEXT:    li a4, 5
92; ZVE32F-NEXT:    li a5, 1024
93; ZVE32F-NEXT:  .LBB1_1: # %vector.body
94; ZVE32F-NEXT:    # =>This Inner Loop Header: Depth=1
95; ZVE32F-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
96; ZVE32F-NEXT:    vmv1r.v v9, v8
97; ZVE32F-NEXT:    vlse8.v v9, (a1), a4, v0.t
98; ZVE32F-NEXT:    add a6, a0, a2
99; ZVE32F-NEXT:    vle8.v v10, (a6)
100; ZVE32F-NEXT:    vadd.vv v9, v10, v9
101; ZVE32F-NEXT:    vse8.v v9, (a6)
102; ZVE32F-NEXT:    addi a2, a2, 32
103; ZVE32F-NEXT:    addi a1, a1, 160
104; ZVE32F-NEXT:    bne a2, a5, .LBB1_1
105; ZVE32F-NEXT:  # %bb.2: # %for.cond.cleanup
106; ZVE32F-NEXT:    ret
107entry:
108  br label %vector.body
109
110vector.body:                                      ; preds = %vector.body, %entry
111  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
112  %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]
113  %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
114  %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
115  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)
116  %2 = getelementptr inbounds i8, i8* %A, i64 %index
117  %3 = bitcast i8* %2 to <32 x i8>*
118  %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
119  %4 = add <32 x i8> %wide.load, %wide.masked.gather
120  %5 = bitcast i8* %2 to <32 x i8>*
121  store <32 x i8> %4, <32 x i8>* %5, align 1
122  %index.next = add nuw i64 %index, 32
123  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
124  %6 = icmp eq i64 %index.next, 1024
125  br i1 %6, label %for.cond.cleanup, label %vector.body
126
127for.cond.cleanup:                                 ; preds = %vector.body
128  ret void
129}
130
131define void @gather_negative_stride(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
132;
133; CHECK-LABEL: gather_negative_stride:
134; CHECK:       # %bb.0: # %entry
135; CHECK-NEXT:    li a2, 0
136; CHECK-NEXT:    addi a1, a1, 155
137; CHECK-NEXT:    li a3, 32
138; CHECK-NEXT:    li a4, -5
139; CHECK-NEXT:    li a5, 1024
140; CHECK-NEXT:  .LBB2_1: # %vector.body
141; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
142; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
143; CHECK-NEXT:    vlse8.v v8, (a1), a4
144; CHECK-NEXT:    add a6, a0, a2
145; CHECK-NEXT:    vle8.v v9, (a6)
146; CHECK-NEXT:    vadd.vv v8, v9, v8
147; CHECK-NEXT:    vse8.v v8, (a6)
148; CHECK-NEXT:    addi a2, a2, 32
149; CHECK-NEXT:    addi a1, a1, 160
150; CHECK-NEXT:    bne a2, a5, .LBB2_1
151; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
152; CHECK-NEXT:    ret
153entry:
154  br label %vector.body
155
156vector.body:                                      ; preds = %vector.body, %entry
157  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
158  %vec.ind = phi <32 x i64> [ <i64 31, i64 30, i64 29, i64 28, i64 27, i64 26, i64 25, i64 24, i64 23, i64 22, i64 21, i64 20, i64 19, i64 18, i64 17, i64 16, i64 15, i64 14, i64 13, i64 12, i64 11, i64 10, i64 9, i64 8, i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, %entry ], [ %vec.ind.next, %vector.body ]
159  %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
160  %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
161  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
162  %2 = getelementptr inbounds i8, i8* %A, i64 %index
163  %3 = bitcast i8* %2 to <32 x i8>*
164  %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
165  %4 = add <32 x i8> %wide.load, %wide.masked.gather
166  %5 = bitcast i8* %2 to <32 x i8>*
167  store <32 x i8> %4, <32 x i8>* %5, align 1
168  %index.next = add nuw i64 %index, 32
169  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
170  %6 = icmp eq i64 %index.next, 1024
171  br i1 %6, label %for.cond.cleanup, label %vector.body
172
173for.cond.cleanup:                                 ; preds = %vector.body
174  ret void
175}
176
177define void @gather_zero_stride(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
178;
179; CHECK-LABEL: gather_zero_stride:
180; CHECK:       # %bb.0: # %entry
181; CHECK-NEXT:    li a2, 0
182; CHECK-NEXT:    li a3, 32
183; CHECK-NEXT:    li a4, 1024
184; CHECK-NEXT:  .LBB3_1: # %vector.body
185; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
186; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
187; CHECK-NEXT:    vlse8.v v8, (a1), zero
188; CHECK-NEXT:    add a5, a0, a2
189; CHECK-NEXT:    vle8.v v9, (a5)
190; CHECK-NEXT:    vadd.vv v8, v9, v8
191; CHECK-NEXT:    vse8.v v8, (a5)
192; CHECK-NEXT:    addi a2, a2, 32
193; CHECK-NEXT:    addi a1, a1, 160
194; CHECK-NEXT:    bne a2, a4, .LBB3_1
195; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
196; CHECK-NEXT:    ret
197entry:
198  br label %vector.body
199
200vector.body:                                      ; preds = %vector.body, %entry
201  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
202  %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]
203  %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
204  %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
205  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
206  %2 = getelementptr inbounds i8, i8* %A, i64 %index
207  %3 = bitcast i8* %2 to <32 x i8>*
208  %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
209  %4 = add <32 x i8> %wide.load, %wide.masked.gather
210  %5 = bitcast i8* %2 to <32 x i8>*
211  store <32 x i8> %4, <32 x i8>* %5, align 1
212  %index.next = add nuw i64 %index, 32
213  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
214  %6 = icmp eq i64 %index.next, 1024
215  br i1 %6, label %for.cond.cleanup, label %vector.body
216
217for.cond.cleanup:                                 ; preds = %vector.body
218  ret void
219}
220
221;void scatter(signed char * __restrict  A, signed char * __restrict B) {
222;  for (int i = 0; i < 1024; ++i)
223;      A[i * 5] += B[i];
224;}
225define void @scatter(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
226;
227; CHECK-LABEL: scatter:
228; CHECK:       # %bb.0: # %entry
229; CHECK-NEXT:    li a2, 0
230; CHECK-NEXT:    li a3, 32
231; CHECK-NEXT:    li a4, 5
232; CHECK-NEXT:    li a5, 1024
233; CHECK-NEXT:  .LBB4_1: # %vector.body
234; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
235; CHECK-NEXT:    add a6, a1, a2
236; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
237; CHECK-NEXT:    vle8.v v8, (a6)
238; CHECK-NEXT:    vlse8.v v9, (a0), a4
239; CHECK-NEXT:    vadd.vv v8, v9, v8
240; CHECK-NEXT:    vsse8.v v8, (a0), a4
241; CHECK-NEXT:    addi a2, a2, 32
242; CHECK-NEXT:    addi a0, a0, 160
243; CHECK-NEXT:    bne a2, a5, .LBB4_1
244; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
245; CHECK-NEXT:    ret
246entry:
247  br label %vector.body
248
249vector.body:                                      ; preds = %vector.body, %entry
250  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
251  %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]
252  %0 = getelementptr inbounds i8, i8* %B, i64 %index
253  %1 = bitcast i8* %0 to <32 x i8>*
254  %wide.load = load <32 x i8>, <32 x i8>* %1, align 1
255  %2 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
256  %3 = getelementptr inbounds i8, i8* %A, <32 x i64> %2
257  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %3, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
258  %4 = add <32 x i8> %wide.masked.gather, %wide.load
259  call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> %4, <32 x i8*> %3, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
260  %index.next = add nuw i64 %index, 32
261  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
262  %5 = icmp eq i64 %index.next, 1024
263  br i1 %5, label %for.cond.cleanup, label %vector.body
264
265for.cond.cleanup:                                 ; preds = %vector.body
266  ret void
267}
268
269define void @scatter_masked(i8* noalias nocapture %A, i8* noalias nocapture readonly %B, <32 x i8> %maskedoff) {
270;
271; V-LABEL: scatter_masked:
272; V:       # %bb.0: # %entry
273; V-NEXT:    li a2, 0
274; V-NEXT:    li a3, 32
275; V-NEXT:    lui a4, 983765
276; V-NEXT:    addiw a4, a4, 873
277; V-NEXT:    vsetivli zero, 1, e32, mf2, ta, mu
278; V-NEXT:    vmv.s.x v0, a4
279; V-NEXT:    li a4, 5
280; V-NEXT:    li a5, 1024
281; V-NEXT:  .LBB5_1: # %vector.body
282; V-NEXT:    # =>This Inner Loop Header: Depth=1
283; V-NEXT:    add a6, a1, a2
284; V-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
285; V-NEXT:    vle8.v v9, (a6)
286; V-NEXT:    vmv1r.v v10, v8
287; V-NEXT:    vlse8.v v10, (a0), a4, v0.t
288; V-NEXT:    vadd.vv v9, v10, v9
289; V-NEXT:    vsse8.v v9, (a0), a4, v0.t
290; V-NEXT:    addi a2, a2, 32
291; V-NEXT:    addi a0, a0, 160
292; V-NEXT:    bne a2, a5, .LBB5_1
293; V-NEXT:  # %bb.2: # %for.cond.cleanup
294; V-NEXT:    ret
295;
296; ZVE32F-LABEL: scatter_masked:
297; ZVE32F:       # %bb.0: # %entry
298; ZVE32F-NEXT:    li a2, 0
299; ZVE32F-NEXT:    li a3, 32
300; ZVE32F-NEXT:    lui a4, 983765
301; ZVE32F-NEXT:    addiw a4, a4, 873
302; ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
303; ZVE32F-NEXT:    vmv.s.x v0, a4
304; ZVE32F-NEXT:    li a4, 5
305; ZVE32F-NEXT:    li a5, 1024
306; ZVE32F-NEXT:  .LBB5_1: # %vector.body
307; ZVE32F-NEXT:    # =>This Inner Loop Header: Depth=1
308; ZVE32F-NEXT:    add a6, a1, a2
309; ZVE32F-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
310; ZVE32F-NEXT:    vle8.v v9, (a6)
311; ZVE32F-NEXT:    vmv1r.v v10, v8
312; ZVE32F-NEXT:    vlse8.v v10, (a0), a4, v0.t
313; ZVE32F-NEXT:    vadd.vv v9, v10, v9
314; ZVE32F-NEXT:    vsse8.v v9, (a0), a4, v0.t
315; ZVE32F-NEXT:    addi a2, a2, 32
316; ZVE32F-NEXT:    addi a0, a0, 160
317; ZVE32F-NEXT:    bne a2, a5, .LBB5_1
318; ZVE32F-NEXT:  # %bb.2: # %for.cond.cleanup
319; ZVE32F-NEXT:    ret
320entry:
321  br label %vector.body
322
323vector.body:                                      ; preds = %vector.body, %entry
324  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
325  %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]
326  %0 = getelementptr inbounds i8, i8* %B, i64 %index
327  %1 = bitcast i8* %0 to <32 x i8>*
328  %wide.load = load <32 x i8>, <32 x i8>* %1, align 1
329  %2 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
330  %3 = getelementptr inbounds i8, i8* %A, <32 x i64> %2
331  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)
332  %4 = add <32 x i8> %wide.masked.gather, %wide.load
333  call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> %4, <32 x i8*> %3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
334  %index.next = add nuw i64 %index, 32
335  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
336  %5 = icmp eq i64 %index.next, 1024
337  br i1 %5, label %for.cond.cleanup, label %vector.body
338
339for.cond.cleanup:                                 ; preds = %vector.body
340  ret void
341}
342
343; void gather_pow2(signed char * __restrict  A, signed char * __restrict B) {
344;   for (int i = 0; i != 1024; ++i)
345;       A[i] += B[i * 4];
346; }
347define void @gather_pow2(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
348;
349; CHECK-LABEL: gather_pow2:
350; CHECK:       # %bb.0: # %entry
351; CHECK-NEXT:    li a2, 1024
352; CHECK-NEXT:    li a3, 16
353; CHECK-NEXT:    li a4, 32
354; CHECK-NEXT:  .LBB6_1: # %vector.body
355; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
356; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
357; CHECK-NEXT:    vlse32.v v8, (a1), a3
358; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, mu
359; CHECK-NEXT:    vle8.v v9, (a0)
360; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
361; CHECK-NEXT:    vadd.vv v8, v9, v8
362; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, mu
363; CHECK-NEXT:    vse8.v v8, (a0)
364; CHECK-NEXT:    addi a2, a2, -8
365; CHECK-NEXT:    addi a0, a0, 32
366; CHECK-NEXT:    addi a1, a1, 128
367; CHECK-NEXT:    bnez a2, .LBB6_1
368; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
369; CHECK-NEXT:    ret
370entry:
371  br label %vector.body
372
373vector.body:                                      ; preds = %vector.body, %entry
374  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
375  %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]
376  %0 = shl nsw <8 x i64> %vec.ind, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
377  %1 = getelementptr inbounds i32, i32* %B, <8 x i64> %0
378  %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %1, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
379  %2 = getelementptr inbounds i32, i32* %A, i64 %index
380  %3 = bitcast i32* %2 to <8 x i32>*
381  %wide.load = load <8 x i32>, <8 x i32>* %3, align 1
382  %4 = add <8 x i32> %wide.load, %wide.masked.gather
383  %5 = bitcast i32* %2 to <8 x i32>*
384  store <8 x i32> %4, <8 x i32>* %5, align 1
385  %index.next = add nuw i64 %index, 8
386  %vec.ind.next = add <8 x i64> %vec.ind, <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
387  %6 = icmp eq i64 %index.next, 1024
388  br i1 %6, label %for.cond.cleanup, label %vector.body
389
390for.cond.cleanup:                                 ; preds = %vector.body
391  ret void
392}
393
394;void scatter_pow2(signed char * __restrict  A, signed char * __restrict B) {
395;  for (int i = 0; i < 1024; ++i)
396;      A[i * 4] += B[i];
397;}
398define void @scatter_pow2(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
399;
400; CHECK-LABEL: scatter_pow2:
401; CHECK:       # %bb.0: # %entry
402; CHECK-NEXT:    li a2, 1024
403; CHECK-NEXT:    li a3, 32
404; CHECK-NEXT:    li a4, 16
405; CHECK-NEXT:  .LBB7_1: # %vector.body
406; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
407; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
408; CHECK-NEXT:    vle8.v v8, (a1)
409; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
410; CHECK-NEXT:    vlse32.v v9, (a0), a4
411; CHECK-NEXT:    vadd.vv v8, v9, v8
412; CHECK-NEXT:    vsse32.v v8, (a0), a4
413; CHECK-NEXT:    addi a2, a2, -8
414; CHECK-NEXT:    addi a1, a1, 32
415; CHECK-NEXT:    addi a0, a0, 128
416; CHECK-NEXT:    bnez a2, .LBB7_1
417; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
418; CHECK-NEXT:    ret
419entry:
420  br label %vector.body
421
422vector.body:                                      ; preds = %vector.body, %entry
423  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
424  %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]
425  %0 = getelementptr inbounds i32, i32* %B, i64 %index
426  %1 = bitcast i32* %0 to <8 x i32>*
427  %wide.load = load <8 x i32>, <8 x i32>* %1, align 1
428  %2 = shl nuw nsw <8 x i64> %vec.ind, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
429  %3 = getelementptr inbounds i32, i32* %A, <8 x i64> %2
430  %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %3, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
431  %4 = add <8 x i32> %wide.masked.gather, %wide.load
432  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %4, <8 x i32*> %3, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
433  %index.next = add nuw i64 %index, 8
434  %vec.ind.next = add <8 x i64> %vec.ind, <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
435  %5 = icmp eq i64 %index.next, 1024
436  br i1 %5, label %for.cond.cleanup, label %vector.body
437
438for.cond.cleanup:                                 ; preds = %vector.body
439  ret void
440}
441
442;struct foo {
443;  int a, b, c, d;
444;};
445;
446;void struct_gather(int * __restrict  A, struct foo * __restrict B) {
447;  for (int i = 0; i < 1024; ++i)
448;      A[i] += B[i].b;
449;}
450define void @struct_gather(i32* noalias nocapture %A, %struct.foo* noalias nocapture readonly %B) {
451;
452; CHECK-LABEL: struct_gather:
453; CHECK:       # %bb.0: # %entry
454; CHECK-NEXT:    addi a1, a1, 132
455; CHECK-NEXT:    li a2, 1024
456; CHECK-NEXT:    li a3, 16
457; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
458; CHECK-NEXT:  .LBB8_1: # %vector.body
459; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
460; CHECK-NEXT:    addi a4, a1, -128
461; CHECK-NEXT:    vlse32.v v8, (a4), a3
462; CHECK-NEXT:    vlse32.v v9, (a1), a3
463; CHECK-NEXT:    vle32.v v10, (a0)
464; CHECK-NEXT:    addi a4, a0, 32
465; CHECK-NEXT:    vle32.v v11, (a4)
466; CHECK-NEXT:    vadd.vv v8, v10, v8
467; CHECK-NEXT:    vadd.vv v9, v11, v9
468; CHECK-NEXT:    vse32.v v8, (a0)
469; CHECK-NEXT:    vse32.v v9, (a4)
470; CHECK-NEXT:    addi a2, a2, -16
471; CHECK-NEXT:    addi a0, a0, 64
472; CHECK-NEXT:    addi a1, a1, 256
473; CHECK-NEXT:    bnez a2, .LBB8_1
474; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
475; CHECK-NEXT:    ret
476entry:
477  br label %vector.body
478
479vector.body:                                      ; preds = %vector.body, %entry
480  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
481  %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]
482  %step.add = add <8 x i64> %vec.ind, <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
483  %0 = getelementptr inbounds %struct.foo, %struct.foo* %B, <8 x i64> %vec.ind, i32 1
484  %1 = getelementptr inbounds %struct.foo, %struct.foo* %B, <8 x i64> %step.add, i32 1
485  %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %0, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
486  %wide.masked.gather9 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %1, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
487  %2 = getelementptr inbounds i32, i32* %A, i64 %index
488  %3 = bitcast i32* %2 to <8 x i32>*
489  %wide.load = load <8 x i32>, <8 x i32>* %3, align 4
490  %4 = getelementptr inbounds i32, i32* %2, i64 8
491  %5 = bitcast i32* %4 to <8 x i32>*
492  %wide.load10 = load <8 x i32>, <8 x i32>* %5, align 4
493  %6 = add nsw <8 x i32> %wide.load, %wide.masked.gather
494  %7 = add nsw <8 x i32> %wide.load10, %wide.masked.gather9
495  %8 = bitcast i32* %2 to <8 x i32>*
496  store <8 x i32> %6, <8 x i32>* %8, align 4
497  %9 = bitcast i32* %4 to <8 x i32>*
498  store <8 x i32> %7, <8 x i32>* %9, align 4
499  %index.next = add nuw i64 %index, 16
500  %vec.ind.next = add <8 x i64> %vec.ind, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>
501  %10 = icmp eq i64 %index.next, 1024
502  br i1 %10, label %for.cond.cleanup, label %vector.body
503
504for.cond.cleanup:                                 ; preds = %vector.body
505  ret void
506}
507
508;void gather_unroll(int * __restrict  A, int * __restrict B) {
509;  for (int i = 0; i < 1024; i+= 4 ) {
510;    A[i] += B[i * 4];
511;    A[i+1] += B[(i+1) * 4];
512;    A[i+2] += B[(i+2) * 4];
513;    A[i+3] += B[(i+3) * 4];
514;  }
515;}
516define void @gather_unroll(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
517;
518; CHECK-LABEL: gather_unroll:
519; CHECK:       # %bb.0: # %entry
520; CHECK-NEXT:    li a2, 256
521; CHECK-NEXT:    li a3, 64
522; CHECK-NEXT:    li a4, 16
523; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
524; CHECK-NEXT:  .LBB9_1: # %vector.body
525; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
526; CHECK-NEXT:    vlse32.v v8, (a1), a3
527; CHECK-NEXT:    vlse32.v v9, (a0), a4
528; CHECK-NEXT:    vadd.vv v8, v9, v8
529; CHECK-NEXT:    vsse32.v v8, (a0), a4
530; CHECK-NEXT:    addi a5, a1, 16
531; CHECK-NEXT:    vlse32.v v8, (a5), a3
532; CHECK-NEXT:    addi a5, a0, 4
533; CHECK-NEXT:    vlse32.v v9, (a5), a4
534; CHECK-NEXT:    vadd.vv v8, v9, v8
535; CHECK-NEXT:    vsse32.v v8, (a5), a4
536; CHECK-NEXT:    addi a5, a1, 32
537; CHECK-NEXT:    vlse32.v v8, (a5), a3
538; CHECK-NEXT:    addi a5, a0, 8
539; CHECK-NEXT:    vlse32.v v9, (a5), a4
540; CHECK-NEXT:    vadd.vv v8, v9, v8
541; CHECK-NEXT:    vsse32.v v8, (a5), a4
542; CHECK-NEXT:    addi a5, a1, 48
543; CHECK-NEXT:    vlse32.v v8, (a5), a3
544; CHECK-NEXT:    addi a5, a0, 12
545; CHECK-NEXT:    vlse32.v v9, (a5), a4
546; CHECK-NEXT:    vadd.vv v8, v9, v8
547; CHECK-NEXT:    vsse32.v v8, (a5), a4
548; CHECK-NEXT:    addi a2, a2, -8
549; CHECK-NEXT:    addi a1, a1, 512
550; CHECK-NEXT:    addi a0, a0, 128
551; CHECK-NEXT:    bnez a2, .LBB9_1
552; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
553; CHECK-NEXT:    ret
554entry:
555  br label %vector.body
556
557vector.body:                                      ; preds = %vector.body, %entry
558  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
559  %vec.ind = phi <8 x i64> [ <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>, %entry ], [ %vec.ind.next, %vector.body ]
560  %0 = shl nuw nsw <8 x i64> %vec.ind, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
561  %1 = getelementptr inbounds i32, i32* %B, <8 x i64> %0
562  %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %1, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
563  %2 = getelementptr inbounds i32, i32* %A, <8 x i64> %vec.ind
564  %wide.masked.gather52 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %2, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
565  %3 = add nsw <8 x i32> %wide.masked.gather52, %wide.masked.gather
566  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %3, <8 x i32*> %2, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
567  %4 = or <8 x i64> %vec.ind, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
568  %5 = shl nsw <8 x i64> %4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
569  %6 = getelementptr inbounds i32, i32* %B, <8 x i64> %5
570  %wide.masked.gather53 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %6, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
571  %7 = getelementptr inbounds i32, i32* %A, <8 x i64> %4
572  %wide.masked.gather54 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %7, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
573  %8 = add nsw <8 x i32> %wide.masked.gather54, %wide.masked.gather53
574  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %8, <8 x i32*> %7, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
575  %9 = or <8 x i64> %vec.ind, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
576  %10 = shl nsw <8 x i64> %9, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
577  %11 = getelementptr inbounds i32, i32* %B, <8 x i64> %10
578  %wide.masked.gather55 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %11, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
579  %12 = getelementptr inbounds i32, i32* %A, <8 x i64> %9
580  %wide.masked.gather56 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %12, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
581  %13 = add nsw <8 x i32> %wide.masked.gather56, %wide.masked.gather55
582  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %13, <8 x i32*> %12, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
583  %14 = or <8 x i64> %vec.ind, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>
584  %15 = shl nsw <8 x i64> %14, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
585  %16 = getelementptr inbounds i32, i32* %B, <8 x i64> %15
586  %wide.masked.gather57 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %16, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
587  %17 = getelementptr inbounds i32, i32* %A, <8 x i64> %14
588  %wide.masked.gather58 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %17, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
589  %18 = add nsw <8 x i32> %wide.masked.gather58, %wide.masked.gather57
590  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %18, <8 x i32*> %17, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
591  %index.next = add nuw i64 %index, 8
592  %vec.ind.next = add <8 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
593  %19 = icmp eq i64 %index.next, 256
594  br i1 %19, label %for.cond.cleanup, label %vector.body
595
596for.cond.cleanup:                                 ; preds = %vector.body
597  ret void
598}
599
600declare <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*>, i32 immarg, <32 x i1>, <32 x i8>)
601declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32 immarg, <8 x i1>, <8 x i32>)
602declare void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8>, <32 x i8*>, i32 immarg, <32 x i1>)
603declare void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, i32 immarg, <8 x i1>)
604
605; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.
606define void @gather_of_pointers(i32** noalias nocapture %0, i32** noalias nocapture readonly %1) {
607;
608; V-LABEL: gather_of_pointers:
609; V:       # %bb.0:
610; V-NEXT:    li a2, 1024
611; V-NEXT:    li a3, 40
612; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
613; V-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
614; V-NEXT:    vlse64.v v8, (a1), a3
615; V-NEXT:    addi a4, a1, 80
616; V-NEXT:    vlse64.v v9, (a4), a3
617; V-NEXT:    vse64.v v8, (a0)
618; V-NEXT:    addi a4, a0, 16
619; V-NEXT:    vse64.v v9, (a4)
620; V-NEXT:    addi a2, a2, -4
621; V-NEXT:    addi a0, a0, 32
622; V-NEXT:    addi a1, a1, 160
623; V-NEXT:    bnez a2, .LBB10_1
624; V-NEXT:  # %bb.2:
625; V-NEXT:    ret
626;
627; ZVE32F-LABEL: gather_of_pointers:
628; ZVE32F:       # %bb.0:
629; ZVE32F-NEXT:    li a2, 0
630; ZVE32F-NEXT:    li a3, 1
631; ZVE32F-NEXT:    li a4, 1024
632; ZVE32F-NEXT:    li a5, 40
633; ZVE32F-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
634; ZVE32F-NEXT:    mul a6, a3, a5
635; ZVE32F-NEXT:    add a6, a1, a6
636; ZVE32F-NEXT:    mul a7, a2, a5
637; ZVE32F-NEXT:    add a7, a1, a7
638; ZVE32F-NEXT:    ld t0, 0(a6)
639; ZVE32F-NEXT:    ld t1, 0(a7)
640; ZVE32F-NEXT:    ld a6, 80(a6)
641; ZVE32F-NEXT:    ld a7, 80(a7)
642; ZVE32F-NEXT:    sd t0, 8(a0)
643; ZVE32F-NEXT:    sd t1, 0(a0)
644; ZVE32F-NEXT:    sd a6, 24(a0)
645; ZVE32F-NEXT:    sd a7, 16(a0)
646; ZVE32F-NEXT:    addi a2, a2, 4
647; ZVE32F-NEXT:    addi a3, a3, 4
648; ZVE32F-NEXT:    addi a4, a4, -4
649; ZVE32F-NEXT:    addi a0, a0, 32
650; ZVE32F-NEXT:    bnez a4, .LBB10_1
651; ZVE32F-NEXT:  # %bb.2:
652; ZVE32F-NEXT:    ret
653  br label %3
654
6553:                                                ; preds = %3, %2
656  %4 = phi i64 [ 0, %2 ], [ %17, %3 ]
657  %5 = phi <2 x i64> [ <i64 0, i64 1>, %2 ], [ %18, %3 ]
658  %6 = mul nuw nsw <2 x i64> %5, <i64 5, i64 5>
659  %7 = mul <2 x i64> %5, <i64 5, i64 5>
660  %8 = add <2 x i64> %7, <i64 10, i64 10>
661  %9 = getelementptr inbounds i32*, i32** %1, <2 x i64> %6
662  %10 = getelementptr inbounds i32*, i32** %1, <2 x i64> %8
663  %11 = call <2 x i32*> @llvm.masked.gather.v2p0i32.v2p0p0i32(<2 x i32**> %9, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i32*> undef)
664  %12 = call <2 x i32*> @llvm.masked.gather.v2p0i32.v2p0p0i32(<2 x i32**> %10, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i32*> undef)
665  %13 = getelementptr inbounds i32*, i32** %0, i64 %4
666  %14 = bitcast i32** %13 to <2 x i32*>*
667  store <2 x i32*> %11, <2 x i32*>* %14, align 8
668  %15 = getelementptr inbounds i32*, i32** %13, i64 2
669  %16 = bitcast i32** %15 to <2 x i32*>*
670  store <2 x i32*> %12, <2 x i32*>* %16, align 8
671  %17 = add nuw i64 %4, 4
672  %18 = add <2 x i64> %5, <i64 4, i64 4>
673  %19 = icmp eq i64 %17, 1024
674  br i1 %19, label %20, label %3
675
67620:                                               ; preds = %3
677  ret void
678}
679
680declare <2 x i32*> @llvm.masked.gather.v2p0i32.v2p0p0i32(<2 x i32**>, i32 immarg, <2 x i1>, <2 x i32*>)
681
682; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.
683define void @scatter_of_pointers(i32** noalias nocapture %0, i32** noalias nocapture readonly %1) {
684;
685; V-LABEL: scatter_of_pointers:
686; V:       # %bb.0:
687; V-NEXT:    li a2, 1024
688; V-NEXT:    li a3, 40
689; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
690; V-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
691; V-NEXT:    vle64.v v8, (a1)
692; V-NEXT:    addi a4, a1, 16
693; V-NEXT:    vle64.v v9, (a4)
694; V-NEXT:    addi a4, a0, 80
695; V-NEXT:    vsse64.v v8, (a0), a3
696; V-NEXT:    vsse64.v v9, (a4), a3
697; V-NEXT:    addi a2, a2, -4
698; V-NEXT:    addi a1, a1, 32
699; V-NEXT:    addi a0, a0, 160
700; V-NEXT:    bnez a2, .LBB11_1
701; V-NEXT:  # %bb.2:
702; V-NEXT:    ret
703;
704; ZVE32F-LABEL: scatter_of_pointers:
705; ZVE32F:       # %bb.0:
706; ZVE32F-NEXT:    li a2, 0
707; ZVE32F-NEXT:    li a3, 1
708; ZVE32F-NEXT:    li a4, 1024
709; ZVE32F-NEXT:    li a5, 40
710; ZVE32F-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
711; ZVE32F-NEXT:    ld a6, 8(a1)
712; ZVE32F-NEXT:    ld a7, 0(a1)
713; ZVE32F-NEXT:    ld t0, 24(a1)
714; ZVE32F-NEXT:    ld t1, 16(a1)
715; ZVE32F-NEXT:    mul t2, a3, a5
716; ZVE32F-NEXT:    add t2, a0, t2
717; ZVE32F-NEXT:    mul t3, a2, a5
718; ZVE32F-NEXT:    add t3, a0, t3
719; ZVE32F-NEXT:    sd a7, 0(t3)
720; ZVE32F-NEXT:    sd a6, 0(t2)
721; ZVE32F-NEXT:    sd t1, 80(t3)
722; ZVE32F-NEXT:    sd t0, 80(t2)
723; ZVE32F-NEXT:    addi a2, a2, 4
724; ZVE32F-NEXT:    addi a3, a3, 4
725; ZVE32F-NEXT:    addi a4, a4, -4
726; ZVE32F-NEXT:    addi a1, a1, 32
727; ZVE32F-NEXT:    bnez a4, .LBB11_1
728; ZVE32F-NEXT:  # %bb.2:
729; ZVE32F-NEXT:    ret
730  br label %3
731
7323:                                                ; preds = %3, %2
733  %4 = phi i64 [ 0, %2 ], [ %17, %3 ]
734  %5 = phi <2 x i64> [ <i64 0, i64 1>, %2 ], [ %18, %3 ]
735  %6 = getelementptr inbounds i32*, i32** %1, i64 %4
736  %7 = bitcast i32** %6 to <2 x i32*>*
737  %8 = load <2 x i32*>, <2 x i32*>* %7, align 8
738  %9 = getelementptr inbounds i32*, i32** %6, i64 2
739  %10 = bitcast i32** %9 to <2 x i32*>*
740  %11 = load <2 x i32*>, <2 x i32*>* %10, align 8
741  %12 = mul nuw nsw <2 x i64> %5, <i64 5, i64 5>
742  %13 = mul <2 x i64> %5, <i64 5, i64 5>
743  %14 = add <2 x i64> %13, <i64 10, i64 10>
744  %15 = getelementptr inbounds i32*, i32** %0, <2 x i64> %12
745  %16 = getelementptr inbounds i32*, i32** %0, <2 x i64> %14
746  call void @llvm.masked.scatter.v2p0i32.v2p0p0i32(<2 x i32*> %8, <2 x i32**> %15, i32 8, <2 x i1> <i1 true, i1 true>)
747  call void @llvm.masked.scatter.v2p0i32.v2p0p0i32(<2 x i32*> %11, <2 x i32**> %16, i32 8, <2 x i1> <i1 true, i1 true>)
748  %17 = add nuw i64 %4, 4
749  %18 = add <2 x i64> %5, <i64 4, i64 4>
750  %19 = icmp eq i64 %17, 1024
751  br i1 %19, label %20, label %3
752
75320:                                               ; preds = %3
754  ret void
755}
756
757declare void @llvm.masked.scatter.v2p0i32.v2p0p0i32(<2 x i32*>, <2 x i32**>, i32 immarg, <2 x i1>)
758
759define void @strided_load_startval_add_with_splat(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i32 signext %2) {
760;
761; CHECK-LABEL: strided_load_startval_add_with_splat:
762; CHECK:       # %bb.0:
763; CHECK-NEXT:    li a3, 1024
764; CHECK-NEXT:    beq a2, a3, .LBB12_7
765; CHECK-NEXT:  # %bb.1:
766; CHECK-NEXT:    li a3, 1023
767; CHECK-NEXT:    subw a4, a3, a2
768; CHECK-NEXT:    li a5, 31
769; CHECK-NEXT:    mv a3, a2
770; CHECK-NEXT:    bltu a4, a5, .LBB12_5
771; CHECK-NEXT:  # %bb.2:
772; CHECK-NEXT:    slli a3, a4, 32
773; CHECK-NEXT:    srli a3, a3, 32
774; CHECK-NEXT:    addi a4, a3, 1
775; CHECK-NEXT:    andi a5, a4, -32
776; CHECK-NEXT:    add a3, a5, a2
777; CHECK-NEXT:    slli a6, a2, 2
778; CHECK-NEXT:    add a6, a6, a2
779; CHECK-NEXT:    add a2, a0, a2
780; CHECK-NEXT:    add a6, a1, a6
781; CHECK-NEXT:    li a7, 32
782; CHECK-NEXT:    li t0, 5
783; CHECK-NEXT:    mv t1, a5
784; CHECK-NEXT:  .LBB12_3: # =>This Inner Loop Header: Depth=1
785; CHECK-NEXT:    vsetvli zero, a7, e8, m1, ta, mu
786; CHECK-NEXT:    vlse8.v v8, (a6), t0
787; CHECK-NEXT:    vle8.v v9, (a2)
788; CHECK-NEXT:    vadd.vv v8, v9, v8
789; CHECK-NEXT:    vse8.v v8, (a2)
790; CHECK-NEXT:    addi t1, t1, -32
791; CHECK-NEXT:    addi a2, a2, 32
792; CHECK-NEXT:    addi a6, a6, 160
793; CHECK-NEXT:    bnez t1, .LBB12_3
794; CHECK-NEXT:  # %bb.4:
795; CHECK-NEXT:    beq a4, a5, .LBB12_7
796; CHECK-NEXT:  .LBB12_5:
797; CHECK-NEXT:    slli a2, a3, 2
798; CHECK-NEXT:    add a2, a2, a3
799; CHECK-NEXT:    add a1, a1, a2
800; CHECK-NEXT:    li a2, 1024
801; CHECK-NEXT:  .LBB12_6: # =>This Inner Loop Header: Depth=1
802; CHECK-NEXT:    lb a4, 0(a1)
803; CHECK-NEXT:    add a5, a0, a3
804; CHECK-NEXT:    lb a6, 0(a5)
805; CHECK-NEXT:    addw a4, a6, a4
806; CHECK-NEXT:    sb a4, 0(a5)
807; CHECK-NEXT:    addiw a4, a3, 1
808; CHECK-NEXT:    addi a3, a3, 1
809; CHECK-NEXT:    addi a1, a1, 5
810; CHECK-NEXT:    bne a4, a2, .LBB12_6
811; CHECK-NEXT:  .LBB12_7:
812; CHECK-NEXT:    ret
813  %4 = icmp eq i32 %2, 1024
814  br i1 %4, label %36, label %5
815
8165:                                                ; preds = %3
817  %6 = sext i32 %2 to i64
818  %7 = sub i32 1023, %2
819  %8 = zext i32 %7 to i64
820  %9 = add nuw nsw i64 %8, 1
821  %10 = icmp ult i32 %7, 31
822  br i1 %10, label %34, label %11
823
82411:                                               ; preds = %5
825  %12 = and i64 %9, 8589934560
826  %13 = add nsw i64 %12, %6
827  %14 = insertelement <32 x i64> poison, i64 %6, i64 0
828  %15 = shufflevector <32 x i64> %14, <32 x i64> poison, <32 x i32> zeroinitializer
829  %16 = add <32 x i64> %15, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>
830  br label %17
831
83217:                                               ; preds = %17, %11
833  %18 = phi i64 [ 0, %11 ], [ %29, %17 ]
834  %19 = phi <32 x i64> [ %16, %11 ], [ %30, %17 ]
835  %20 = add i64 %18, %6
836  %21 = mul nsw <32 x i64> %19, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
837  %22 = getelementptr inbounds i8, i8* %1, <32 x i64> %21
838  %23 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %22, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
839  %24 = getelementptr inbounds i8, i8* %0, i64 %20
840  %25 = bitcast i8* %24 to <32 x i8>*
841  %26 = load <32 x i8>, <32 x i8>* %25, align 1
842  %27 = add <32 x i8> %26, %23
843  %28 = bitcast i8* %24 to <32 x i8>*
844  store <32 x i8> %27, <32 x i8>* %28, align 1
845  %29 = add nuw i64 %18, 32
846  %30 = add <32 x i64> %19, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
847  %31 = icmp eq i64 %29, %12
848  br i1 %31, label %32, label %17
849
85032:                                               ; preds = %17
851  %33 = icmp eq i64 %9, %12
852  br i1 %33, label %36, label %34
853
85434:                                               ; preds = %5, %32
855  %35 = phi i64 [ %6, %5 ], [ %13, %32 ]
856  br label %37
857
85836:                                               ; preds = %37, %32, %3
859  ret void
860
86137:                                               ; preds = %34, %37
862  %38 = phi i64 [ %45, %37 ], [ %35, %34 ]
863  %39 = mul nsw i64 %38, 5
864  %40 = getelementptr inbounds i8, i8* %1, i64 %39
865  %41 = load i8, i8* %40, align 1
866  %42 = getelementptr inbounds i8, i8* %0, i64 %38
867  %43 = load i8, i8* %42, align 1
868  %44 = add i8 %43, %41
869  store i8 %44, i8* %42, align 1
870  %45 = add nsw i64 %38, 1
871  %46 = trunc i64 %45 to i32
872  %47 = icmp eq i32 %46, 1024
873  br i1 %47, label %36, label %37
874}
875
876declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32 immarg, <16 x i1>, <16 x i8>)
877declare void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8>, <16 x i8*>, i32 immarg, <16 x i1>)
878
879define void @gather_no_scalar_remainder(i8* noalias nocapture noundef %arg, i8* noalias nocapture noundef readonly %arg1, i64 noundef %arg2) {
880; CHECK-LABEL: gather_no_scalar_remainder:
881; CHECK:       # %bb.0: # %bb
882; CHECK-NEXT:    slli a2, a2, 4
883; CHECK-NEXT:    beqz a2, .LBB13_3
884; CHECK-NEXT:  # %bb.1: # %bb2
885; CHECK-NEXT:    li a3, 5
886; CHECK-NEXT:    vsetivli zero, 16, e8, mf2, ta, mu
887; CHECK-NEXT:  .LBB13_2: # %bb4
888; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
889; CHECK-NEXT:    vlse8.v v8, (a1), a3
890; CHECK-NEXT:    vle8.v v9, (a0)
891; CHECK-NEXT:    vadd.vv v8, v9, v8
892; CHECK-NEXT:    vse8.v v8, (a0)
893; CHECK-NEXT:    addi a2, a2, -16
894; CHECK-NEXT:    addi a0, a0, 16
895; CHECK-NEXT:    addi a1, a1, 80
896; CHECK-NEXT:    bnez a2, .LBB13_2
897; CHECK-NEXT:  .LBB13_3: # %bb16
898; CHECK-NEXT:    ret
899bb:
900  %i = shl i64 %arg2, 4
901  %i3 = icmp eq i64 %i, 0
902  br i1 %i3, label %bb16, label %bb2
903
904bb2:
905  br label %bb4
906
907bb4:                                              ; preds = %bb4, %bb
908  %i5 = phi i64 [ %i13, %bb4 ], [ 0, %bb2 ]
909  %i6 = phi <16 x i64> [ %i14, %bb4 ], [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %bb2 ]
910  %i7 = mul <16 x i64> %i6, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
911  %i8 = getelementptr inbounds i8, i8* %arg1, <16 x i64> %i7
912  %i9 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %i8, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
913  %i10 = getelementptr inbounds i8, i8* %arg, i64 %i5
914  %cast = bitcast i8* %i10 to <16 x i8>*
915  %i11 = load <16 x i8>, <16 x i8>* %cast, align 1
916  %i12 = add <16 x i8> %i11, %i9
917  %cast2 = bitcast i8* %i10 to <16 x i8>*
918  store <16 x i8> %i12, <16 x i8>* %cast2, align 1
919  %i13 = add nuw i64 %i5, 16
920  %i14 = add <16 x i64> %i6, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>
921  %i15 = icmp eq i64 %i13, %i
922  br i1 %i15, label %bb16, label %bb4
923
924bb16:                                             ; preds = %bb4, %bb
925  ret void
926}
927