1; REQUIRES: asserts 2; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s 3; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-cond-stores-vec -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s --check-prefix=INTER 4 5target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 6 7%pair = type { i32, i32 } 8 9; CHECK-LABEL: consecutive_ptr_forward 10; 11; Check that a forward consecutive pointer is recognized as uniform and remains 12; uniform after vectorization. 13; 14; CHECK: LV: Found uniform instruction: %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i 15; CHECK: vector.body 16; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 17; CHECK-NOT: getelementptr 18; CHECK: getelementptr inbounds i32, i32* %a, i64 %index 19; CHECK-NOT: getelementptr 20; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body 21; 22define i32 @consecutive_ptr_forward(i32* %a, i64 %n) { 23entry: 24 br label %for.body 25 26for.body: 27 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 28 %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ] 29 %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i 30 %tmp2 = load i32, i32* %tmp1, align 8 31 %tmp3 = add i32 %tmp0, %tmp2 32 %i.next = add nuw nsw i64 %i, 1 33 %cond = icmp slt i64 %i.next, %n 34 br i1 %cond, label %for.body, label %for.end 35 36for.end: 37 %tmp4 = phi i32 [ %tmp3, %for.body ] 38 ret i32 %tmp4 39} 40 41; CHECK-LABEL: consecutive_ptr_reverse 42; 43; Check that a reverse consecutive pointer is recognized as uniform and remains 44; uniform after vectorization. 45; 46; CHECK: LV: Found uniform instruction: %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i 47; CHECK: vector.body 48; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 49; CHECK: %offset.idx = sub i64 %n, %index 50; CHECK-NOT: getelementptr 51; CHECK: %[[G0:.+]] = getelementptr inbounds i32, i32* %a, i64 %offset.idx 52; CHECK: getelementptr i32, i32* %[[G0]], i64 -3 53; CHECK-NOT: getelementptr 54; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body 55; 56define i32 @consecutive_ptr_reverse(i32* %a, i64 %n) { 57entry: 58 br label %for.body 59 60for.body: 61 %i = phi i64 [ %i.next, %for.body ], [ %n, %entry ] 62 %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ] 63 %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i 64 %tmp2 = load i32, i32* %tmp1, align 8 65 %tmp3 = add i32 %tmp0, %tmp2 66 %i.next = add nuw nsw i64 %i, -1 67 %cond = icmp sgt i64 %i.next, 0 68 br i1 %cond, label %for.body, label %for.end 69 70for.end: 71 %tmp4 = phi i32 [ %tmp3, %for.body ] 72 ret i32 %tmp4 73} 74 75; CHECK-LABEL: interleaved_access_forward 76; INTER-LABEL: interleaved_access_forward 77; 78; Check that a consecutive-like pointer used by a forward interleaved group is 79; recognized as uniform and remains uniform after vectorization. When 80; interleaved memory accesses aren't enabled, the pointer should not be 81; recognized as uniform, and it should not be uniform after vectorization. 82; 83; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 84; CHECK-NOT: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 85; CHECK: vector.body 86; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 87; CHECK: %[[I1:.+]] = or i64 %index, 1 88; CHECK: %[[I2:.+]] = or i64 %index, 2 89; CHECK: %[[I3:.+]] = or i64 %index, 3 90; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0 91; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0 92; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0 93; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0 94; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %index, i32 1 95; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 1 96; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 1 97; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 1 98; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body 99; 100; INTER: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 101; INTER: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 102; INTER: vector.body 103; INTER: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 104; INTER-NOT: getelementptr 105; INTER: getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0 106; INTER-NOT: getelementptr 107; INTER: br i1 {{.*}}, label %middle.block, label %vector.body 108; 109define i32 @interleaved_access_forward(%pair* %p, i64 %n) { 110entry: 111 br label %for.body 112 113for.body: 114 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 115 %tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ] 116 %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 117 %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 118 %tmp3 = load i32, i32* %tmp1, align 8 119 %tmp4 = load i32, i32* %tmp2, align 8 120 %tmp5 = add i32 %tmp3, %tmp4 121 %tmp6 = add i32 %tmp0, %tmp5 122 %i.next = add nuw nsw i64 %i, 1 123 %cond = icmp slt i64 %i.next, %n 124 br i1 %cond, label %for.body, label %for.end 125 126for.end: 127 %tmp14 = phi i32 [ %tmp6, %for.body ] 128 ret i32 %tmp14 129} 130 131; CHECK-LABEL: interleaved_access_reverse 132; INTER-LABEL: interleaved_access_reverse 133; 134; Check that a consecutive-like pointer used by a reverse interleaved group is 135; recognized as uniform and remains uniform after vectorization. When 136; interleaved memory accesses aren't enabled, the pointer should not be 137; recognized as uniform, and it should not be uniform after vectorization. 138; 139; recognized as uniform, and it should not be uniform after vectorization. 140; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 141; CHECK-NOT: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 142; CHECK: vector.body 143; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 144; CHECK: %offset.idx = sub i64 %n, %index 145; CHECK: %[[I1:.+]] = add i64 %offset.idx, -1 146; CHECK: %[[I2:.+]] = add i64 %offset.idx, -2 147; CHECK: %[[I3:.+]] = add i64 %offset.idx, -3 148; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 0 149; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0 150; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0 151; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0 152; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 1 153; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 1 154; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 1 155; CHECK: getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 1 156; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body 157; 158; INTER: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 159; INTER: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 160; INTER: vector.body 161; INTER: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 162; INTER: %offset.idx = sub i64 %n, %index 163; INTER-NOT: getelementptr 164; INTER: %[[G0:.+]] = getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 0 165; INTER: getelementptr i32, i32* %[[G0]], i64 -6 166; INTER-NOT: getelementptr 167; INTER: br i1 {{.*}}, label %middle.block, label %vector.body 168; 169define i32 @interleaved_access_reverse(%pair* %p, i64 %n) { 170entry: 171 br label %for.body 172 173for.body: 174 %i = phi i64 [ %i.next, %for.body ], [ %n, %entry ] 175 %tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ] 176 %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 177 %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 178 %tmp3 = load i32, i32* %tmp1, align 8 179 %tmp4 = load i32, i32* %tmp2, align 8 180 %tmp5 = add i32 %tmp3, %tmp4 181 %tmp6 = add i32 %tmp0, %tmp5 182 %i.next = add nuw nsw i64 %i, -1 183 %cond = icmp sgt i64 %i.next, 0 184 br i1 %cond, label %for.body, label %for.end 185 186for.end: 187 %tmp14 = phi i32 [ %tmp6, %for.body ] 188 ret i32 %tmp14 189} 190 191; INTER-LABEL: predicated_store 192; 193; Check that a consecutive-like pointer used by a forward interleaved group and 194; scalarized store is not recognized as uniform and is not uniform after 195; vectorization. The store is scalarized because it's in a predicated block. 196; Even though the load in this example is vectorized and only uses the pointer 197; as if it were uniform, the store is scalarized, making the pointer 198; non-uniform. 199; 200; INTER-NOT: LV: Found uniform instruction: %tmp0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 201; INTER: vector.body 202; INTER: %index = phi i64 [ 0, %vector.ph ], [ %index.next, {{.*}} ] 203; INTER: %[[I1:.+]] = or i64 %index, 1 204; INTER: %[[I2:.+]] = or i64 %index, 2 205; INTER: %[[I3:.+]] = or i64 %index, 3 206; INTER: %[[G0:.+]] = getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0 207; INTER: getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0 208; INTER: getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0 209; INTER: getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0 210; INTER: %[[B0:.+]] = bitcast i32* %[[G0]] to <8 x i32>* 211; INTER: %wide.vec = load <8 x i32>, <8 x i32>* %[[B0]], align 8 212; INTER: br i1 {{.*}}, label %middle.block, label %vector.body 213; 214define void @predicated_store(%pair *%p, i32 %x, i64 %n) { 215entry: 216 br label %for.body 217 218for.body: 219 %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] 220 %tmp0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 221 %tmp1 = load i32, i32* %tmp0, align 8 222 %tmp2 = icmp eq i32 %tmp1, %x 223 br i1 %tmp2, label %if.then, label %if.merge 224 225if.then: 226 store i32 %tmp1, i32* %tmp0, align 8 227 br label %if.merge 228 229if.merge: 230 %i.next = add nuw nsw i64 %i, 1 231 %cond = icmp slt i64 %i.next, %n 232 br i1 %cond, label %for.body, label %for.end 233 234for.end: 235 ret void 236} 237 238; CHECK-LABEL: irregular_type 239; 240; Check that a consecutive pointer used by a scalarized store is not recognized 241; as uniform and is not uniform after vectorization. The store is scalarized 242; because the stored type may required padding. 243; 244; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %i 245; CHECK: vector.body 246; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] 247; CHECK: %[[I1:.+]] = or i64 %index, 1 248; CHECK: %[[I2:.+]] = or i64 %index, 2 249; CHECK: %[[I3:.+]] = or i64 %index, 3 250; CHECK: getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %index 251; CHECK: getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I1]] 252; CHECK: getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I2]] 253; CHECK: getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I3]] 254; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body 255; 256define void @irregular_type(x86_fp80* %a, i64 %n) { 257entry: 258 br label %for.body 259 260for.body: 261 %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ] 262 %tmp0 = sitofp i32 1 to x86_fp80 263 %tmp1 = getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %i 264 store x86_fp80 %tmp0, x86_fp80* %tmp1, align 16 265 %i.next = add i64 %i, 1 266 %cond = icmp slt i64 %i.next, %n 267 br i1 %cond, label %for.body, label %for.end 268 269for.end: 270 ret void 271} 272