1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple riscv64-linux-gnu -mattr=+v,+f -S 2>%t | FileCheck %s -check-prefix=VLENUNK 3; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -riscv-v-vector-bits-min=-1 -mtriple riscv64-linux-gnu -mattr=+v,+f -S 2>%t | FileCheck %s -check-prefix=VLEN128 4 5; Note: +v implies a Zvl128b (i.e. minimal VLEN of 128), but as can be seen, we're currently 6; not using that information unless an explicit vector width is set. (FIXME) 7 8; A collection of fairly basic functional tests when both fixed and scalable vectorization is 9; allowed. The primary goal of this is check for crashes during cost modeling, but it also 10; exercises the default heuristics in a useful way. 11 12target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" 13target triple = "riscv64" 14 15define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { 16; VLENUNK-LABEL: @vector_add( 17; VLENUNK-NEXT: entry: 18; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 19; VLENUNK: for.body: 20; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 21; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 22; VLENUNK-NEXT: [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8 23; VLENUNK-NEXT: [[ADD:%.*]] = add i64 [[ELEM]], [[V:%.*]] 24; VLENUNK-NEXT: store i64 [[ADD]], ptr [[ARRAYIDX]], align 8 25; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 26; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 27; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 28; VLENUNK: for.end: 29; VLENUNK-NEXT: ret void 30; 31; VLEN128-LABEL: @vector_add( 32; VLEN128-NEXT: entry: 33; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 34; VLEN128: vector.ph: 35; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[V:%.*]], i32 0 36; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 37; VLEN128-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i32 0 38; VLEN128-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT2]], <2 x i64> poison, <2 x i32> zeroinitializer 39; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 40; VLEN128: vector.body: 41; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 42; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 43; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 44; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 45; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 46; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 47; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8 48; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 49; VLEN128-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 50; VLEN128-NEXT: [[TMP6:%.*]] = add <2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] 51; VLEN128-NEXT: [[TMP7:%.*]] = add <2 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT3]] 52; VLEN128-NEXT: store <2 x i64> [[TMP6]], ptr [[TMP4]], align 8 53; VLEN128-NEXT: store <2 x i64> [[TMP7]], ptr [[TMP5]], align 8 54; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 55; VLEN128-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 56; VLEN128-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 57; VLEN128: middle.block: 58; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 59; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 60; VLEN128: scalar.ph: 61; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 62; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 63; VLEN128: for.body: 64; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 65; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 66; VLEN128-NEXT: [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8 67; VLEN128-NEXT: [[ADD:%.*]] = add i64 [[ELEM]], [[V]] 68; VLEN128-NEXT: store i64 [[ADD]], ptr [[ARRAYIDX]], align 8 69; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 70; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 71; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] 72; VLEN128: for.end: 73; VLEN128-NEXT: ret void 74; 75entry: 76 br label %for.body 77 78for.body: 79 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 80 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 81 %elem = load i64, ptr %arrayidx 82 %add = add i64 %elem, %v 83 store i64 %add, ptr %arrayidx 84 %iv.next = add nuw nsw i64 %iv, 1 85 %exitcond.not = icmp eq i64 %iv.next, 1024 86 br i1 %exitcond.not, label %for.end, label %for.body 87 88for.end: 89 ret void 90} 91 92; a[b[i]] += v, mostly to exercise scatter/gather costing 93; TODO: Currently fails to vectorize due to a memory conflict 94define void @indexed_add(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) { 95; VLENUNK-LABEL: @indexed_add( 96; VLENUNK-NEXT: entry: 97; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 98; VLENUNK: for.body: 99; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 100; VLENUNK-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[IV]] 101; VLENUNK-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 102; VLENUNK-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[AIDX]] 103; VLENUNK-NEXT: [[ELEM:%.*]] = load i64, ptr [[AADDR]], align 8 104; VLENUNK-NEXT: [[ADD:%.*]] = add i64 [[ELEM]], [[V:%.*]] 105; VLENUNK-NEXT: store i64 [[ADD]], ptr [[AADDR]], align 8 106; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 107; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 108; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 109; VLENUNK: for.end: 110; VLENUNK-NEXT: ret void 111; 112; VLEN128-LABEL: @indexed_add( 113; VLEN128-NEXT: entry: 114; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 115; VLEN128: for.body: 116; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 117; VLEN128-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[IV]] 118; VLEN128-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 119; VLEN128-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[AIDX]] 120; VLEN128-NEXT: [[ELEM:%.*]] = load i64, ptr [[AADDR]], align 8 121; VLEN128-NEXT: [[ADD:%.*]] = add i64 [[ELEM]], [[V:%.*]] 122; VLEN128-NEXT: store i64 [[ADD]], ptr [[AADDR]], align 8 123; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 124; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 125; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 126; VLEN128: for.end: 127; VLEN128-NEXT: ret void 128; 129entry: 130 br label %for.body 131 132for.body: 133 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 134 %baddr = getelementptr inbounds i64, ptr %b, i64 %iv 135 %aidx = load i64, ptr %baddr 136 %aaddr = getelementptr inbounds i64, ptr %a, i64 %aidx 137 %elem = load i64, ptr %aaddr 138 %add = add i64 %elem, %v 139 store i64 %add, ptr %aaddr 140 %iv.next = add nuw nsw i64 %iv, 1 141 %exitcond.not = icmp eq i64 %iv.next, 1024 142 br i1 %exitcond.not, label %for.end, label %for.body 143 144for.end: 145 ret void 146} 147 148; a[b[i]] = v, exercise scatter support 149define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) { 150; VLENUNK-LABEL: @indexed_store( 151; VLENUNK-NEXT: entry: 152; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 153; VLENUNK: for.body: 154; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 155; VLENUNK-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[IV]] 156; VLENUNK-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 157; VLENUNK-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[AIDX]] 158; VLENUNK-NEXT: store i64 [[V:%.*]], ptr [[AADDR]], align 8 159; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 160; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 161; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 162; VLENUNK: for.end: 163; VLENUNK-NEXT: ret void 164; 165; VLEN128-LABEL: @indexed_store( 166; VLEN128-NEXT: entry: 167; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 168; VLEN128: vector.ph: 169; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[V:%.*]], i32 0 170; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 171; VLEN128-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i32 0 172; VLEN128-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT2]], <2 x i64> poison, <2 x i32> zeroinitializer 173; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 174; VLEN128: vector.body: 175; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 176; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 177; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 178; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP0]] 179; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]] 180; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 181; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8 182; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 183; VLEN128-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 184; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], <2 x i64> [[WIDE_LOAD]] 185; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], <2 x i64> [[WIDE_LOAD1]] 186; VLEN128-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP6]], i32 8, <2 x i1> <i1 true, i1 true>) 187; VLEN128-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[BROADCAST_SPLAT3]], <2 x ptr> [[TMP7]], i32 8, <2 x i1> <i1 true, i1 true>) 188; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 189; VLEN128-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 190; VLEN128-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 191; VLEN128: middle.block: 192; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 193; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 194; VLEN128: scalar.ph: 195; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 196; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 197; VLEN128: for.body: 198; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 199; VLEN128-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] 200; VLEN128-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 201; VLEN128-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]] 202; VLEN128-NEXT: store i64 [[V]], ptr [[AADDR]], align 8 203; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 204; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 205; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 206; VLEN128: for.end: 207; VLEN128-NEXT: ret void 208; 209entry: 210 br label %for.body 211 212for.body: 213 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 214 %baddr = getelementptr inbounds i64, ptr %b, i64 %iv 215 %aidx = load i64, ptr %baddr 216 %aaddr = getelementptr inbounds i64, ptr %a, i64 %aidx 217 store i64 %v, ptr %aaddr 218 %iv.next = add nuw nsw i64 %iv, 1 219 %exitcond.not = icmp eq i64 %iv.next, 1024 220 br i1 %exitcond.not, label %for.end, label %for.body 221 222for.end: 223 ret void 224} 225 226define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) { 227; VLENUNK-LABEL: @indexed_load( 228; VLENUNK-NEXT: entry: 229; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 230; VLENUNK: for.body: 231; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 232; VLENUNK-NEXT: [[SUM:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ] 233; VLENUNK-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[IV]] 234; VLENUNK-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 235; VLENUNK-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[AIDX]] 236; VLENUNK-NEXT: [[ELEM:%.*]] = load i64, ptr [[AADDR]], align 8 237; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 238; VLENUNK-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[ELEM]] 239; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 240; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 241; VLENUNK: for.end: 242; VLENUNK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], [[FOR_BODY]] ] 243; VLENUNK-NEXT: ret i64 [[SUM_NEXT_LCSSA]] 244; 245; VLEN128-LABEL: @indexed_load( 246; VLEN128-NEXT: entry: 247; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 248; VLEN128: vector.ph: 249; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 250; VLEN128: vector.body: 251; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 252; VLEN128-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] 253; VLEN128-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] 254; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 255; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 256; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP0]] 257; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]] 258; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 259; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8 260; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 261; VLEN128-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 262; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], <2 x i64> [[WIDE_LOAD]] 263; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], <2 x i64> [[WIDE_LOAD2]] 264; VLEN128-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> [[TMP6]], i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> undef) 265; VLEN128-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> [[TMP7]], i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> undef) 266; VLEN128-NEXT: [[TMP8]] = add <2 x i64> [[VEC_PHI]], [[WIDE_MASKED_GATHER]] 267; VLEN128-NEXT: [[TMP9]] = add <2 x i64> [[VEC_PHI1]], [[WIDE_MASKED_GATHER3]] 268; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 269; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 270; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 271; VLEN128: middle.block: 272; VLEN128-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP9]], [[TMP8]] 273; VLEN128-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) 274; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 275; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 276; VLEN128: scalar.ph: 277; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 278; VLEN128-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] 279; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 280; VLEN128: for.body: 281; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 282; VLEN128-NEXT: [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ] 283; VLEN128-NEXT: [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] 284; VLEN128-NEXT: [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8 285; VLEN128-NEXT: [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]] 286; VLEN128-NEXT: [[ELEM:%.*]] = load i64, ptr [[AADDR]], align 8 287; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 288; VLEN128-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[ELEM]] 289; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 290; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 291; VLEN128: for.end: 292; VLEN128-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] 293; VLEN128-NEXT: ret i64 [[SUM_NEXT_LCSSA]] 294; 295entry: 296 br label %for.body 297 298for.body: 299 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 300 %sum = phi i64 [0, %entry], [%sum.next, %for.body] 301 %baddr = getelementptr inbounds i64, ptr %b, i64 %iv 302 %aidx = load i64, ptr %baddr 303 %aaddr = getelementptr inbounds i64, ptr %a, i64 %aidx 304 %elem = load i64, ptr %aaddr 305 %iv.next = add nuw nsw i64 %iv, 1 306 %sum.next = add i64 %sum, %elem 307 %exitcond.not = icmp eq i64 %iv.next, 1024 308 br i1 %exitcond.not, label %for.end, label %for.body 309 310for.end: 311 ret i64 %sum.next 312} 313 314define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { 315; VLENUNK-LABEL: @splat_int( 316; VLENUNK-NEXT: entry: 317; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 318; VLENUNK: for.body: 319; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 320; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 321; VLENUNK-NEXT: store i64 [[V:%.*]], ptr [[ARRAYIDX]], align 8 322; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 323; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 324; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 325; VLENUNK: for.end: 326; VLENUNK-NEXT: ret void 327; 328; VLEN128-LABEL: @splat_int( 329; VLEN128-NEXT: entry: 330; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 331; VLEN128: vector.ph: 332; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[V:%.*]], i32 0 333; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 334; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i32 0 335; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer 336; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 337; VLEN128: vector.body: 338; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 339; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 340; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 341; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 342; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 343; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 344; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 8 345; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 346; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP5]], align 8 347; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 348; VLEN128-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 349; VLEN128-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 350; VLEN128: middle.block: 351; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 352; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 353; VLEN128: scalar.ph: 354; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 355; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 356; VLEN128: for.body: 357; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 358; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 359; VLEN128-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 360; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 361; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 362; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 363; VLEN128: for.end: 364; VLEN128-NEXT: ret void 365; 366entry: 367 br label %for.body 368 369for.body: 370 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 371 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 372 store i64 %v, ptr %arrayidx 373 %iv.next = add nuw nsw i64 %iv, 1 374 %exitcond.not = icmp eq i64 %iv.next, 1024 375 br i1 %exitcond.not, label %for.end, label %for.body 376 377for.end: 378 ret void 379} 380 381define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { 382; VLENUNK-LABEL: @splat_ptr( 383; VLENUNK-NEXT: entry: 384; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 385; VLENUNK: for.body: 386; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 387; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 388; VLENUNK-NEXT: store ptr [[V:%.*]], ptr [[ARRAYIDX]], align 8 389; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 390; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 391; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 392; VLENUNK: for.end: 393; VLENUNK-NEXT: ret void 394; 395; VLEN128-LABEL: @splat_ptr( 396; VLEN128-NEXT: entry: 397; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 398; VLEN128: vector.ph: 399; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[V:%.*]], i32 0 400; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer 401; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x ptr> poison, ptr [[V]], i32 0 402; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT1]], <2 x ptr> poison, <2 x i32> zeroinitializer 403; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 404; VLEN128: vector.body: 405; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 406; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 407; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 408; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 409; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 410; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i32 0 411; VLEN128-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 8 412; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i32 2 413; VLEN128-NEXT: store <2 x ptr> [[BROADCAST_SPLAT2]], ptr [[TMP5]], align 8 414; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 415; VLEN128-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 416; VLEN128-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 417; VLEN128: middle.block: 418; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 419; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 420; VLEN128: scalar.ph: 421; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 422; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 423; VLEN128: for.body: 424; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 425; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 426; VLEN128-NEXT: store ptr [[V]], ptr [[ARRAYIDX]], align 8 427; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 428; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 429; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 430; VLEN128: for.end: 431; VLEN128-NEXT: ret void 432; 433entry: 434 br label %for.body 435 436for.body: 437 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 438 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 439 store ptr %v, ptr %arrayidx 440 %iv.next = add nuw nsw i64 %iv, 1 441 %exitcond.not = icmp eq i64 %iv.next, 1024 442 br i1 %exitcond.not, label %for.end, label %for.body 443 444for.end: 445 ret void 446} 447 448define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) { 449; VLENUNK-LABEL: @uniform_store( 450; VLENUNK-NEXT: entry: 451; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 452; VLENUNK: for.body: 453; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 454; VLENUNK-NEXT: store i64 [[V:%.*]], ptr [[B:%.*]], align 8 455; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 456; VLENUNK-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 457; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 458; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 459; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 460; VLENUNK: for.end: 461; VLENUNK-NEXT: ret void 462; 463; VLEN128-LABEL: @uniform_store( 464; VLEN128-NEXT: entry: 465; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 466; VLEN128: vector.ph: 467; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[V:%.*]], i32 0 468; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 469; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i32 0 470; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer 471; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 472; VLEN128: vector.body: 473; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 474; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 475; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 476; VLEN128-NEXT: store i64 [[V]], ptr [[B:%.*]], align 8 477; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 8 478; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 8 479; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 8 480; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 481; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 482; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 483; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 8 484; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 485; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP5]], align 8 486; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 487; VLEN128-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 488; VLEN128-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] 489; VLEN128: middle.block: 490; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 491; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 492; VLEN128: scalar.ph: 493; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 494; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 495; VLEN128: for.body: 496; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 497; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 8 498; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 499; VLEN128-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 500; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 501; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 502; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 503; VLEN128: for.end: 504; VLEN128-NEXT: ret void 505; 506entry: 507 br label %for.body 508 509for.body: 510 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 511 store i64 %v, ptr %b, align 8 512 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 513 store i64 %v, ptr %arrayidx 514 %iv.next = add nuw nsw i64 %iv, 1 515 %exitcond.not = icmp eq i64 %iv.next, 1024 516 br i1 %exitcond.not, label %for.end, label %for.body 517 518for.end: 519 ret void 520} 521 522define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) { 523; VLENUNK-LABEL: @uniform_store_unaligned( 524; VLENUNK-NEXT: entry: 525; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 526; VLENUNK: for.body: 527; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 528; VLENUNK-NEXT: store i64 [[V:%.*]], ptr [[B:%.*]], align 1 529; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 530; VLENUNK-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 531; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 532; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 533; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 534; VLENUNK: for.end: 535; VLENUNK-NEXT: ret void 536; 537; VLEN128-LABEL: @uniform_store_unaligned( 538; VLEN128-NEXT: entry: 539; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 540; VLEN128: vector.ph: 541; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[V:%.*]], i32 0 542; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 543; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i32 0 544; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer 545; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 546; VLEN128: vector.body: 547; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 548; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 549; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 550; VLEN128-NEXT: store i64 [[V]], ptr [[B:%.*]], align 1 551; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 1 552; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 1 553; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 1 554; VLEN128-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 555; VLEN128-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 556; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 557; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 8 558; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 559; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP5]], align 8 560; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 561; VLEN128-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 562; VLEN128-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] 563; VLEN128: middle.block: 564; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 565; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 566; VLEN128: scalar.ph: 567; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 568; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 569; VLEN128: for.body: 570; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 571; VLEN128-NEXT: store i64 [[V]], ptr [[B]], align 1 572; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 573; VLEN128-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 574; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 575; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 576; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 577; VLEN128: for.end: 578; VLEN128-NEXT: ret void 579; 580entry: 581 br label %for.body 582 583for.body: 584 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 585 store i64 %v, ptr %b, align 1 586 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 587 store i64 %v, ptr %arrayidx 588 %iv.next = add nuw nsw i64 %iv, 1 589 %exitcond.not = icmp eq i64 %iv.next, 1024 590 br i1 %exitcond.not, label %for.end, label %for.body 591 592for.end: 593 ret void 594} 595 596define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) { 597; VLENUNK-LABEL: @uniform_load( 598; VLENUNK-NEXT: entry: 599; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 600; VLENUNK: for.body: 601; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 602; VLENUNK-NEXT: [[V:%.*]] = load i64, ptr [[B:%.*]], align 8 603; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 604; VLENUNK-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 605; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 606; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 607; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 608; VLENUNK: for.end: 609; VLENUNK-NEXT: [[V_LCSSA:%.*]] = phi i64 [ [[V]], [[FOR_BODY]] ] 610; VLENUNK-NEXT: ret i64 [[V_LCSSA]] 611; 612; VLEN128-LABEL: @uniform_load( 613; VLEN128-NEXT: entry: 614; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 615; VLEN128: vector.ph: 616; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 617; VLEN128: vector.body: 618; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 619; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 620; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 621; VLEN128-NEXT: [[TMP2:%.*]] = load i64, ptr [[B:%.*]], align 8 622; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i32 0 623; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 624; VLEN128-NEXT: [[TMP3:%.*]] = load i64, ptr [[B]], align 8 625; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i32 0 626; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer 627; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 628; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 629; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0 630; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 631; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 2 632; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP7]], align 8 633; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 634; VLEN128-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 635; VLEN128-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] 636; VLEN128: middle.block: 637; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 638; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 639; VLEN128: scalar.ph: 640; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 641; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 642; VLEN128: for.body: 643; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 644; VLEN128-NEXT: [[V:%.*]] = load i64, ptr [[B]], align 8 645; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 646; VLEN128-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 647; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 648; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 649; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] 650; VLEN128: for.end: 651; VLEN128-NEXT: [[V_LCSSA:%.*]] = phi i64 [ [[V]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 652; VLEN128-NEXT: ret i64 [[V_LCSSA]] 653; 654entry: 655 br label %for.body 656 657for.body: 658 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 659 %v = load i64, ptr %b, align 8 660 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 661 store i64 %v, ptr %arrayidx 662 %iv.next = add nuw nsw i64 %iv, 1 663 %exitcond.not = icmp eq i64 %iv.next, 1024 664 br i1 %exitcond.not, label %for.end, label %for.body 665 666for.end: 667 ret i64 %v 668} 669 670define i64 @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) { 671; VLENUNK-LABEL: @uniform_load_unaligned( 672; VLENUNK-NEXT: entry: 673; VLENUNK-NEXT: br label [[FOR_BODY:%.*]] 674; VLENUNK: for.body: 675; VLENUNK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 676; VLENUNK-NEXT: [[V:%.*]] = load i64, ptr [[B:%.*]], align 1 677; VLENUNK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]] 678; VLENUNK-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 679; VLENUNK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 680; VLENUNK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 681; VLENUNK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] 682; VLENUNK: for.end: 683; VLENUNK-NEXT: [[V_LCSSA:%.*]] = phi i64 [ [[V]], [[FOR_BODY]] ] 684; VLENUNK-NEXT: ret i64 [[V_LCSSA]] 685; 686; VLEN128-LABEL: @uniform_load_unaligned( 687; VLEN128-NEXT: entry: 688; VLEN128-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 689; VLEN128: vector.ph: 690; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] 691; VLEN128: vector.body: 692; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 693; VLEN128-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 694; VLEN128-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 695; VLEN128-NEXT: [[TMP2:%.*]] = load i64, ptr [[B:%.*]], align 1 696; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i32 0 697; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 698; VLEN128-NEXT: [[TMP3:%.*]] = load i64, ptr [[B]], align 1 699; VLEN128-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i32 0 700; VLEN128-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer 701; VLEN128-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 702; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] 703; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0 704; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 705; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 2 706; VLEN128-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP7]], align 8 707; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 708; VLEN128-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 709; VLEN128-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] 710; VLEN128: middle.block: 711; VLEN128-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 712; VLEN128-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 713; VLEN128: scalar.ph: 714; VLEN128-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 715; VLEN128-NEXT: br label [[FOR_BODY:%.*]] 716; VLEN128: for.body: 717; VLEN128-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 718; VLEN128-NEXT: [[V:%.*]] = load i64, ptr [[B]], align 1 719; VLEN128-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] 720; VLEN128-NEXT: store i64 [[V]], ptr [[ARRAYIDX]], align 8 721; VLEN128-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 722; VLEN128-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 723; VLEN128-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] 724; VLEN128: for.end: 725; VLEN128-NEXT: [[V_LCSSA:%.*]] = phi i64 [ [[V]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 726; VLEN128-NEXT: ret i64 [[V_LCSSA]] 727; 728entry: 729 br label %for.body 730 731for.body: 732 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 733 %v = load i64, ptr %b, align 1 734 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv 735 store i64 %v, ptr %arrayidx 736 %iv.next = add nuw nsw i64 %iv, 1 737 %exitcond.not = icmp eq i64 %iv.next, 1024 738 br i1 %exitcond.not, label %for.end, label %for.body 739 740for.end: 741 ret i64 %v 742} 743