1; RUN: opt -S -mtriple=amdgcn-- -mcpu=bonaire -loop-reduce < %s | FileCheck -check-prefix=OPT %s 2 3target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64" 4 5; Make sure the pointer / address space of AtomicRMW is considered 6 7; OPT-LABEL: @test_local_atomicrmw_addressing_loop_uniform_index_max_offset_i32( 8 9; OPT-NOT: getelementptr 10 11; OPT: .lr.ph: 12; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ] 13; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ] 14; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ] 15; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383 16; OPT: %tmp4 = atomicrmw add i32 addrspace(3)* %scevgep4, i32 undef seq_cst 17; OPT: %tmp7 = atomicrmw add i32 addrspace(3)* %lsr.iv1, i32 undef seq_cst 18; OPT: %0 = atomicrmw add i32 addrspace(3)* %lsr.iv1, i32 %tmp8 seq_cst 19; OPT: br i1 %exitcond 20define amdgpu_kernel void @test_local_atomicrmw_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 { 21bb: 22 %tmp = icmp sgt i32 %n, 0 23 br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge 24 25.lr.ph.preheader: ; preds = %bb 26 br label %.lr.ph 27 28._crit_edge.loopexit: ; preds = %.lr.ph 29 br label %._crit_edge 30 31._crit_edge: ; preds = %._crit_edge.loopexit, %bb 32 ret void 33 34.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader 35 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ] 36 %tmp1 = add nuw nsw i32 %indvars.iv, 16383 37 %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1 38 %tmp4 = atomicrmw add i32 addrspace(3)* %tmp3, i32 undef seq_cst 39 %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv 40 %tmp7 = atomicrmw add i32 addrspace(3)* %tmp6, i32 undef seq_cst 41 %tmp8 = add nsw i32 %tmp7, %tmp4 42 atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst 43 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 44 %exitcond = icmp eq i32 %indvars.iv.next, %n 45 br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph 46} 47 48; OPT-LABEL: test_local_cmpxchg_addressing_loop_uniform_index_max_offset_i32( 49; OPT-NOT: getelementptr 50 51; OPT: .lr.ph: 52; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ] 53; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ] 54; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ] 55; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383 56; OPT: %tmp4 = cmpxchg i32 addrspace(3)* %scevgep4, i32 undef, i32 undef seq_cst monotonic 57define amdgpu_kernel void @test_local_cmpxchg_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 { 58bb: 59 %tmp = icmp sgt i32 %n, 0 60 br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge 61 62.lr.ph.preheader: ; preds = %bb 63 br label %.lr.ph 64 65._crit_edge.loopexit: ; preds = %.lr.ph 66 br label %._crit_edge 67 68._crit_edge: ; preds = %._crit_edge.loopexit, %bb 69 ret void 70 71.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader 72 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ] 73 %tmp1 = add nuw nsw i32 %indvars.iv, 16383 74 %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1 75 %tmp4 = cmpxchg i32 addrspace(3)* %tmp3, i32 undef, i32 undef seq_cst monotonic 76 %tmp4.0 = extractvalue { i32, i1 } %tmp4, 0 77 %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv 78 %tmp7 = cmpxchg i32 addrspace(3)* %tmp6, i32 undef, i32 undef seq_cst monotonic 79 %tmp7.0 = extractvalue { i32, i1 } %tmp7, 0 80 %tmp8 = add nsw i32 %tmp7.0, %tmp4.0 81 atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst 82 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 83 %exitcond = icmp eq i32 %indvars.iv.next, %n 84 br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph 85} 86 87; OPT-LABEL: @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32( 88; OPT-NOT: getelementptr 89 90; OPT: .lr.ph: 91; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ] 92; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ] 93; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ] 94; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383 95; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %scevgep4, i32 undef, i32 0, i32 0, i1 false) 96; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false) 97define amdgpu_kernel void @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 { 98bb: 99 %tmp = icmp sgt i32 %n, 0 100 br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge 101 102.lr.ph.preheader: ; preds = %bb 103 br label %.lr.ph 104 105._crit_edge.loopexit: ; preds = %.lr.ph 106 br label %._crit_edge 107 108._crit_edge: ; preds = %._crit_edge.loopexit, %bb 109 ret void 110 111.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader 112 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ] 113 %tmp1 = add nuw nsw i32 %indvars.iv, 16383 114 %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1 115 %tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false) 116 %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv 117 %tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false) 118 %tmp8 = add nsw i32 %tmp7, %tmp4 119 atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst 120 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 121 %exitcond = icmp eq i32 %indvars.iv.next, %n 122 br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph 123} 124 125; OPT-LABEL: @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32( 126; OPT-NOT: getelementptr 127 128; OPT: .lr.ph: 129; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ] 130; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ] 131; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ] 132; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383 133; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %scevgep4, i32 undef, i32 0, i32 0, i1 false) 134; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false) 135define amdgpu_kernel void @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 { 136bb: 137 %tmp = icmp sgt i32 %n, 0 138 br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge 139 140.lr.ph.preheader: ; preds = %bb 141 br label %.lr.ph 142 143._crit_edge.loopexit: ; preds = %.lr.ph 144 br label %._crit_edge 145 146._crit_edge: ; preds = %._crit_edge.loopexit, %bb 147 ret void 148 149.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader 150 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ] 151 %tmp1 = add nuw nsw i32 %indvars.iv, 16383 152 %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1 153 %tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false) 154 %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv 155 %tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false) 156 %tmp8 = add nsw i32 %tmp7, %tmp4 157 atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst 158 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 159 %exitcond = icmp eq i32 %indvars.iv.next, %n 160 br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph 161} 162 163declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1 164declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1 165 166attributes #0 = { nounwind } 167attributes #1 = { nounwind argmemonly } 168