1; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s 2; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 3; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 4; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s 5; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s 6 7declare i32 @llvm.amdgcn.workitem.id.x() 8declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32) 9declare i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32, i32) 10 11; Show what the atomic optimization pass will do for struct buffers. 12 13; GCN-LABEL: add_i32_constant: 14; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 15; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 16; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 17; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 18; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 19; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 20; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 21; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 22; GCN: s_mul_i32 s[[popcount]], s[[popcount]], 5 23; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]] 24; GCN: buffer_atomic_add v[[value]] 25define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) { 26entry: 27 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 28 store i32 %old, i32 addrspace(1)* %out 29 ret void 30} 31 32; GCN-LABEL: add_i32_uniform: 33; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 34; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 35; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 36; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 37; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 38; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 39; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 40; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 41; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 42; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 43; GCN: buffer_atomic_add v[[value]] 44define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, <4 x i32> %inout, i32 %additive) { 45entry: 46 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 %additive, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 47 store i32 %old, i32 addrspace(1)* %out 48 ret void 49} 50 51; GCN-LABEL: add_i32_varying_vdata: 52; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 53; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 54; GFX7LESS-NOT: s_bcnt1_i32_b64 55; GFX7LESS: buffer_atomic_add v{{[0-9]+}} 56; DPPCOMB: v_add_u32_dpp 57; DPPCOMB: v_add_u32_dpp 58; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 59; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 60; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 61; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 62; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 63; GFX8MORE: buffer_atomic_add v[[value]] 64define amdgpu_kernel void @add_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) { 65entry: 66 %lane = call i32 @llvm.amdgcn.workitem.id.x() 67 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 %lane, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 68 store i32 %old, i32 addrspace(1)* %out 69 ret void 70} 71 72; GCN-LABEL: add_i32_varying_vindex: 73; GCN-NOT: v_mbcnt_lo_u32_b32 74; GCN-NOT: v_mbcnt_hi_u32_b32 75; GCN-NOT: s_bcnt1_i32_b64 76; GCN: buffer_atomic_add v{{[0-9]+}} 77define amdgpu_kernel void @add_i32_varying_vindex(i32 addrspace(1)* %out, <4 x i32> %inout) { 78entry: 79 %lane = call i32 @llvm.amdgcn.workitem.id.x() 80 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 1, <4 x i32> %inout, i32 %lane, i32 0, i32 0, i32 0) 81 store i32 %old, i32 addrspace(1)* %out 82 ret void 83} 84 85; GCN-LABEL: add_i32_varying_offset: 86; GCN-NOT: v_mbcnt_lo_u32_b32 87; GCN-NOT: v_mbcnt_hi_u32_b32 88; GCN-NOT: s_bcnt1_i32_b64 89; GCN: buffer_atomic_add v{{[0-9]+}} 90define amdgpu_kernel void @add_i32_varying_offset(i32 addrspace(1)* %out, <4 x i32> %inout) { 91entry: 92 %lane = call i32 @llvm.amdgcn.workitem.id.x() 93 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 1, <4 x i32> %inout, i32 0, i32 %lane, i32 0, i32 0) 94 store i32 %old, i32 addrspace(1)* %out 95 ret void 96} 97 98; GCN-LABEL: sub_i32_constant: 99; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 100; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 101; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 102; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 103; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 104; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 105; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 106; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 107; GCN: s_mul_i32 s[[popcount]], s[[popcount]], 5 108; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]] 109; GCN: buffer_atomic_sub v[[value]] 110define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) { 111entry: 112 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 113 store i32 %old, i32 addrspace(1)* %out 114 ret void 115} 116 117; GCN-LABEL: sub_i32_uniform: 118; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 119; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 120; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 121; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 122; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 123; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 124; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 125; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 126; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 127; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 128; GCN: buffer_atomic_sub v[[value]] 129define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, <4 x i32> %inout, i32 %subitive) { 130entry: 131 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 %subitive, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 132 store i32 %old, i32 addrspace(1)* %out 133 ret void 134} 135 136; GCN-LABEL: sub_i32_varying_vdata: 137; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 138; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 139; GFX7LESS-NOT: s_bcnt1_i32_b64 140; GFX7LESS: buffer_atomic_sub v{{[0-9]+}} 141; DPPCOMB: v_add_u32_dpp 142; DPPCOMB: v_add_u32_dpp 143; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 144; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 145; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 146; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 147; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 148; GFX8MORE: buffer_atomic_sub v[[value]] 149define amdgpu_kernel void @sub_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) { 150entry: 151 %lane = call i32 @llvm.amdgcn.workitem.id.x() 152 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 %lane, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0) 153 store i32 %old, i32 addrspace(1)* %out 154 ret void 155} 156 157; GCN-LABEL: sub_i32_varying_vindex: 158; GCN-NOT: v_mbcnt_lo_u32_b32 159; GCN-NOT: v_mbcnt_hi_u32_b32 160; GCN-NOT: s_bcnt1_i32_b64 161; GCN: buffer_atomic_sub v{{[0-9]+}} 162define amdgpu_kernel void @sub_i32_varying_vindex(i32 addrspace(1)* %out, <4 x i32> %inout) { 163entry: 164 %lane = call i32 @llvm.amdgcn.workitem.id.x() 165 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 1, <4 x i32> %inout, i32 %lane, i32 0, i32 0, i32 0) 166 store i32 %old, i32 addrspace(1)* %out 167 ret void 168} 169 170; GCN-LABEL: sub_i32_varying_offset: 171; GCN-NOT: v_mbcnt_lo_u32_b32 172; GCN-NOT: v_mbcnt_hi_u32_b32 173; GCN-NOT: s_bcnt1_i32_b64 174; GCN: buffer_atomic_sub v{{[0-9]+}} 175define amdgpu_kernel void @sub_i32_varying_offset(i32 addrspace(1)* %out, <4 x i32> %inout) { 176entry: 177 %lane = call i32 @llvm.amdgcn.workitem.id.x() 178 %old = call i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32 1, <4 x i32> %inout, i32 0, i32 %lane, i32 0, i32 0) 179 store i32 %old, i32 addrspace(1)* %out 180 ret void 181} 182