1; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s 2; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 3; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s 4; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s 5; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s 6 7declare i32 @llvm.amdgcn.workitem.id.x() 8 9; Show what the atomic optimization pass will do for global pointers. 10 11; GCN-LABEL: add_i32_constant: 12; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 13; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 14; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 15; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 16; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 17; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 18; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 19; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 20; GCN: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5 21; GCN: v_mov_b32_e32 v[[data:[0-9]+]], s[[value]] 22; GCN: {{flat|buffer|global}}_atomic_add v[[data]] 23define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) { 24entry: 25 %old = atomicrmw add i32 addrspace(1)* %inout, i32 5 acq_rel 26 store i32 %old, i32 addrspace(1)* %out 27 ret void 28} 29 30; GCN-LABEL: add_i32_uniform: 31; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 32; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 33; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 34; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 35; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 36; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 37; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 38; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 39; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 40; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 41; GCN: {{flat|buffer|global}}_atomic_add v[[value]] 42define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 addrspace(1)* %inout, i32 %additive) { 43entry: 44 %old = atomicrmw add i32 addrspace(1)* %inout, i32 %additive acq_rel 45 store i32 %old, i32 addrspace(1)* %out 46 ret void 47} 48 49; GCN-LABEL: add_i32_varying: 50; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 51; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 52; GFX7LESS-NOT: s_bcnt1_i32_b64 53; GFX7LESS: buffer_atomic_add v{{[0-9]+}} 54; DPPCOMB: v_add_u32_dpp 55; DPPCOMB: v_add_u32_dpp 56; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 57; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 58; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 59; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 60; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 61; GFX8MORE: buffer_atomic_add v[[value]] 62define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) { 63entry: 64 %lane = call i32 @llvm.amdgcn.workitem.id.x() 65 %old = atomicrmw add i32 addrspace(1)* %inout, i32 %lane acq_rel 66 store i32 %old, i32 addrspace(1)* %out 67 ret void 68} 69 70; GCN-LABEL: add_i64_constant: 71; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 72; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 73; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 74; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 75; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 76; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 77; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 78; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 79; GCN-DAG: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5 80; GCN-DAG: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5 81; GCN: v_mov_b32_e32 v[[value_lo:[0-9]+]], s[[value]] 82; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}} 83define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) { 84entry: 85 %old = atomicrmw add i64 addrspace(1)* %inout, i64 5 acq_rel 86 store i64 %old, i64 addrspace(1)* %out 87 ret void 88} 89 90; GCN-LABEL: add_i64_uniform: 91; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 92; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 93; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 94; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 95; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 96; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 97; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 98; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 99; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}} 100define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %additive) { 101entry: 102 %old = atomicrmw add i64 addrspace(1)* %inout, i64 %additive acq_rel 103 store i64 %old, i64 addrspace(1)* %out 104 ret void 105} 106 107; GCN-LABEL: add_i64_varying: 108; GCN-NOT: v_mbcnt_lo_u32_b32 109; GCN-NOT: v_mbcnt_hi_u32_b32 110; GCN-NOT: s_bcnt1_i32_b64 111; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}} 112define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) { 113entry: 114 %lane = call i32 @llvm.amdgcn.workitem.id.x() 115 %zext = zext i32 %lane to i64 116 %old = atomicrmw add i64 addrspace(1)* %inout, i64 %zext acq_rel 117 store i64 %old, i64 addrspace(1)* %out 118 ret void 119} 120 121; GCN-LABEL: sub_i32_constant: 122; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 123; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 124; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 125; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 126; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 127; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 128; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 129; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 130; GCN: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5 131; GCN: v_mov_b32_e32 v[[data:[0-9]+]], s[[value]] 132; GCN: {{flat|buffer|global}}_atomic_sub v[[data]] 133define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) { 134entry: 135 %old = atomicrmw sub i32 addrspace(1)* %inout, i32 5 acq_rel 136 store i32 %old, i32 addrspace(1)* %out 137 ret void 138} 139 140; GCN-LABEL: sub_i32_uniform: 141; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 142; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 143; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 144; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 145; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 146; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 147; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 148; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 149; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]] 150; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 151; GCN: {{flat|buffer|global}}_atomic_sub v[[value]] 152define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 addrspace(1)* %inout, i32 %subitive) { 153entry: 154 %old = atomicrmw sub i32 addrspace(1)* %inout, i32 %subitive acq_rel 155 store i32 %old, i32 addrspace(1)* %out 156 ret void 157} 158 159; GCN-LABEL: sub_i32_varying: 160; GFX7LESS-NOT: v_mbcnt_lo_u32_b32 161; GFX7LESS-NOT: v_mbcnt_hi_u32_b32 162; GFX7LESS-NOT: s_bcnt1_i32_b64 163; GFX7LESS: buffer_atomic_sub v{{[0-9]+}} 164; DPPCOMB: v_add_u32_dpp 165; DPPCOMB: v_add_u32_dpp 166; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31 167; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63 168; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]] 169; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]] 170; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]] 171; GFX8MORE: buffer_atomic_sub v[[value]] 172define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) { 173entry: 174 %lane = call i32 @llvm.amdgcn.workitem.id.x() 175 %old = atomicrmw sub i32 addrspace(1)* %inout, i32 %lane acq_rel 176 store i32 %old, i32 addrspace(1)* %out 177 ret void 178} 179 180; GCN-LABEL: sub_i64_constant: 181; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 182; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 183; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 184; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 185; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 186; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 187; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 188; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 189; GCN-DAG: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5 190; GCN-DAG: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5 191; GCN: v_mov_b32_e32 v[[value_lo:[0-9]+]], s[[value]] 192; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}} 193define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) { 194entry: 195 %old = atomicrmw sub i64 addrspace(1)* %inout, i64 5 acq_rel 196 store i64 %old, i64 addrspace(1)* %out 197 ret void 198} 199 200; GCN-LABEL: sub_i64_uniform: 201; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo 202; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec 203; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0 204; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]] 205; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]] 206; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc 207; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]] 208; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}} 209; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}} 210define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %subitive) { 211entry: 212 %old = atomicrmw sub i64 addrspace(1)* %inout, i64 %subitive acq_rel 213 store i64 %old, i64 addrspace(1)* %out 214 ret void 215} 216 217; GCN-LABEL: sub_i64_varying: 218; GCN-NOT: v_mbcnt_lo_u32_b32 219; GCN-NOT: v_mbcnt_hi_u32_b32 220; GCN-NOT: s_bcnt1_i32_b64 221; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}} 222define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) { 223entry: 224 %lane = call i32 @llvm.amdgcn.workitem.id.x() 225 %zext = zext i32 %lane to i64 226 %old = atomicrmw sub i64 addrspace(1)* %inout, i64 %zext acq_rel 227 store i64 %old, i64 addrspace(1)* %out 228 ret void 229} 230