1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS %s 3; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s 4; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s 5; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s 6; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032 %s 7 8declare i32 @llvm.amdgcn.workitem.id.x() 9 10@local_var32 = addrspace(3) global i32 undef, align 4 11@local_var64 = addrspace(3) global i64 undef, align 8 12 13; Show what the atomic optimization pass will do for local pointers. 14 15define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) { 16; 17; 18; GFX7LESS-LABEL: add_i32_constant: 19; GFX7LESS: ; %bb.0: ; %entry 20; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec 21; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 22; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 23; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 24; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 25; GFX7LESS-NEXT: ; implicit-def: $vgpr1 26; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc 27; GFX7LESS-NEXT: s_cbranch_execz BB0_2 28; GFX7LESS-NEXT: ; %bb.1: 29; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 30; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 31; GFX7LESS-NEXT: v_mul_u32_u24_e64 v2, s2, 5 32; GFX7LESS-NEXT: s_mov_b32 m0, -1 33; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 34; GFX7LESS-NEXT: ds_add_rtn_u32 v1, v1, v2 35; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 36; GFX7LESS-NEXT: buffer_wbinvl1 37; GFX7LESS-NEXT: BB0_2: 38; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5] 39; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v1 40; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 41; GFX7LESS-NEXT: v_mad_u32_u24 v0, v0, 5, s2 42; GFX7LESS-NEXT: s_mov_b32 s2, -1 43; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 44; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 45; GFX7LESS-NEXT: s_endpgm 46; 47; GFX8-LABEL: add_i32_constant: 48; GFX8: ; %bb.0: ; %entry 49; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 50; GFX8-NEXT: s_mov_b64 s[2:3], exec 51; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 52; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 53; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 54; GFX8-NEXT: ; implicit-def: $vgpr1 55; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 56; GFX8-NEXT: s_cbranch_execz BB0_2 57; GFX8-NEXT: ; %bb.1: 58; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 59; GFX8-NEXT: v_mul_u32_u24_e64 v1, s2, 5 60; GFX8-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 61; GFX8-NEXT: s_mov_b32 m0, -1 62; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 63; GFX8-NEXT: ds_add_rtn_u32 v1, v2, v1 64; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 65; GFX8-NEXT: buffer_wbinvl1_vol 66; GFX8-NEXT: BB0_2: 67; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 68; GFX8-NEXT: v_readfirstlane_b32 s2, v1 69; GFX8-NEXT: v_mad_u32_u24 v0, v0, 5, s2 70; GFX8-NEXT: s_mov_b32 s3, 0xf000 71; GFX8-NEXT: s_mov_b32 s2, -1 72; GFX8-NEXT: s_nop 1 73; GFX8-NEXT: s_waitcnt lgkmcnt(0) 74; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 75; GFX8-NEXT: s_endpgm 76; 77; GFX9-LABEL: add_i32_constant: 78; GFX9: ; %bb.0: ; %entry 79; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 80; GFX9-NEXT: s_mov_b64 s[2:3], exec 81; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 82; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 83; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 84; GFX9-NEXT: ; implicit-def: $vgpr1 85; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 86; GFX9-NEXT: s_cbranch_execz BB0_2 87; GFX9-NEXT: ; %bb.1: 88; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 89; GFX9-NEXT: v_mul_u32_u24_e64 v1, s2, 5 90; GFX9-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 91; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 92; GFX9-NEXT: ds_add_rtn_u32 v1, v2, v1 93; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 94; GFX9-NEXT: buffer_wbinvl1_vol 95; GFX9-NEXT: BB0_2: 96; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 97; GFX9-NEXT: v_readfirstlane_b32 s2, v1 98; GFX9-NEXT: v_mad_u32_u24 v0, v0, 5, s2 99; GFX9-NEXT: s_mov_b32 s3, 0xf000 100; GFX9-NEXT: s_mov_b32 s2, -1 101; GFX9-NEXT: s_nop 1 102; GFX9-NEXT: s_waitcnt lgkmcnt(0) 103; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 104; GFX9-NEXT: s_endpgm 105; 106; GFX1064-LABEL: add_i32_constant: 107; GFX1064: ; %bb.0: ; %entry 108; GFX1064-NEXT: s_mov_b64 s[2:3], exec 109; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 110; GFX1064-NEXT: ; implicit-def: $vgpr1 111; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 112; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 113; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 114; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 115; GFX1064-NEXT: s_cbranch_execz BB0_2 116; GFX1064-NEXT: ; %bb.1: 117; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 118; GFX1064-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 119; GFX1064-NEXT: v_mul_u32_u24_e64 v1, s2, 5 120; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 121; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 122; GFX1064-NEXT: ds_add_rtn_u32 v1, v2, v1 123; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 124; GFX1064-NEXT: buffer_gl0_inv 125; GFX1064-NEXT: buffer_gl1_inv 126; GFX1064-NEXT: BB0_2: 127; GFX1064-NEXT: v_nop 128; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 129; GFX1064-NEXT: v_readfirstlane_b32 s2, v1 130; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 131; GFX1064-NEXT: v_mad_u32_u24 v0, v0, 5, s2 132; GFX1064-NEXT: s_mov_b32 s2, -1 133; GFX1064-NEXT: s_nop 1 134; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 135; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 136; GFX1064-NEXT: s_endpgm 137; 138; GFX1032-LABEL: add_i32_constant: 139; GFX1032: ; %bb.0: ; %entry 140; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 141; GFX1032-NEXT: s_mov_b32 s2, exec_lo 142; GFX1032-NEXT: ; implicit-def: $vcc_hi 143; GFX1032-NEXT: ; implicit-def: $vgpr1 144; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 145; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 146; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo 147; GFX1032-NEXT: s_cbranch_execz BB0_2 148; GFX1032-NEXT: ; %bb.1: 149; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2 150; GFX1032-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 151; GFX1032-NEXT: v_mul_u32_u24_e64 v1, s2, 5 152; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 153; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 154; GFX1032-NEXT: ds_add_rtn_u32 v1, v2, v1 155; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 156; GFX1032-NEXT: buffer_gl0_inv 157; GFX1032-NEXT: buffer_gl1_inv 158; GFX1032-NEXT: BB0_2: 159; GFX1032-NEXT: v_nop 160; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3 161; GFX1032-NEXT: v_readfirstlane_b32 s2, v1 162; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 163; GFX1032-NEXT: v_mad_u32_u24 v0, v0, 5, s2 164; GFX1032-NEXT: s_mov_b32 s2, -1 165; GFX1032-NEXT: s_nop 1 166; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 167; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 168; GFX1032-NEXT: s_endpgm 169entry: 170 %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 5 acq_rel 171 store i32 %old, i32 addrspace(1)* %out 172 ret void 173} 174 175define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive) { 176; 177; 178; GFX7LESS-LABEL: add_i32_uniform: 179; GFX7LESS: ; %bb.0: ; %entry 180; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec 181; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 182; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0xb 183; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 184; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 185; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 186; GFX7LESS-NEXT: ; implicit-def: $vgpr1 187; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc 188; GFX7LESS-NEXT: s_cbranch_execz BB1_2 189; GFX7LESS-NEXT: ; %bb.1: 190; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[6:7] 191; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 192; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3 193; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 194; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3 195; GFX7LESS-NEXT: s_mov_b32 m0, -1 196; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 197; GFX7LESS-NEXT: ds_add_rtn_u32 v1, v1, v2 198; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 199; GFX7LESS-NEXT: buffer_wbinvl1 200; GFX7LESS-NEXT: BB1_2: 201; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1] 202; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1 203; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 204; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0 205; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000 206; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s0, v0 207; GFX7LESS-NEXT: s_mov_b32 s6, -1 208; GFX7LESS-NEXT: buffer_store_dword v0, off, s[4:7], 0 209; GFX7LESS-NEXT: s_endpgm 210; 211; GFX8-LABEL: add_i32_uniform: 212; GFX8: ; %bb.0: ; %entry 213; GFX8-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 214; GFX8-NEXT: s_load_dword s0, s[0:1], 0x2c 215; GFX8-NEXT: s_mov_b64 s[2:3], exec 216; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 217; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 218; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 219; GFX8-NEXT: ; implicit-def: $vgpr1 220; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc 221; GFX8-NEXT: s_cbranch_execz BB1_2 222; GFX8-NEXT: ; %bb.1: 223; GFX8-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 224; GFX8-NEXT: s_waitcnt lgkmcnt(0) 225; GFX8-NEXT: s_mul_i32 s1, s0, s1 226; GFX8-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 227; GFX8-NEXT: v_mov_b32_e32 v2, s1 228; GFX8-NEXT: s_mov_b32 m0, -1 229; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 230; GFX8-NEXT: ds_add_rtn_u32 v1, v1, v2 231; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 232; GFX8-NEXT: buffer_wbinvl1_vol 233; GFX8-NEXT: BB1_2: 234; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] 235; GFX8-NEXT: s_waitcnt lgkmcnt(0) 236; GFX8-NEXT: v_mul_lo_u32 v0, s0, v0 237; GFX8-NEXT: v_readfirstlane_b32 s0, v1 238; GFX8-NEXT: s_mov_b32 s7, 0xf000 239; GFX8-NEXT: s_mov_b32 s6, -1 240; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0 241; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0 242; GFX8-NEXT: s_endpgm 243; 244; GFX9-LABEL: add_i32_uniform: 245; GFX9: ; %bb.0: ; %entry 246; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 247; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c 248; GFX9-NEXT: s_mov_b64 s[2:3], exec 249; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 250; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 251; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 252; GFX9-NEXT: ; implicit-def: $vgpr1 253; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc 254; GFX9-NEXT: s_cbranch_execz BB1_2 255; GFX9-NEXT: ; %bb.1: 256; GFX9-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 257; GFX9-NEXT: s_waitcnt lgkmcnt(0) 258; GFX9-NEXT: s_mul_i32 s1, s0, s1 259; GFX9-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 260; GFX9-NEXT: v_mov_b32_e32 v2, s1 261; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 262; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2 263; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 264; GFX9-NEXT: buffer_wbinvl1_vol 265; GFX9-NEXT: BB1_2: 266; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] 267; GFX9-NEXT: s_waitcnt lgkmcnt(0) 268; GFX9-NEXT: v_mul_lo_u32 v0, s0, v0 269; GFX9-NEXT: v_readfirstlane_b32 s0, v1 270; GFX9-NEXT: s_mov_b32 s7, 0xf000 271; GFX9-NEXT: s_mov_b32 s6, -1 272; GFX9-NEXT: v_add_u32_e32 v0, s0, v0 273; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0 274; GFX9-NEXT: s_endpgm 275; 276; GFX1064-LABEL: add_i32_uniform: 277; GFX1064: ; %bb.0: ; %entry 278; GFX1064-NEXT: s_mov_b64 s[2:3], exec 279; GFX1064-NEXT: s_clause 0x1 280; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 281; GFX1064-NEXT: s_load_dword s0, s[0:1], 0x2c 282; GFX1064-NEXT: ; implicit-def: $vgpr1 283; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 284; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 285; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 286; GFX1064-NEXT: s_and_saveexec_b64 s[6:7], vcc 287; GFX1064-NEXT: s_cbranch_execz BB1_2 288; GFX1064-NEXT: ; %bb.1: 289; GFX1064-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 290; GFX1064-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 291; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 292; GFX1064-NEXT: s_mul_i32 s1, s0, s1 293; GFX1064-NEXT: v_mov_b32_e32 v2, s1 294; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 295; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 296; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2 297; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 298; GFX1064-NEXT: buffer_gl0_inv 299; GFX1064-NEXT: buffer_gl1_inv 300; GFX1064-NEXT: BB1_2: 301; GFX1064-NEXT: v_nop 302; GFX1064-NEXT: s_or_b64 exec, exec, s[6:7] 303; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 304; GFX1064-NEXT: v_mul_lo_u32 v0, s0, v0 305; GFX1064-NEXT: v_readfirstlane_b32 s0, v1 306; GFX1064-NEXT: s_mov_b32 s7, 0x31016000 307; GFX1064-NEXT: s_mov_b32 s6, -1 308; GFX1064-NEXT: v_add_nc_u32_e32 v0, s0, v0 309; GFX1064-NEXT: buffer_store_dword v0, off, s[4:7], 0 310; GFX1064-NEXT: s_endpgm 311; 312; GFX1032-LABEL: add_i32_uniform: 313; GFX1032: ; %bb.0: ; %entry 314; GFX1032-NEXT: s_clause 0x1 315; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 316; GFX1032-NEXT: s_load_dword s0, s[0:1], 0x2c 317; GFX1032-NEXT: s_mov_b32 s2, exec_lo 318; GFX1032-NEXT: ; implicit-def: $vcc_hi 319; GFX1032-NEXT: ; implicit-def: $vgpr1 320; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 321; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 322; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo 323; GFX1032-NEXT: s_cbranch_execz BB1_2 324; GFX1032-NEXT: ; %bb.1: 325; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2 326; GFX1032-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 327; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 328; GFX1032-NEXT: s_mul_i32 s2, s0, s2 329; GFX1032-NEXT: v_mov_b32_e32 v2, s2 330; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 331; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 332; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2 333; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 334; GFX1032-NEXT: buffer_gl0_inv 335; GFX1032-NEXT: buffer_gl1_inv 336; GFX1032-NEXT: BB1_2: 337; GFX1032-NEXT: v_nop 338; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1 339; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 340; GFX1032-NEXT: v_mul_lo_u32 v0, s0, v0 341; GFX1032-NEXT: v_readfirstlane_b32 s0, v1 342; GFX1032-NEXT: s_mov_b32 s7, 0x31016000 343; GFX1032-NEXT: s_mov_b32 s6, -1 344; GFX1032-NEXT: v_add_nc_u32_e32 v0, s0, v0 345; GFX1032-NEXT: buffer_store_dword v0, off, s[4:7], 0 346; GFX1032-NEXT: s_endpgm 347entry: 348 %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %additive acq_rel 349 store i32 %old, i32 addrspace(1)* %out 350 ret void 351} 352 353define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) { 354; 355; 356; GFX7LESS-LABEL: add_i32_varying: 357; GFX7LESS: ; %bb.0: ; %entry 358; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 359; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 360; GFX7LESS-NEXT: s_mov_b32 m0, -1 361; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 362; GFX7LESS-NEXT: ds_add_rtn_u32 v0, v1, v0 363; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 364; GFX7LESS-NEXT: buffer_wbinvl1 365; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 366; GFX7LESS-NEXT: s_mov_b32 s2, -1 367; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 368; GFX7LESS-NEXT: s_endpgm 369; 370; GFX8-LABEL: add_i32_varying: 371; GFX8: ; %bb.0: ; %entry 372; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 373; GFX8-NEXT: s_mov_b64 s[2:3], exec 374; GFX8-NEXT: v_mov_b32_e32 v2, v0 375; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 376; GFX8-NEXT: v_mov_b32_e32 v1, 0 377; GFX8-NEXT: s_mov_b64 exec, s[4:5] 378; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 379; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 380; GFX8-NEXT: s_not_b64 exec, exec 381; GFX8-NEXT: v_mov_b32_e32 v2, 0 382; GFX8-NEXT: s_not_b64 exec, exec 383; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 384; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 385; GFX8-NEXT: s_nop 1 386; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 387; GFX8-NEXT: s_nop 1 388; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 389; GFX8-NEXT: s_nop 1 390; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 391; GFX8-NEXT: s_nop 1 392; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 393; GFX8-NEXT: s_nop 1 394; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 395; GFX8-NEXT: v_readlane_b32 s2, v2, 63 396; GFX8-NEXT: s_nop 0 397; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 398; GFX8-NEXT: s_mov_b64 exec, s[4:5] 399; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 400; GFX8-NEXT: ; implicit-def: $vgpr0 401; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 402; GFX8-NEXT: s_cbranch_execz BB2_2 403; GFX8-NEXT: ; %bb.1: 404; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 405; GFX8-NEXT: v_mov_b32_e32 v3, s2 406; GFX8-NEXT: s_mov_b32 m0, -1 407; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 408; GFX8-NEXT: ds_add_rtn_u32 v0, v0, v3 409; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 410; GFX8-NEXT: buffer_wbinvl1_vol 411; GFX8-NEXT: BB2_2: 412; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 413; GFX8-NEXT: v_readfirstlane_b32 s2, v0 414; GFX8-NEXT: v_mov_b32_e32 v0, v1 415; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0 416; GFX8-NEXT: s_mov_b32 s3, 0xf000 417; GFX8-NEXT: s_mov_b32 s2, -1 418; GFX8-NEXT: s_nop 0 419; GFX8-NEXT: s_waitcnt lgkmcnt(0) 420; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 421; GFX8-NEXT: s_endpgm 422; 423; GFX9-LABEL: add_i32_varying: 424; GFX9: ; %bb.0: ; %entry 425; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 426; GFX9-NEXT: s_mov_b64 s[2:3], exec 427; GFX9-NEXT: v_mov_b32_e32 v2, v0 428; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 429; GFX9-NEXT: v_mov_b32_e32 v1, 0 430; GFX9-NEXT: s_mov_b64 exec, s[4:5] 431; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 432; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 433; GFX9-NEXT: s_not_b64 exec, exec 434; GFX9-NEXT: v_mov_b32_e32 v2, 0 435; GFX9-NEXT: s_not_b64 exec, exec 436; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 437; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 438; GFX9-NEXT: s_nop 1 439; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 440; GFX9-NEXT: s_nop 1 441; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 442; GFX9-NEXT: s_nop 1 443; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 444; GFX9-NEXT: s_nop 1 445; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 446; GFX9-NEXT: s_nop 1 447; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 448; GFX9-NEXT: v_readlane_b32 s2, v2, 63 449; GFX9-NEXT: s_nop 0 450; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 451; GFX9-NEXT: s_mov_b64 exec, s[4:5] 452; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 453; GFX9-NEXT: ; implicit-def: $vgpr0 454; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 455; GFX9-NEXT: s_cbranch_execz BB2_2 456; GFX9-NEXT: ; %bb.1: 457; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 458; GFX9-NEXT: v_mov_b32_e32 v3, s2 459; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 460; GFX9-NEXT: ds_add_rtn_u32 v0, v0, v3 461; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 462; GFX9-NEXT: buffer_wbinvl1_vol 463; GFX9-NEXT: BB2_2: 464; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 465; GFX9-NEXT: v_readfirstlane_b32 s2, v0 466; GFX9-NEXT: v_mov_b32_e32 v0, v1 467; GFX9-NEXT: v_add_u32_e32 v0, s2, v0 468; GFX9-NEXT: s_mov_b32 s3, 0xf000 469; GFX9-NEXT: s_mov_b32 s2, -1 470; GFX9-NEXT: s_nop 0 471; GFX9-NEXT: s_waitcnt lgkmcnt(0) 472; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 473; GFX9-NEXT: s_endpgm 474; 475; GFX1064-LABEL: add_i32_varying: 476; GFX1064: ; %bb.0: ; %entry 477; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 478; GFX1064-NEXT: s_mov_b64 s[2:3], exec 479; GFX1064-NEXT: v_mov_b32_e32 v2, v0 480; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 481; GFX1064-NEXT: v_mov_b32_e32 v1, 0 482; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 483; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 484; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 485; GFX1064-NEXT: s_not_b64 exec, exec 486; GFX1064-NEXT: v_mov_b32_e32 v2, 0 487; GFX1064-NEXT: s_not_b64 exec, exec 488; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 489; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 490; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 491; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 492; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 493; GFX1064-NEXT: v_mov_b32_e32 v3, v2 494; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 495; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 496; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 497; GFX1064-NEXT: v_mov_b32_e32 v3, s2 498; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 499; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 500; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 501; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 502; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 503; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 504; GFX1064-NEXT: s_mov_b32 s2, -1 505; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 506; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 507; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 508; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 509; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 510; GFX1064-NEXT: ; implicit-def: $vgpr0 511; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 512; GFX1064-NEXT: s_cbranch_execz BB2_2 513; GFX1064-NEXT: ; %bb.1: 514; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 515; GFX1064-NEXT: v_mov_b32_e32 v7, s3 516; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 517; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 518; GFX1064-NEXT: ds_add_rtn_u32 v0, v0, v7 519; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 520; GFX1064-NEXT: buffer_gl0_inv 521; GFX1064-NEXT: buffer_gl1_inv 522; GFX1064-NEXT: BB2_2: 523; GFX1064-NEXT: v_nop 524; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 525; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 526; GFX1064-NEXT: v_mov_b32_e32 v0, v1 527; GFX1064-NEXT: v_add_nc_u32_e32 v0, s3, v0 528; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 529; GFX1064-NEXT: s_nop 1 530; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 531; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 532; GFX1064-NEXT: s_endpgm 533; 534; GFX1032-LABEL: add_i32_varying: 535; GFX1032: ; %bb.0: ; %entry 536; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 537; GFX1032-NEXT: s_mov_b32 s2, exec_lo 538; GFX1032-NEXT: ; implicit-def: $vcc_hi 539; GFX1032-NEXT: v_mov_b32_e32 v2, v0 540; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 541; GFX1032-NEXT: v_mov_b32_e32 v1, 0 542; GFX1032-NEXT: s_mov_b32 exec_lo, s3 543; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 544; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 545; GFX1032-NEXT: v_mov_b32_e32 v2, 0 546; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 547; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 548; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 549; GFX1032-NEXT: s_mov_b32 s2, -1 550; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 551; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 552; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 553; GFX1032-NEXT: v_mov_b32_e32 v3, v2 554; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 555; GFX1032-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 556; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 557; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 558; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 559; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 560; GFX1032-NEXT: s_mov_b32 exec_lo, s4 561; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 562; GFX1032-NEXT: ; implicit-def: $vgpr0 563; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 564; GFX1032-NEXT: s_cbranch_execz BB2_2 565; GFX1032-NEXT: ; %bb.1: 566; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 567; GFX1032-NEXT: v_mov_b32_e32 v7, s3 568; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 569; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 570; GFX1032-NEXT: ds_add_rtn_u32 v0, v0, v7 571; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 572; GFX1032-NEXT: buffer_gl0_inv 573; GFX1032-NEXT: buffer_gl1_inv 574; GFX1032-NEXT: BB2_2: 575; GFX1032-NEXT: v_nop 576; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 577; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 578; GFX1032-NEXT: v_mov_b32_e32 v0, v1 579; GFX1032-NEXT: v_add_nc_u32_e32 v0, s3, v0 580; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 581; GFX1032-NEXT: s_nop 1 582; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 583; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 584; GFX1032-NEXT: s_endpgm 585entry: 586 %lane = call i32 @llvm.amdgcn.workitem.id.x() 587 %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel 588 store i32 %old, i32 addrspace(1)* %out 589 ret void 590} 591 592define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) { 593; 594; 595; GFX7LESS-LABEL: add_i32_varying_gfx1032: 596; GFX7LESS: ; %bb.0: ; %entry 597; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 598; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 599; GFX7LESS-NEXT: s_mov_b32 m0, -1 600; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 601; GFX7LESS-NEXT: ds_add_rtn_u32 v0, v1, v0 602; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 603; GFX7LESS-NEXT: buffer_wbinvl1 604; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 605; GFX7LESS-NEXT: s_mov_b32 s2, -1 606; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 607; GFX7LESS-NEXT: s_endpgm 608; 609; GFX8-LABEL: add_i32_varying_gfx1032: 610; GFX8: ; %bb.0: ; %entry 611; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 612; GFX8-NEXT: s_mov_b64 s[2:3], exec 613; GFX8-NEXT: v_mov_b32_e32 v2, v0 614; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 615; GFX8-NEXT: v_mov_b32_e32 v1, 0 616; GFX8-NEXT: s_mov_b64 exec, s[4:5] 617; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 618; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 619; GFX8-NEXT: s_not_b64 exec, exec 620; GFX8-NEXT: v_mov_b32_e32 v2, 0 621; GFX8-NEXT: s_not_b64 exec, exec 622; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 623; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 624; GFX8-NEXT: s_nop 1 625; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 626; GFX8-NEXT: s_nop 1 627; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 628; GFX8-NEXT: s_nop 1 629; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 630; GFX8-NEXT: s_nop 1 631; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 632; GFX8-NEXT: s_nop 1 633; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 634; GFX8-NEXT: v_readlane_b32 s2, v2, 63 635; GFX8-NEXT: s_nop 0 636; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 637; GFX8-NEXT: s_mov_b64 exec, s[4:5] 638; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 639; GFX8-NEXT: ; implicit-def: $vgpr0 640; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 641; GFX8-NEXT: s_cbranch_execz BB3_2 642; GFX8-NEXT: ; %bb.1: 643; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 644; GFX8-NEXT: v_mov_b32_e32 v3, s2 645; GFX8-NEXT: s_mov_b32 m0, -1 646; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 647; GFX8-NEXT: ds_add_rtn_u32 v0, v0, v3 648; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 649; GFX8-NEXT: buffer_wbinvl1_vol 650; GFX8-NEXT: BB3_2: 651; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 652; GFX8-NEXT: v_readfirstlane_b32 s2, v0 653; GFX8-NEXT: v_mov_b32_e32 v0, v1 654; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0 655; GFX8-NEXT: s_mov_b32 s3, 0xf000 656; GFX8-NEXT: s_mov_b32 s2, -1 657; GFX8-NEXT: s_nop 0 658; GFX8-NEXT: s_waitcnt lgkmcnt(0) 659; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 660; GFX8-NEXT: s_endpgm 661; 662; GFX9-LABEL: add_i32_varying_gfx1032: 663; GFX9: ; %bb.0: ; %entry 664; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 665; GFX9-NEXT: s_mov_b64 s[2:3], exec 666; GFX9-NEXT: v_mov_b32_e32 v2, v0 667; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 668; GFX9-NEXT: v_mov_b32_e32 v1, 0 669; GFX9-NEXT: s_mov_b64 exec, s[4:5] 670; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 671; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 672; GFX9-NEXT: s_not_b64 exec, exec 673; GFX9-NEXT: v_mov_b32_e32 v2, 0 674; GFX9-NEXT: s_not_b64 exec, exec 675; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 676; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 677; GFX9-NEXT: s_nop 1 678; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 679; GFX9-NEXT: s_nop 1 680; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 681; GFX9-NEXT: s_nop 1 682; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 683; GFX9-NEXT: s_nop 1 684; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 685; GFX9-NEXT: s_nop 1 686; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 687; GFX9-NEXT: v_readlane_b32 s2, v2, 63 688; GFX9-NEXT: s_nop 0 689; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 690; GFX9-NEXT: s_mov_b64 exec, s[4:5] 691; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 692; GFX9-NEXT: ; implicit-def: $vgpr0 693; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 694; GFX9-NEXT: s_cbranch_execz BB3_2 695; GFX9-NEXT: ; %bb.1: 696; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 697; GFX9-NEXT: v_mov_b32_e32 v3, s2 698; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 699; GFX9-NEXT: ds_add_rtn_u32 v0, v0, v3 700; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 701; GFX9-NEXT: buffer_wbinvl1_vol 702; GFX9-NEXT: BB3_2: 703; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 704; GFX9-NEXT: v_readfirstlane_b32 s2, v0 705; GFX9-NEXT: v_mov_b32_e32 v0, v1 706; GFX9-NEXT: v_add_u32_e32 v0, s2, v0 707; GFX9-NEXT: s_mov_b32 s3, 0xf000 708; GFX9-NEXT: s_mov_b32 s2, -1 709; GFX9-NEXT: s_nop 0 710; GFX9-NEXT: s_waitcnt lgkmcnt(0) 711; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 712; GFX9-NEXT: s_endpgm 713; 714; GFX1064-LABEL: add_i32_varying_gfx1032: 715; GFX1064: ; %bb.0: ; %entry 716; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 717; GFX1064-NEXT: s_mov_b64 s[2:3], exec 718; GFX1064-NEXT: v_mov_b32_e32 v2, v0 719; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 720; GFX1064-NEXT: v_mov_b32_e32 v1, 0 721; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 722; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 723; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 724; GFX1064-NEXT: s_not_b64 exec, exec 725; GFX1064-NEXT: v_mov_b32_e32 v2, 0 726; GFX1064-NEXT: s_not_b64 exec, exec 727; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 728; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 729; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 730; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 731; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 732; GFX1064-NEXT: v_mov_b32_e32 v3, v2 733; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 734; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 735; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 736; GFX1064-NEXT: v_mov_b32_e32 v3, s2 737; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 738; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 739; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 740; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 741; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 742; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 743; GFX1064-NEXT: s_mov_b32 s2, -1 744; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 745; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 746; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 747; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 748; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 749; GFX1064-NEXT: ; implicit-def: $vgpr0 750; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 751; GFX1064-NEXT: s_cbranch_execz BB3_2 752; GFX1064-NEXT: ; %bb.1: 753; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 754; GFX1064-NEXT: v_mov_b32_e32 v7, s3 755; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 756; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 757; GFX1064-NEXT: ds_add_rtn_u32 v0, v0, v7 758; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 759; GFX1064-NEXT: buffer_gl0_inv 760; GFX1064-NEXT: buffer_gl1_inv 761; GFX1064-NEXT: BB3_2: 762; GFX1064-NEXT: v_nop 763; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 764; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 765; GFX1064-NEXT: v_mov_b32_e32 v0, v1 766; GFX1064-NEXT: v_add_nc_u32_e32 v0, s3, v0 767; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 768; GFX1064-NEXT: s_nop 1 769; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 770; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 771; GFX1064-NEXT: s_endpgm 772; 773; GFX1032-LABEL: add_i32_varying_gfx1032: 774; GFX1032: ; %bb.0: ; %entry 775; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 776; GFX1032-NEXT: s_mov_b32 s2, exec_lo 777; GFX1032-NEXT: ; implicit-def: $vcc_hi 778; GFX1032-NEXT: v_mov_b32_e32 v2, v0 779; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 780; GFX1032-NEXT: v_mov_b32_e32 v1, 0 781; GFX1032-NEXT: s_mov_b32 exec_lo, s3 782; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 783; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 784; GFX1032-NEXT: v_mov_b32_e32 v2, 0 785; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 786; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 787; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 788; GFX1032-NEXT: s_mov_b32 s2, -1 789; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 790; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 791; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 792; GFX1032-NEXT: v_mov_b32_e32 v3, v2 793; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 794; GFX1032-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 795; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 796; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 797; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 798; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 799; GFX1032-NEXT: s_mov_b32 exec_lo, s4 800; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 801; GFX1032-NEXT: ; implicit-def: $vgpr0 802; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 803; GFX1032-NEXT: s_cbranch_execz BB3_2 804; GFX1032-NEXT: ; %bb.1: 805; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 806; GFX1032-NEXT: v_mov_b32_e32 v7, s3 807; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 808; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 809; GFX1032-NEXT: ds_add_rtn_u32 v0, v0, v7 810; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 811; GFX1032-NEXT: buffer_gl0_inv 812; GFX1032-NEXT: buffer_gl1_inv 813; GFX1032-NEXT: BB3_2: 814; GFX1032-NEXT: v_nop 815; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 816; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 817; GFX1032-NEXT: v_mov_b32_e32 v0, v1 818; GFX1032-NEXT: v_add_nc_u32_e32 v0, s3, v0 819; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 820; GFX1032-NEXT: s_nop 1 821; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 822; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 823; GFX1032-NEXT: s_endpgm 824entry: 825 %lane = call i32 @llvm.amdgcn.workitem.id.x() 826 %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel 827 store i32 %old, i32 addrspace(1)* %out 828 ret void 829} 830 831define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) { 832; 833; 834; GFX7LESS-LABEL: add_i32_varying_gfx1064: 835; GFX7LESS: ; %bb.0: ; %entry 836; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 837; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 838; GFX7LESS-NEXT: s_mov_b32 m0, -1 839; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 840; GFX7LESS-NEXT: ds_add_rtn_u32 v0, v1, v0 841; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 842; GFX7LESS-NEXT: buffer_wbinvl1 843; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 844; GFX7LESS-NEXT: s_mov_b32 s2, -1 845; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 846; GFX7LESS-NEXT: s_endpgm 847; 848; GFX8-LABEL: add_i32_varying_gfx1064: 849; GFX8: ; %bb.0: ; %entry 850; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 851; GFX8-NEXT: s_mov_b64 s[2:3], exec 852; GFX8-NEXT: v_mov_b32_e32 v2, v0 853; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 854; GFX8-NEXT: v_mov_b32_e32 v1, 0 855; GFX8-NEXT: s_mov_b64 exec, s[4:5] 856; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 857; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 858; GFX8-NEXT: s_not_b64 exec, exec 859; GFX8-NEXT: v_mov_b32_e32 v2, 0 860; GFX8-NEXT: s_not_b64 exec, exec 861; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 862; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 863; GFX8-NEXT: s_nop 1 864; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 865; GFX8-NEXT: s_nop 1 866; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 867; GFX8-NEXT: s_nop 1 868; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 869; GFX8-NEXT: s_nop 1 870; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 871; GFX8-NEXT: s_nop 1 872; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 873; GFX8-NEXT: v_readlane_b32 s2, v2, 63 874; GFX8-NEXT: s_nop 0 875; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 876; GFX8-NEXT: s_mov_b64 exec, s[4:5] 877; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 878; GFX8-NEXT: ; implicit-def: $vgpr0 879; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 880; GFX8-NEXT: s_cbranch_execz BB4_2 881; GFX8-NEXT: ; %bb.1: 882; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 883; GFX8-NEXT: v_mov_b32_e32 v3, s2 884; GFX8-NEXT: s_mov_b32 m0, -1 885; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 886; GFX8-NEXT: ds_add_rtn_u32 v0, v0, v3 887; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 888; GFX8-NEXT: buffer_wbinvl1_vol 889; GFX8-NEXT: BB4_2: 890; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 891; GFX8-NEXT: v_readfirstlane_b32 s2, v0 892; GFX8-NEXT: v_mov_b32_e32 v0, v1 893; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0 894; GFX8-NEXT: s_mov_b32 s3, 0xf000 895; GFX8-NEXT: s_mov_b32 s2, -1 896; GFX8-NEXT: s_nop 0 897; GFX8-NEXT: s_waitcnt lgkmcnt(0) 898; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 899; GFX8-NEXT: s_endpgm 900; 901; GFX9-LABEL: add_i32_varying_gfx1064: 902; GFX9: ; %bb.0: ; %entry 903; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 904; GFX9-NEXT: s_mov_b64 s[2:3], exec 905; GFX9-NEXT: v_mov_b32_e32 v2, v0 906; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 907; GFX9-NEXT: v_mov_b32_e32 v1, 0 908; GFX9-NEXT: s_mov_b64 exec, s[4:5] 909; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 910; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 911; GFX9-NEXT: s_not_b64 exec, exec 912; GFX9-NEXT: v_mov_b32_e32 v2, 0 913; GFX9-NEXT: s_not_b64 exec, exec 914; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 915; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 916; GFX9-NEXT: s_nop 1 917; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 918; GFX9-NEXT: s_nop 1 919; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 920; GFX9-NEXT: s_nop 1 921; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 922; GFX9-NEXT: s_nop 1 923; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 924; GFX9-NEXT: s_nop 1 925; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 926; GFX9-NEXT: v_readlane_b32 s2, v2, 63 927; GFX9-NEXT: s_nop 0 928; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 929; GFX9-NEXT: s_mov_b64 exec, s[4:5] 930; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 931; GFX9-NEXT: ; implicit-def: $vgpr0 932; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 933; GFX9-NEXT: s_cbranch_execz BB4_2 934; GFX9-NEXT: ; %bb.1: 935; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 936; GFX9-NEXT: v_mov_b32_e32 v3, s2 937; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 938; GFX9-NEXT: ds_add_rtn_u32 v0, v0, v3 939; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 940; GFX9-NEXT: buffer_wbinvl1_vol 941; GFX9-NEXT: BB4_2: 942; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 943; GFX9-NEXT: v_readfirstlane_b32 s2, v0 944; GFX9-NEXT: v_mov_b32_e32 v0, v1 945; GFX9-NEXT: v_add_u32_e32 v0, s2, v0 946; GFX9-NEXT: s_mov_b32 s3, 0xf000 947; GFX9-NEXT: s_mov_b32 s2, -1 948; GFX9-NEXT: s_nop 0 949; GFX9-NEXT: s_waitcnt lgkmcnt(0) 950; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 951; GFX9-NEXT: s_endpgm 952; 953; GFX1064-LABEL: add_i32_varying_gfx1064: 954; GFX1064: ; %bb.0: ; %entry 955; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 956; GFX1064-NEXT: s_mov_b64 s[2:3], exec 957; GFX1064-NEXT: v_mov_b32_e32 v2, v0 958; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 959; GFX1064-NEXT: v_mov_b32_e32 v1, 0 960; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 961; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 962; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 963; GFX1064-NEXT: s_not_b64 exec, exec 964; GFX1064-NEXT: v_mov_b32_e32 v2, 0 965; GFX1064-NEXT: s_not_b64 exec, exec 966; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 967; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 968; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 969; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 970; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 971; GFX1064-NEXT: v_mov_b32_e32 v3, v2 972; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 973; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 974; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 975; GFX1064-NEXT: v_mov_b32_e32 v3, s2 976; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 977; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 978; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 979; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 980; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 981; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 982; GFX1064-NEXT: s_mov_b32 s2, -1 983; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 984; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 985; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 986; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 987; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 988; GFX1064-NEXT: ; implicit-def: $vgpr0 989; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 990; GFX1064-NEXT: s_cbranch_execz BB4_2 991; GFX1064-NEXT: ; %bb.1: 992; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 993; GFX1064-NEXT: v_mov_b32_e32 v7, s3 994; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 995; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 996; GFX1064-NEXT: ds_add_rtn_u32 v0, v0, v7 997; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 998; GFX1064-NEXT: buffer_gl0_inv 999; GFX1064-NEXT: buffer_gl1_inv 1000; GFX1064-NEXT: BB4_2: 1001; GFX1064-NEXT: v_nop 1002; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 1003; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 1004; GFX1064-NEXT: v_mov_b32_e32 v0, v1 1005; GFX1064-NEXT: v_add_nc_u32_e32 v0, s3, v0 1006; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 1007; GFX1064-NEXT: s_nop 1 1008; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1009; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 1010; GFX1064-NEXT: s_endpgm 1011; 1012; GFX1032-LABEL: add_i32_varying_gfx1064: 1013; GFX1032: ; %bb.0: ; %entry 1014; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1015; GFX1032-NEXT: s_mov_b32 s2, exec_lo 1016; GFX1032-NEXT: ; implicit-def: $vcc_hi 1017; GFX1032-NEXT: v_mov_b32_e32 v2, v0 1018; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 1019; GFX1032-NEXT: v_mov_b32_e32 v1, 0 1020; GFX1032-NEXT: s_mov_b32 exec_lo, s3 1021; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1022; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 1023; GFX1032-NEXT: v_mov_b32_e32 v2, 0 1024; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 1025; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 1026; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 1027; GFX1032-NEXT: s_mov_b32 s2, -1 1028; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 1029; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 1030; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 1031; GFX1032-NEXT: v_mov_b32_e32 v3, v2 1032; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 1033; GFX1032-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 1034; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 1035; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 1036; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 1037; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 1038; GFX1032-NEXT: s_mov_b32 exec_lo, s4 1039; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 1040; GFX1032-NEXT: ; implicit-def: $vgpr0 1041; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 1042; GFX1032-NEXT: s_cbranch_execz BB4_2 1043; GFX1032-NEXT: ; %bb.1: 1044; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 1045; GFX1032-NEXT: v_mov_b32_e32 v7, s3 1046; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1047; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1048; GFX1032-NEXT: ds_add_rtn_u32 v0, v0, v7 1049; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1050; GFX1032-NEXT: buffer_gl0_inv 1051; GFX1032-NEXT: buffer_gl1_inv 1052; GFX1032-NEXT: BB4_2: 1053; GFX1032-NEXT: v_nop 1054; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 1055; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 1056; GFX1032-NEXT: v_mov_b32_e32 v0, v1 1057; GFX1032-NEXT: v_add_nc_u32_e32 v0, s3, v0 1058; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 1059; GFX1032-NEXT: s_nop 1 1060; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1061; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 1062; GFX1032-NEXT: s_endpgm 1063entry: 1064 %lane = call i32 @llvm.amdgcn.workitem.id.x() 1065 %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel 1066 store i32 %old, i32 addrspace(1)* %out 1067 ret void 1068} 1069 1070define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) { 1071; 1072; 1073; GFX7LESS-LABEL: add_i64_constant: 1074; GFX7LESS: ; %bb.0: ; %entry 1075; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec 1076; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 1077; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0 1078; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0 1079; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1080; GFX7LESS-NEXT: ; implicit-def: $vgpr1_vgpr2 1081; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 1082; GFX7LESS-NEXT: s_cbranch_execz BB5_2 1083; GFX7LESS-NEXT: ; %bb.1: 1084; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 1085; GFX7LESS-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1086; GFX7LESS-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 1087; GFX7LESS-NEXT: v_mul_u32_u24_e64 v1, s4, 5 1088; GFX7LESS-NEXT: s_mov_b32 m0, -1 1089; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1090; GFX7LESS-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1091; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1092; GFX7LESS-NEXT: buffer_wbinvl1 1093; GFX7LESS-NEXT: BB5_2: 1094; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 1095; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v1 1096; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v2 1097; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v0 1098; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1099; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 1100; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4 1101; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s2, v0 1102; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc 1103; GFX7LESS-NEXT: s_mov_b32 s2, -1 1104; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1105; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1106; GFX7LESS-NEXT: s_endpgm 1107; 1108; GFX8-LABEL: add_i64_constant: 1109; GFX8: ; %bb.0: ; %entry 1110; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1111; GFX8-NEXT: s_mov_b64 s[4:5], exec 1112; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 1113; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 1114; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1115; GFX8-NEXT: ; implicit-def: $vgpr1_vgpr2 1116; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 1117; GFX8-NEXT: s_cbranch_execz BB5_2 1118; GFX8-NEXT: ; %bb.1: 1119; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 1120; GFX8-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 1121; GFX8-NEXT: v_mul_u32_u24_e64 v1, s4, 5 1122; GFX8-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1123; GFX8-NEXT: s_mov_b32 m0, -1 1124; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1125; GFX8-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1126; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1127; GFX8-NEXT: buffer_wbinvl1_vol 1128; GFX8-NEXT: BB5_2: 1129; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 1130; GFX8-NEXT: v_readfirstlane_b32 s2, v1 1131; GFX8-NEXT: v_readfirstlane_b32 s3, v2 1132; GFX8-NEXT: v_mov_b32_e32 v1, s2 1133; GFX8-NEXT: v_mov_b32_e32 v2, s3 1134; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2] 1135; GFX8-NEXT: s_mov_b32 s3, 0xf000 1136; GFX8-NEXT: s_mov_b32 s2, -1 1137; GFX8-NEXT: s_nop 2 1138; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1139; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1140; GFX8-NEXT: s_endpgm 1141; 1142; GFX9-LABEL: add_i64_constant: 1143; GFX9: ; %bb.0: ; %entry 1144; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1145; GFX9-NEXT: s_mov_b64 s[4:5], exec 1146; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 1147; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 1148; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1149; GFX9-NEXT: ; implicit-def: $vgpr1_vgpr2 1150; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 1151; GFX9-NEXT: s_cbranch_execz BB5_2 1152; GFX9-NEXT: ; %bb.1: 1153; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 1154; GFX9-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 1155; GFX9-NEXT: v_mul_u32_u24_e64 v1, s4, 5 1156; GFX9-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1157; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1158; GFX9-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1159; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1160; GFX9-NEXT: buffer_wbinvl1_vol 1161; GFX9-NEXT: BB5_2: 1162; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 1163; GFX9-NEXT: v_readfirstlane_b32 s2, v1 1164; GFX9-NEXT: v_readfirstlane_b32 s3, v2 1165; GFX9-NEXT: v_mov_b32_e32 v1, s2 1166; GFX9-NEXT: v_mov_b32_e32 v2, s3 1167; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2] 1168; GFX9-NEXT: s_mov_b32 s3, 0xf000 1169; GFX9-NEXT: s_mov_b32 s2, -1 1170; GFX9-NEXT: s_nop 2 1171; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1172; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1173; GFX9-NEXT: s_endpgm 1174; 1175; GFX1064-LABEL: add_i64_constant: 1176; GFX1064: ; %bb.0: ; %entry 1177; GFX1064-NEXT: s_mov_b64 s[4:5], exec 1178; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1179; GFX1064-NEXT: ; implicit-def: $vgpr1_vgpr2 1180; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0 1181; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s5, v0 1182; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1183; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 1184; GFX1064-NEXT: s_cbranch_execz BB5_2 1185; GFX1064-NEXT: ; %bb.1: 1186; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 1187; GFX1064-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1188; GFX1064-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 1189; GFX1064-NEXT: v_mul_u32_u24_e64 v1, s4, 5 1190; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1191; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 1192; GFX1064-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1193; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1194; GFX1064-NEXT: buffer_gl0_inv 1195; GFX1064-NEXT: buffer_gl1_inv 1196; GFX1064-NEXT: BB5_2: 1197; GFX1064-NEXT: v_nop 1198; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 1199; GFX1064-NEXT: v_readfirstlane_b32 s2, v1 1200; GFX1064-NEXT: v_readfirstlane_b32 s3, v2 1201; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v0, 5, s[2:3] 1202; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 1203; GFX1064-NEXT: s_mov_b32 s2, -1 1204; GFX1064-NEXT: s_nop 2 1205; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1206; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1207; GFX1064-NEXT: s_endpgm 1208; 1209; GFX1032-LABEL: add_i64_constant: 1210; GFX1032: ; %bb.0: ; %entry 1211; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1212; GFX1032-NEXT: s_mov_b32 s3, exec_lo 1213; GFX1032-NEXT: ; implicit-def: $vcc_hi 1214; GFX1032-NEXT: ; implicit-def: $vgpr1_vgpr2 1215; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s3, 0 1216; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 1217; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 1218; GFX1032-NEXT: s_cbranch_execz BB5_2 1219; GFX1032-NEXT: ; %bb.1: 1220; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3 1221; GFX1032-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1222; GFX1032-NEXT: v_mul_hi_u32_u24_e64 v2, s3, 5 1223; GFX1032-NEXT: v_mul_u32_u24_e64 v1, s3, 5 1224; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1225; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1226; GFX1032-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1227; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1228; GFX1032-NEXT: buffer_gl0_inv 1229; GFX1032-NEXT: buffer_gl1_inv 1230; GFX1032-NEXT: BB5_2: 1231; GFX1032-NEXT: v_nop 1232; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 1233; GFX1032-NEXT: v_readfirstlane_b32 s2, v1 1234; GFX1032-NEXT: v_readfirstlane_b32 s3, v2 1235; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, v0, 5, s[2:3] 1236; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 1237; GFX1032-NEXT: s_mov_b32 s2, -1 1238; GFX1032-NEXT: s_nop 2 1239; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1240; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1241; GFX1032-NEXT: s_endpgm 1242entry: 1243 %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 5 acq_rel 1244 store i64 %old, i64 addrspace(1)* %out 1245 ret void 1246} 1247 1248define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive) { 1249; 1250; 1251; GFX7LESS-LABEL: add_i64_uniform: 1252; GFX7LESS: ; %bb.0: ; %entry 1253; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec 1254; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 1255; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 1256; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 1257; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1258; GFX7LESS-NEXT: ; implicit-def: $vgpr1_vgpr2 1259; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc 1260; GFX7LESS-NEXT: s_cbranch_execz BB6_2 1261; GFX7LESS-NEXT: ; %bb.1: 1262; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 1263; GFX7LESS-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1264; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1265; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6 1266; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6 1267; GFX7LESS-NEXT: v_mul_hi_u32 v1, s2, v1 1268; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6 1269; GFX7LESS-NEXT: v_add_i32_e32 v2, vcc, s7, v1 1270; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6 1271; GFX7LESS-NEXT: s_mov_b32 m0, -1 1272; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1273; GFX7LESS-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1274; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1275; GFX7LESS-NEXT: buffer_wbinvl1 1276; GFX7LESS-NEXT: BB6_2: 1277; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5] 1278; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000 1279; GFX7LESS-NEXT: s_mov_b32 s6, -1 1280; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1281; GFX7LESS-NEXT: s_mov_b32 s4, s0 1282; GFX7LESS-NEXT: s_mov_b32 s5, s1 1283; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1 1284; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v2 1285; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v0 1286; GFX7LESS-NEXT: v_mul_hi_u32 v2, s2, v0 1287; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0 1288; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v2, v1 1289; GFX7LESS-NEXT: v_mov_b32_e32 v2, s1 1290; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s0, v0 1291; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc 1292; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 1293; GFX7LESS-NEXT: s_endpgm 1294; 1295; GFX8-LABEL: add_i64_uniform: 1296; GFX8: ; %bb.0: ; %entry 1297; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 1298; GFX8-NEXT: s_mov_b64 s[6:7], exec 1299; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 1300; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0 1301; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1302; GFX8-NEXT: ; implicit-def: $vgpr1_vgpr2 1303; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 1304; GFX8-NEXT: s_cbranch_execz BB6_2 1305; GFX8-NEXT: ; %bb.1: 1306; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 1307; GFX8-NEXT: v_mov_b32_e32 v1, s6 1308; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1309; GFX8-NEXT: v_mul_hi_u32 v1, s2, v1 1310; GFX8-NEXT: s_mul_i32 s7, s3, s6 1311; GFX8-NEXT: s_mul_i32 s6, s2, s6 1312; GFX8-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1313; GFX8-NEXT: v_add_u32_e32 v2, vcc, s7, v1 1314; GFX8-NEXT: v_mov_b32_e32 v1, s6 1315; GFX8-NEXT: s_mov_b32 m0, -1 1316; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1317; GFX8-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1318; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1319; GFX8-NEXT: buffer_wbinvl1_vol 1320; GFX8-NEXT: BB6_2: 1321; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 1322; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1323; GFX8-NEXT: s_mov_b32 s4, s0 1324; GFX8-NEXT: v_readfirstlane_b32 s0, v1 1325; GFX8-NEXT: v_mul_lo_u32 v1, s3, v0 1326; GFX8-NEXT: v_mul_hi_u32 v3, s2, v0 1327; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0 1328; GFX8-NEXT: s_mov_b32 s5, s1 1329; GFX8-NEXT: v_readfirstlane_b32 s1, v2 1330; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1 1331; GFX8-NEXT: v_mov_b32_e32 v2, s1 1332; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0 1333; GFX8-NEXT: s_mov_b32 s7, 0xf000 1334; GFX8-NEXT: s_mov_b32 s6, -1 1335; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc 1336; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 1337; GFX8-NEXT: s_endpgm 1338; 1339; GFX9-LABEL: add_i64_uniform: 1340; GFX9: ; %bb.0: ; %entry 1341; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 1342; GFX9-NEXT: s_mov_b64 s[6:7], exec 1343; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 1344; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0 1345; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1346; GFX9-NEXT: ; implicit-def: $vgpr1_vgpr2 1347; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 1348; GFX9-NEXT: s_cbranch_execz BB6_2 1349; GFX9-NEXT: ; %bb.1: 1350; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 1351; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1352; GFX9-NEXT: s_mul_i32 s7, s3, s6 1353; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6 1354; GFX9-NEXT: s_add_i32 s8, s8, s7 1355; GFX9-NEXT: s_mul_i32 s6, s2, s6 1356; GFX9-NEXT: v_mov_b32_e32 v1, s6 1357; GFX9-NEXT: v_mov_b32_e32 v2, s8 1358; GFX9-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1359; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1360; GFX9-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1361; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1362; GFX9-NEXT: buffer_wbinvl1_vol 1363; GFX9-NEXT: BB6_2: 1364; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 1365; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1366; GFX9-NEXT: v_mul_lo_u32 v3, s3, v0 1367; GFX9-NEXT: v_mul_hi_u32 v4, s2, v0 1368; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0 1369; GFX9-NEXT: s_mov_b32 s4, s0 1370; GFX9-NEXT: v_readfirstlane_b32 s0, v1 1371; GFX9-NEXT: s_mov_b32 s5, s1 1372; GFX9-NEXT: v_readfirstlane_b32 s1, v2 1373; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 1374; GFX9-NEXT: v_mov_b32_e32 v2, s1 1375; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 1376; GFX9-NEXT: s_mov_b32 s7, 0xf000 1377; GFX9-NEXT: s_mov_b32 s6, -1 1378; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc 1379; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 1380; GFX9-NEXT: s_endpgm 1381; 1382; GFX1064-LABEL: add_i64_uniform: 1383; GFX1064: ; %bb.0: ; %entry 1384; GFX1064-NEXT: s_mov_b64 s[6:7], exec 1385; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 1386; GFX1064-NEXT: ; implicit-def: $vgpr1_vgpr2 1387; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 1388; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s7, v0 1389; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1390; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 1391; GFX1064-NEXT: s_cbranch_execz BB6_2 1392; GFX1064-NEXT: ; %bb.1: 1393; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 1394; GFX1064-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1395; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1396; GFX1064-NEXT: s_mul_i32 s7, s3, s6 1397; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6 1398; GFX1064-NEXT: s_mul_i32 s6, s2, s6 1399; GFX1064-NEXT: s_add_i32 s8, s8, s7 1400; GFX1064-NEXT: v_mov_b32_e32 v1, s6 1401; GFX1064-NEXT: v_mov_b32_e32 v2, s8 1402; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1403; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 1404; GFX1064-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1405; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1406; GFX1064-NEXT: buffer_gl0_inv 1407; GFX1064-NEXT: buffer_gl1_inv 1408; GFX1064-NEXT: BB6_2: 1409; GFX1064-NEXT: v_nop 1410; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 1411; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1412; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v0 1413; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v0 1414; GFX1064-NEXT: v_mul_lo_u32 v0, s2, v0 1415; GFX1064-NEXT: v_readfirstlane_b32 s4, v1 1416; GFX1064-NEXT: v_readfirstlane_b32 s5, v2 1417; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 1418; GFX1064-NEXT: s_mov_b32 s2, -1 1419; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 1420; GFX1064-NEXT: v_add_co_u32_e64 v0, vcc, s4, v0 1421; GFX1064-NEXT: v_add_co_ci_u32_e32 v1, vcc, s5, v1, vcc 1422; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1423; GFX1064-NEXT: s_endpgm 1424; 1425; GFX1032-LABEL: add_i64_uniform: 1426; GFX1032: ; %bb.0: ; %entry 1427; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 1428; GFX1032-NEXT: s_mov_b32 s5, exec_lo 1429; GFX1032-NEXT: ; implicit-def: $vcc_hi 1430; GFX1032-NEXT: ; implicit-def: $vgpr1_vgpr2 1431; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s5, 0 1432; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 1433; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 1434; GFX1032-NEXT: s_cbranch_execz BB6_2 1435; GFX1032-NEXT: ; %bb.1: 1436; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s5 1437; GFX1032-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 1438; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1439; GFX1032-NEXT: s_mul_i32 s6, s3, s5 1440; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5 1441; GFX1032-NEXT: s_mul_i32 s5, s2, s5 1442; GFX1032-NEXT: s_add_i32 s7, s7, s6 1443; GFX1032-NEXT: v_mov_b32_e32 v1, s5 1444; GFX1032-NEXT: v_mov_b32_e32 v2, s7 1445; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1446; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1447; GFX1032-NEXT: ds_add_rtn_u64 v[1:2], v3, v[1:2] 1448; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1449; GFX1032-NEXT: buffer_gl0_inv 1450; GFX1032-NEXT: buffer_gl1_inv 1451; GFX1032-NEXT: BB6_2: 1452; GFX1032-NEXT: v_nop 1453; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 1454; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1455; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v0 1456; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v0 1457; GFX1032-NEXT: v_mul_lo_u32 v0, s2, v0 1458; GFX1032-NEXT: v_readfirstlane_b32 s4, v1 1459; GFX1032-NEXT: v_readfirstlane_b32 s5, v2 1460; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 1461; GFX1032-NEXT: s_mov_b32 s2, -1 1462; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 1463; GFX1032-NEXT: v_add_co_u32_e64 v0, vcc_lo, s4, v0 1464; GFX1032-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo 1465; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1466; GFX1032-NEXT: s_endpgm 1467entry: 1468 %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %additive acq_rel 1469 store i64 %old, i64 addrspace(1)* %out 1470 ret void 1471} 1472 1473; GCN-NOT: v_mbcnt_lo_u32_b32 1474; GCN-NOT: v_mbcnt_hi_u32_b32 1475; GCN-NOT: s_bcnt1_i32_b64 1476define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) { 1477; 1478; 1479; GFX7LESS-LABEL: add_i64_varying: 1480; GFX7LESS: ; %bb.0: ; %entry 1481; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 1482; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 1483; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 1484; GFX7LESS-NEXT: s_mov_b32 m0, -1 1485; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1486; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v2, v[0:1] 1487; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1488; GFX7LESS-NEXT: buffer_wbinvl1 1489; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 1490; GFX7LESS-NEXT: s_mov_b32 s2, -1 1491; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1492; GFX7LESS-NEXT: s_endpgm 1493; 1494; GFX8-LABEL: add_i64_varying: 1495; GFX8: ; %bb.0: ; %entry 1496; GFX8-NEXT: v_mov_b32_e32 v1, 0 1497; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 1498; GFX8-NEXT: s_mov_b32 m0, -1 1499; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1500; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1501; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v2, v[0:1] 1502; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1503; GFX8-NEXT: buffer_wbinvl1_vol 1504; GFX8-NEXT: s_mov_b32 s3, 0xf000 1505; GFX8-NEXT: s_mov_b32 s2, -1 1506; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1507; GFX8-NEXT: s_endpgm 1508; 1509; GFX9-LABEL: add_i64_varying: 1510; GFX9: ; %bb.0: ; %entry 1511; GFX9-NEXT: v_mov_b32_e32 v1, 0 1512; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 1513; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1514; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1515; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v2, v[0:1] 1516; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1517; GFX9-NEXT: buffer_wbinvl1_vol 1518; GFX9-NEXT: s_mov_b32 s3, 0xf000 1519; GFX9-NEXT: s_mov_b32 s2, -1 1520; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1521; GFX9-NEXT: s_endpgm 1522; 1523; GFX1064-LABEL: add_i64_varying: 1524; GFX1064: ; %bb.0: ; %entry 1525; GFX1064-NEXT: v_mov_b32_e32 v1, 0 1526; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 1527; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1528; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 1529; GFX1064-NEXT: s_mov_b32 s2, -1 1530; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1531; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 1532; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v2, v[0:1] 1533; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1534; GFX1064-NEXT: buffer_gl0_inv 1535; GFX1064-NEXT: buffer_gl1_inv 1536; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1537; GFX1064-NEXT: s_endpgm 1538; 1539; GFX1032-LABEL: add_i64_varying: 1540; GFX1032: ; %bb.0: ; %entry 1541; GFX1032-NEXT: v_mov_b32_e32 v1, 0 1542; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 1543; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1544; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 1545; GFX1032-NEXT: s_mov_b32 s2, -1 1546; GFX1032-NEXT: ; implicit-def: $vcc_hi 1547; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1548; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1549; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v2, v[0:1] 1550; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1551; GFX1032-NEXT: buffer_gl0_inv 1552; GFX1032-NEXT: buffer_gl1_inv 1553; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 1554; GFX1032-NEXT: s_endpgm 1555entry: 1556 %lane = call i32 @llvm.amdgcn.workitem.id.x() 1557 %zext = zext i32 %lane to i64 1558 %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %zext acq_rel 1559 store i64 %old, i64 addrspace(1)* %out 1560 ret void 1561} 1562 1563define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) { 1564; 1565; 1566; GFX7LESS-LABEL: sub_i32_constant: 1567; GFX7LESS: ; %bb.0: ; %entry 1568; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec 1569; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 1570; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1571; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 1572; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1573; GFX7LESS-NEXT: ; implicit-def: $vgpr1 1574; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc 1575; GFX7LESS-NEXT: s_cbranch_execz BB8_2 1576; GFX7LESS-NEXT: ; %bb.1: 1577; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 1578; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1579; GFX7LESS-NEXT: v_mul_u32_u24_e64 v2, s2, 5 1580; GFX7LESS-NEXT: s_mov_b32 m0, -1 1581; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1582; GFX7LESS-NEXT: ds_sub_rtn_u32 v1, v1, v2 1583; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1584; GFX7LESS-NEXT: buffer_wbinvl1 1585; GFX7LESS-NEXT: BB8_2: 1586; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5] 1587; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v1 1588; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1589; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 1590; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 1591; GFX7LESS-NEXT: s_mov_b32 s2, -1 1592; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1593; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 1594; GFX7LESS-NEXT: s_endpgm 1595; 1596; GFX8-LABEL: sub_i32_constant: 1597; GFX8: ; %bb.0: ; %entry 1598; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1599; GFX8-NEXT: s_mov_b64 s[2:3], exec 1600; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1601; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1602; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1603; GFX8-NEXT: ; implicit-def: $vgpr1 1604; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 1605; GFX8-NEXT: s_cbranch_execz BB8_2 1606; GFX8-NEXT: ; %bb.1: 1607; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 1608; GFX8-NEXT: v_mul_u32_u24_e64 v1, s2, 5 1609; GFX8-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 1610; GFX8-NEXT: s_mov_b32 m0, -1 1611; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1612; GFX8-NEXT: ds_sub_rtn_u32 v1, v2, v1 1613; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1614; GFX8-NEXT: buffer_wbinvl1_vol 1615; GFX8-NEXT: BB8_2: 1616; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 1617; GFX8-NEXT: v_readfirstlane_b32 s2, v1 1618; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1619; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v0 1620; GFX8-NEXT: s_mov_b32 s3, 0xf000 1621; GFX8-NEXT: s_mov_b32 s2, -1 1622; GFX8-NEXT: s_nop 0 1623; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1624; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 1625; GFX8-NEXT: s_endpgm 1626; 1627; GFX9-LABEL: sub_i32_constant: 1628; GFX9: ; %bb.0: ; %entry 1629; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1630; GFX9-NEXT: s_mov_b64 s[2:3], exec 1631; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1632; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1633; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1634; GFX9-NEXT: ; implicit-def: $vgpr1 1635; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 1636; GFX9-NEXT: s_cbranch_execz BB8_2 1637; GFX9-NEXT: ; %bb.1: 1638; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 1639; GFX9-NEXT: v_mul_u32_u24_e64 v1, s2, 5 1640; GFX9-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 1641; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1642; GFX9-NEXT: ds_sub_rtn_u32 v1, v2, v1 1643; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1644; GFX9-NEXT: buffer_wbinvl1_vol 1645; GFX9-NEXT: BB8_2: 1646; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 1647; GFX9-NEXT: v_readfirstlane_b32 s2, v1 1648; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1649; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0 1650; GFX9-NEXT: s_mov_b32 s3, 0xf000 1651; GFX9-NEXT: s_mov_b32 s2, -1 1652; GFX9-NEXT: s_nop 0 1653; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1654; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 1655; GFX9-NEXT: s_endpgm 1656; 1657; GFX1064-LABEL: sub_i32_constant: 1658; GFX1064: ; %bb.0: ; %entry 1659; GFX1064-NEXT: s_mov_b64 s[2:3], exec 1660; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1661; GFX1064-NEXT: ; implicit-def: $vgpr1 1662; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1663; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 1664; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1665; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 1666; GFX1064-NEXT: s_cbranch_execz BB8_2 1667; GFX1064-NEXT: ; %bb.1: 1668; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] 1669; GFX1064-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 1670; GFX1064-NEXT: v_mul_u32_u24_e64 v1, s2, 5 1671; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1672; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 1673; GFX1064-NEXT: ds_sub_rtn_u32 v1, v2, v1 1674; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1675; GFX1064-NEXT: buffer_gl0_inv 1676; GFX1064-NEXT: buffer_gl1_inv 1677; GFX1064-NEXT: BB8_2: 1678; GFX1064-NEXT: v_nop 1679; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 1680; GFX1064-NEXT: v_readfirstlane_b32 s2, v1 1681; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1682; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 1683; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0 1684; GFX1064-NEXT: s_mov_b32 s2, -1 1685; GFX1064-NEXT: s_nop 0 1686; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1687; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 1688; GFX1064-NEXT: s_endpgm 1689; 1690; GFX1032-LABEL: sub_i32_constant: 1691; GFX1032: ; %bb.0: ; %entry 1692; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1693; GFX1032-NEXT: s_mov_b32 s2, exec_lo 1694; GFX1032-NEXT: ; implicit-def: $vcc_hi 1695; GFX1032-NEXT: ; implicit-def: $vgpr1 1696; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1697; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 1698; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo 1699; GFX1032-NEXT: s_cbranch_execz BB8_2 1700; GFX1032-NEXT: ; %bb.1: 1701; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2 1702; GFX1032-NEXT: v_mov_b32_e32 v2, local_var32@abs32@lo 1703; GFX1032-NEXT: v_mul_u32_u24_e64 v1, s2, 5 1704; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1705; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1706; GFX1032-NEXT: ds_sub_rtn_u32 v1, v2, v1 1707; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1708; GFX1032-NEXT: buffer_gl0_inv 1709; GFX1032-NEXT: buffer_gl1_inv 1710; GFX1032-NEXT: BB8_2: 1711; GFX1032-NEXT: v_nop 1712; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3 1713; GFX1032-NEXT: v_readfirstlane_b32 s2, v1 1714; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v0 1715; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 1716; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v0 1717; GFX1032-NEXT: s_mov_b32 s2, -1 1718; GFX1032-NEXT: s_nop 0 1719; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1720; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 1721; GFX1032-NEXT: s_endpgm 1722entry: 1723 %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 5 acq_rel 1724 store i32 %old, i32 addrspace(1)* %out 1725 ret void 1726} 1727 1728define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive) { 1729; 1730; 1731; GFX7LESS-LABEL: sub_i32_uniform: 1732; GFX7LESS: ; %bb.0: ; %entry 1733; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec 1734; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 1735; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0xb 1736; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 1737; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 1738; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1739; GFX7LESS-NEXT: ; implicit-def: $vgpr1 1740; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc 1741; GFX7LESS-NEXT: s_cbranch_execz BB9_2 1742; GFX7LESS-NEXT: ; %bb.1: 1743; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[6:7] 1744; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1745; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3 1746; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1747; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3 1748; GFX7LESS-NEXT: s_mov_b32 m0, -1 1749; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1750; GFX7LESS-NEXT: ds_sub_rtn_u32 v1, v1, v2 1751; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1752; GFX7LESS-NEXT: buffer_wbinvl1 1753; GFX7LESS-NEXT: BB9_2: 1754; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1] 1755; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1 1756; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 1757; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0 1758; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000 1759; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s0, v0 1760; GFX7LESS-NEXT: s_mov_b32 s6, -1 1761; GFX7LESS-NEXT: buffer_store_dword v0, off, s[4:7], 0 1762; GFX7LESS-NEXT: s_endpgm 1763; 1764; GFX8-LABEL: sub_i32_uniform: 1765; GFX8: ; %bb.0: ; %entry 1766; GFX8-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 1767; GFX8-NEXT: s_load_dword s0, s[0:1], 0x2c 1768; GFX8-NEXT: s_mov_b64 s[2:3], exec 1769; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1770; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1771; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1772; GFX8-NEXT: ; implicit-def: $vgpr1 1773; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc 1774; GFX8-NEXT: s_cbranch_execz BB9_2 1775; GFX8-NEXT: ; %bb.1: 1776; GFX8-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 1777; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1778; GFX8-NEXT: s_mul_i32 s1, s0, s1 1779; GFX8-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1780; GFX8-NEXT: v_mov_b32_e32 v2, s1 1781; GFX8-NEXT: s_mov_b32 m0, -1 1782; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1783; GFX8-NEXT: ds_sub_rtn_u32 v1, v1, v2 1784; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1785; GFX8-NEXT: buffer_wbinvl1_vol 1786; GFX8-NEXT: BB9_2: 1787; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] 1788; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1789; GFX8-NEXT: v_mul_lo_u32 v0, s0, v0 1790; GFX8-NEXT: v_readfirstlane_b32 s0, v1 1791; GFX8-NEXT: s_mov_b32 s7, 0xf000 1792; GFX8-NEXT: s_mov_b32 s6, -1 1793; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0 1794; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0 1795; GFX8-NEXT: s_endpgm 1796; 1797; GFX9-LABEL: sub_i32_uniform: 1798; GFX9: ; %bb.0: ; %entry 1799; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 1800; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c 1801; GFX9-NEXT: s_mov_b64 s[2:3], exec 1802; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1803; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1804; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1805; GFX9-NEXT: ; implicit-def: $vgpr1 1806; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc 1807; GFX9-NEXT: s_cbranch_execz BB9_2 1808; GFX9-NEXT: ; %bb.1: 1809; GFX9-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 1810; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1811; GFX9-NEXT: s_mul_i32 s1, s0, s1 1812; GFX9-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1813; GFX9-NEXT: v_mov_b32_e32 v2, s1 1814; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1815; GFX9-NEXT: ds_sub_rtn_u32 v1, v1, v2 1816; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1817; GFX9-NEXT: buffer_wbinvl1_vol 1818; GFX9-NEXT: BB9_2: 1819; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] 1820; GFX9-NEXT: s_waitcnt lgkmcnt(0) 1821; GFX9-NEXT: v_mul_lo_u32 v0, s0, v0 1822; GFX9-NEXT: v_readfirstlane_b32 s0, v1 1823; GFX9-NEXT: s_mov_b32 s7, 0xf000 1824; GFX9-NEXT: s_mov_b32 s6, -1 1825; GFX9-NEXT: v_sub_u32_e32 v0, s0, v0 1826; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0 1827; GFX9-NEXT: s_endpgm 1828; 1829; GFX1064-LABEL: sub_i32_uniform: 1830; GFX1064: ; %bb.0: ; %entry 1831; GFX1064-NEXT: s_mov_b64 s[2:3], exec 1832; GFX1064-NEXT: s_clause 0x1 1833; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 1834; GFX1064-NEXT: s_load_dword s0, s[0:1], 0x2c 1835; GFX1064-NEXT: ; implicit-def: $vgpr1 1836; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1837; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 1838; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1839; GFX1064-NEXT: s_and_saveexec_b64 s[6:7], vcc 1840; GFX1064-NEXT: s_cbranch_execz BB9_2 1841; GFX1064-NEXT: ; %bb.1: 1842; GFX1064-NEXT: s_bcnt1_i32_b64 s1, s[2:3] 1843; GFX1064-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1844; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1845; GFX1064-NEXT: s_mul_i32 s1, s0, s1 1846; GFX1064-NEXT: v_mov_b32_e32 v2, s1 1847; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1848; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 1849; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2 1850; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1851; GFX1064-NEXT: buffer_gl0_inv 1852; GFX1064-NEXT: buffer_gl1_inv 1853; GFX1064-NEXT: BB9_2: 1854; GFX1064-NEXT: v_nop 1855; GFX1064-NEXT: s_or_b64 exec, exec, s[6:7] 1856; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 1857; GFX1064-NEXT: v_mul_lo_u32 v0, s0, v0 1858; GFX1064-NEXT: v_readfirstlane_b32 s0, v1 1859; GFX1064-NEXT: s_mov_b32 s7, 0x31016000 1860; GFX1064-NEXT: s_mov_b32 s6, -1 1861; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s0, v0 1862; GFX1064-NEXT: buffer_store_dword v0, off, s[4:7], 0 1863; GFX1064-NEXT: s_endpgm 1864; 1865; GFX1032-LABEL: sub_i32_uniform: 1866; GFX1032: ; %bb.0: ; %entry 1867; GFX1032-NEXT: s_clause 0x1 1868; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 1869; GFX1032-NEXT: s_load_dword s0, s[0:1], 0x2c 1870; GFX1032-NEXT: s_mov_b32 s2, exec_lo 1871; GFX1032-NEXT: ; implicit-def: $vcc_hi 1872; GFX1032-NEXT: ; implicit-def: $vgpr1 1873; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 1874; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 1875; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo 1876; GFX1032-NEXT: s_cbranch_execz BB9_2 1877; GFX1032-NEXT: ; %bb.1: 1878; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2 1879; GFX1032-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1880; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1881; GFX1032-NEXT: s_mul_i32 s2, s0, s2 1882; GFX1032-NEXT: v_mov_b32_e32 v2, s2 1883; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1884; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 1885; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2 1886; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1887; GFX1032-NEXT: buffer_gl0_inv 1888; GFX1032-NEXT: buffer_gl1_inv 1889; GFX1032-NEXT: BB9_2: 1890; GFX1032-NEXT: v_nop 1891; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1 1892; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 1893; GFX1032-NEXT: v_mul_lo_u32 v0, s0, v0 1894; GFX1032-NEXT: v_readfirstlane_b32 s0, v1 1895; GFX1032-NEXT: s_mov_b32 s7, 0x31016000 1896; GFX1032-NEXT: s_mov_b32 s6, -1 1897; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s0, v0 1898; GFX1032-NEXT: buffer_store_dword v0, off, s[4:7], 0 1899; GFX1032-NEXT: s_endpgm 1900entry: 1901 %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %subitive acq_rel 1902 store i32 %old, i32 addrspace(1)* %out 1903 ret void 1904} 1905 1906define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) { 1907; 1908; 1909; GFX7LESS-LABEL: sub_i32_varying: 1910; GFX7LESS: ; %bb.0: ; %entry 1911; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 1912; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 1913; GFX7LESS-NEXT: s_mov_b32 m0, -1 1914; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1915; GFX7LESS-NEXT: ds_sub_rtn_u32 v0, v1, v0 1916; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1917; GFX7LESS-NEXT: buffer_wbinvl1 1918; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 1919; GFX7LESS-NEXT: s_mov_b32 s2, -1 1920; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 1921; GFX7LESS-NEXT: s_endpgm 1922; 1923; GFX8-LABEL: sub_i32_varying: 1924; GFX8: ; %bb.0: ; %entry 1925; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1926; GFX8-NEXT: s_mov_b64 s[2:3], exec 1927; GFX8-NEXT: v_mov_b32_e32 v2, v0 1928; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 1929; GFX8-NEXT: v_mov_b32_e32 v1, 0 1930; GFX8-NEXT: s_mov_b64 exec, s[4:5] 1931; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1932; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1933; GFX8-NEXT: s_not_b64 exec, exec 1934; GFX8-NEXT: v_mov_b32_e32 v2, 0 1935; GFX8-NEXT: s_not_b64 exec, exec 1936; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 1937; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 1938; GFX8-NEXT: s_nop 1 1939; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 1940; GFX8-NEXT: s_nop 1 1941; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 1942; GFX8-NEXT: s_nop 1 1943; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 1944; GFX8-NEXT: s_nop 1 1945; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 1946; GFX8-NEXT: s_nop 1 1947; GFX8-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 1948; GFX8-NEXT: v_readlane_b32 s2, v2, 63 1949; GFX8-NEXT: s_nop 0 1950; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 1951; GFX8-NEXT: s_mov_b64 exec, s[4:5] 1952; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 1953; GFX8-NEXT: ; implicit-def: $vgpr0 1954; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 1955; GFX8-NEXT: s_cbranch_execz BB10_2 1956; GFX8-NEXT: ; %bb.1: 1957; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 1958; GFX8-NEXT: v_mov_b32_e32 v3, s2 1959; GFX8-NEXT: s_mov_b32 m0, -1 1960; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1961; GFX8-NEXT: ds_sub_rtn_u32 v0, v0, v3 1962; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1963; GFX8-NEXT: buffer_wbinvl1_vol 1964; GFX8-NEXT: BB10_2: 1965; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 1966; GFX8-NEXT: v_readfirstlane_b32 s2, v0 1967; GFX8-NEXT: v_mov_b32_e32 v0, v1 1968; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v0 1969; GFX8-NEXT: s_mov_b32 s3, 0xf000 1970; GFX8-NEXT: s_mov_b32 s2, -1 1971; GFX8-NEXT: s_nop 0 1972; GFX8-NEXT: s_waitcnt lgkmcnt(0) 1973; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 1974; GFX8-NEXT: s_endpgm 1975; 1976; GFX9-LABEL: sub_i32_varying: 1977; GFX9: ; %bb.0: ; %entry 1978; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 1979; GFX9-NEXT: s_mov_b64 s[2:3], exec 1980; GFX9-NEXT: v_mov_b32_e32 v2, v0 1981; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 1982; GFX9-NEXT: v_mov_b32_e32 v1, 0 1983; GFX9-NEXT: s_mov_b64 exec, s[4:5] 1984; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 1985; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 1986; GFX9-NEXT: s_not_b64 exec, exec 1987; GFX9-NEXT: v_mov_b32_e32 v2, 0 1988; GFX9-NEXT: s_not_b64 exec, exec 1989; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 1990; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 1991; GFX9-NEXT: s_nop 1 1992; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 1993; GFX9-NEXT: s_nop 1 1994; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 1995; GFX9-NEXT: s_nop 1 1996; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 1997; GFX9-NEXT: s_nop 1 1998; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 1999; GFX9-NEXT: s_nop 1 2000; GFX9-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 2001; GFX9-NEXT: v_readlane_b32 s2, v2, 63 2002; GFX9-NEXT: s_nop 0 2003; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 2004; GFX9-NEXT: s_mov_b64 exec, s[4:5] 2005; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2006; GFX9-NEXT: ; implicit-def: $vgpr0 2007; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 2008; GFX9-NEXT: s_cbranch_execz BB10_2 2009; GFX9-NEXT: ; %bb.1: 2010; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2011; GFX9-NEXT: v_mov_b32_e32 v3, s2 2012; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2013; GFX9-NEXT: ds_sub_rtn_u32 v0, v0, v3 2014; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2015; GFX9-NEXT: buffer_wbinvl1_vol 2016; GFX9-NEXT: BB10_2: 2017; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 2018; GFX9-NEXT: v_readfirstlane_b32 s2, v0 2019; GFX9-NEXT: v_mov_b32_e32 v0, v1 2020; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0 2021; GFX9-NEXT: s_mov_b32 s3, 0xf000 2022; GFX9-NEXT: s_mov_b32 s2, -1 2023; GFX9-NEXT: s_nop 0 2024; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2025; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 2026; GFX9-NEXT: s_endpgm 2027; 2028; GFX1064-LABEL: sub_i32_varying: 2029; GFX1064: ; %bb.0: ; %entry 2030; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2031; GFX1064-NEXT: s_mov_b64 s[2:3], exec 2032; GFX1064-NEXT: v_mov_b32_e32 v2, v0 2033; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 2034; GFX1064-NEXT: v_mov_b32_e32 v1, 0 2035; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 2036; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 2037; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 2038; GFX1064-NEXT: s_not_b64 exec, exec 2039; GFX1064-NEXT: v_mov_b32_e32 v2, 0 2040; GFX1064-NEXT: s_not_b64 exec, exec 2041; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 2042; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 2043; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 2044; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 2045; GFX1064-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 2046; GFX1064-NEXT: v_mov_b32_e32 v3, v2 2047; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 2048; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 2049; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 2050; GFX1064-NEXT: v_mov_b32_e32 v3, s2 2051; GFX1064-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 2052; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 2053; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2054; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 2055; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 2056; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 2057; GFX1064-NEXT: s_mov_b32 s2, -1 2058; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 2059; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 2060; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 2061; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 2062; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2063; GFX1064-NEXT: ; implicit-def: $vgpr0 2064; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 2065; GFX1064-NEXT: s_cbranch_execz BB10_2 2066; GFX1064-NEXT: ; %bb.1: 2067; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2068; GFX1064-NEXT: v_mov_b32_e32 v7, s3 2069; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2070; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 2071; GFX1064-NEXT: ds_sub_rtn_u32 v0, v0, v7 2072; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2073; GFX1064-NEXT: buffer_gl0_inv 2074; GFX1064-NEXT: buffer_gl1_inv 2075; GFX1064-NEXT: BB10_2: 2076; GFX1064-NEXT: v_nop 2077; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 2078; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 2079; GFX1064-NEXT: v_mov_b32_e32 v0, v1 2080; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s3, v0 2081; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 2082; GFX1064-NEXT: s_nop 1 2083; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 2084; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 2085; GFX1064-NEXT: s_endpgm 2086; 2087; GFX1032-LABEL: sub_i32_varying: 2088; GFX1032: ; %bb.0: ; %entry 2089; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2090; GFX1032-NEXT: s_mov_b32 s2, exec_lo 2091; GFX1032-NEXT: ; implicit-def: $vcc_hi 2092; GFX1032-NEXT: v_mov_b32_e32 v2, v0 2093; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 2094; GFX1032-NEXT: v_mov_b32_e32 v1, 0 2095; GFX1032-NEXT: s_mov_b32 exec_lo, s3 2096; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 2097; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 2098; GFX1032-NEXT: v_mov_b32_e32 v2, 0 2099; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 2100; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 2101; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 2102; GFX1032-NEXT: s_mov_b32 s2, -1 2103; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 2104; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 2105; GFX1032-NEXT: v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 2106; GFX1032-NEXT: v_mov_b32_e32 v3, v2 2107; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 2108; GFX1032-NEXT: v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 2109; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 2110; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2111; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 2112; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 2113; GFX1032-NEXT: s_mov_b32 exec_lo, s4 2114; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 2115; GFX1032-NEXT: ; implicit-def: $vgpr0 2116; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 2117; GFX1032-NEXT: s_cbranch_execz BB10_2 2118; GFX1032-NEXT: ; %bb.1: 2119; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2120; GFX1032-NEXT: v_mov_b32_e32 v7, s3 2121; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2122; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 2123; GFX1032-NEXT: ds_sub_rtn_u32 v0, v0, v7 2124; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2125; GFX1032-NEXT: buffer_gl0_inv 2126; GFX1032-NEXT: buffer_gl1_inv 2127; GFX1032-NEXT: BB10_2: 2128; GFX1032-NEXT: v_nop 2129; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 2130; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 2131; GFX1032-NEXT: v_mov_b32_e32 v0, v1 2132; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s3, v0 2133; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 2134; GFX1032-NEXT: s_nop 1 2135; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 2136; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 2137; GFX1032-NEXT: s_endpgm 2138entry: 2139 %lane = call i32 @llvm.amdgcn.workitem.id.x() 2140 %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %lane acq_rel 2141 store i32 %old, i32 addrspace(1)* %out 2142 ret void 2143} 2144 2145define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) { 2146; 2147; 2148; GFX7LESS-LABEL: sub_i64_constant: 2149; GFX7LESS: ; %bb.0: ; %entry 2150; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec 2151; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 2152; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0 2153; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0 2154; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2155; GFX7LESS-NEXT: ; implicit-def: $vgpr1_vgpr2 2156; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 2157; GFX7LESS-NEXT: s_cbranch_execz BB11_2 2158; GFX7LESS-NEXT: ; %bb.1: 2159; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 2160; GFX7LESS-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2161; GFX7LESS-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 2162; GFX7LESS-NEXT: v_mul_u32_u24_e64 v1, s4, 5 2163; GFX7LESS-NEXT: s_mov_b32 m0, -1 2164; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2165; GFX7LESS-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2166; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2167; GFX7LESS-NEXT: buffer_wbinvl1 2168; GFX7LESS-NEXT: BB11_2: 2169; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 2170; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v1 2171; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v2 2172; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v0 2173; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v0 2174; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 2175; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4 2176; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 2177; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc 2178; GFX7LESS-NEXT: s_mov_b32 s2, -1 2179; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 2180; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2181; GFX7LESS-NEXT: s_endpgm 2182; 2183; GFX8-LABEL: sub_i64_constant: 2184; GFX8: ; %bb.0: ; %entry 2185; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2186; GFX8-NEXT: s_mov_b64 s[4:5], exec 2187; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 2188; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 2189; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2190; GFX8-NEXT: ; implicit-def: $vgpr1_vgpr2 2191; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 2192; GFX8-NEXT: s_cbranch_execz BB11_2 2193; GFX8-NEXT: ; %bb.1: 2194; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 2195; GFX8-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 2196; GFX8-NEXT: v_mul_u32_u24_e64 v1, s4, 5 2197; GFX8-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2198; GFX8-NEXT: s_mov_b32 m0, -1 2199; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2200; GFX8-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2201; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2202; GFX8-NEXT: buffer_wbinvl1_vol 2203; GFX8-NEXT: BB11_2: 2204; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 2205; GFX8-NEXT: v_readfirstlane_b32 s3, v2 2206; GFX8-NEXT: v_readfirstlane_b32 s2, v1 2207; GFX8-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v0 2208; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v0 2209; GFX8-NEXT: v_mov_b32_e32 v2, s3 2210; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v0 2211; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc 2212; GFX8-NEXT: s_mov_b32 s3, 0xf000 2213; GFX8-NEXT: s_mov_b32 s2, -1 2214; GFX8-NEXT: s_waitcnt lgkmcnt(0) 2215; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2216; GFX8-NEXT: s_endpgm 2217; 2218; GFX9-LABEL: sub_i64_constant: 2219; GFX9: ; %bb.0: ; %entry 2220; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2221; GFX9-NEXT: s_mov_b64 s[4:5], exec 2222; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 2223; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 2224; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2225; GFX9-NEXT: ; implicit-def: $vgpr1_vgpr2 2226; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 2227; GFX9-NEXT: s_cbranch_execz BB11_2 2228; GFX9-NEXT: ; %bb.1: 2229; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 2230; GFX9-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 2231; GFX9-NEXT: v_mul_u32_u24_e64 v1, s4, 5 2232; GFX9-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2233; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2234; GFX9-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2235; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2236; GFX9-NEXT: buffer_wbinvl1_vol 2237; GFX9-NEXT: BB11_2: 2238; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 2239; GFX9-NEXT: v_readfirstlane_b32 s3, v2 2240; GFX9-NEXT: v_readfirstlane_b32 s2, v1 2241; GFX9-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v0 2242; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v0 2243; GFX9-NEXT: v_mov_b32_e32 v2, s3 2244; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s2, v0 2245; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc 2246; GFX9-NEXT: s_mov_b32 s3, 0xf000 2247; GFX9-NEXT: s_mov_b32 s2, -1 2248; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2249; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2250; GFX9-NEXT: s_endpgm 2251; 2252; GFX1064-LABEL: sub_i64_constant: 2253; GFX1064: ; %bb.0: ; %entry 2254; GFX1064-NEXT: s_mov_b64 s[4:5], exec 2255; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2256; GFX1064-NEXT: ; implicit-def: $vgpr1_vgpr2 2257; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0 2258; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s5, v0 2259; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2260; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 2261; GFX1064-NEXT: s_cbranch_execz BB11_2 2262; GFX1064-NEXT: ; %bb.1: 2263; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5] 2264; GFX1064-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2265; GFX1064-NEXT: v_mul_hi_u32_u24_e64 v2, s4, 5 2266; GFX1064-NEXT: v_mul_u32_u24_e64 v1, s4, 5 2267; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2268; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 2269; GFX1064-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2270; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2271; GFX1064-NEXT: buffer_gl0_inv 2272; GFX1064-NEXT: buffer_gl1_inv 2273; GFX1064-NEXT: BB11_2: 2274; GFX1064-NEXT: v_nop 2275; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 2276; GFX1064-NEXT: v_readfirstlane_b32 s2, v1 2277; GFX1064-NEXT: v_mul_u32_u24_e32 v1, 5, v0 2278; GFX1064-NEXT: v_readfirstlane_b32 s3, v2 2279; GFX1064-NEXT: v_mul_hi_u32_u24_e32 v2, 5, v0 2280; GFX1064-NEXT: v_sub_co_u32_e64 v0, vcc, s2, v1 2281; GFX1064-NEXT: s_mov_b32 s2, -1 2282; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v2, vcc 2283; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 2284; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 2285; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2286; GFX1064-NEXT: s_endpgm 2287; 2288; GFX1032-LABEL: sub_i64_constant: 2289; GFX1032: ; %bb.0: ; %entry 2290; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2291; GFX1032-NEXT: s_mov_b32 s3, exec_lo 2292; GFX1032-NEXT: ; implicit-def: $vcc_hi 2293; GFX1032-NEXT: ; implicit-def: $vgpr1_vgpr2 2294; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s3, 0 2295; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 2296; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 2297; GFX1032-NEXT: s_cbranch_execz BB11_2 2298; GFX1032-NEXT: ; %bb.1: 2299; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3 2300; GFX1032-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2301; GFX1032-NEXT: v_mul_hi_u32_u24_e64 v2, s3, 5 2302; GFX1032-NEXT: v_mul_u32_u24_e64 v1, s3, 5 2303; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2304; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 2305; GFX1032-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2306; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2307; GFX1032-NEXT: buffer_gl0_inv 2308; GFX1032-NEXT: buffer_gl1_inv 2309; GFX1032-NEXT: BB11_2: 2310; GFX1032-NEXT: v_nop 2311; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 2312; GFX1032-NEXT: v_readfirstlane_b32 s2, v1 2313; GFX1032-NEXT: v_mul_u32_u24_e32 v1, 5, v0 2314; GFX1032-NEXT: v_readfirstlane_b32 s3, v2 2315; GFX1032-NEXT: v_mul_hi_u32_u24_e32 v2, 5, v0 2316; GFX1032-NEXT: v_sub_co_u32_e64 v0, vcc_lo, s2, v1 2317; GFX1032-NEXT: s_mov_b32 s2, -1 2318; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo 2319; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 2320; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 2321; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2322; GFX1032-NEXT: s_endpgm 2323entry: 2324 %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 5 acq_rel 2325 store i64 %old, i64 addrspace(1)* %out 2326 ret void 2327} 2328 2329define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive) { 2330; 2331; 2332; GFX7LESS-LABEL: sub_i64_uniform: 2333; GFX7LESS: ; %bb.0: ; %entry 2334; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec 2335; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 2336; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 2337; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 2338; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2339; GFX7LESS-NEXT: ; implicit-def: $vgpr1_vgpr2 2340; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc 2341; GFX7LESS-NEXT: s_cbranch_execz BB12_2 2342; GFX7LESS-NEXT: ; %bb.1: 2343; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 2344; GFX7LESS-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2345; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 2346; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6 2347; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6 2348; GFX7LESS-NEXT: v_mul_hi_u32 v1, s2, v1 2349; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6 2350; GFX7LESS-NEXT: v_add_i32_e32 v2, vcc, s7, v1 2351; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6 2352; GFX7LESS-NEXT: s_mov_b32 m0, -1 2353; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2354; GFX7LESS-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2355; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2356; GFX7LESS-NEXT: buffer_wbinvl1 2357; GFX7LESS-NEXT: BB12_2: 2358; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5] 2359; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000 2360; GFX7LESS-NEXT: s_mov_b32 s6, -1 2361; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 2362; GFX7LESS-NEXT: s_mov_b32 s4, s0 2363; GFX7LESS-NEXT: s_mov_b32 s5, s1 2364; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1 2365; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v2 2366; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v0 2367; GFX7LESS-NEXT: v_mul_hi_u32 v2, s2, v0 2368; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0 2369; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v2, v1 2370; GFX7LESS-NEXT: v_mov_b32_e32 v2, s1 2371; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s0, v0 2372; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc 2373; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 2374; GFX7LESS-NEXT: s_endpgm 2375; 2376; GFX8-LABEL: sub_i64_uniform: 2377; GFX8: ; %bb.0: ; %entry 2378; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 2379; GFX8-NEXT: s_mov_b64 s[6:7], exec 2380; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 2381; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0 2382; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2383; GFX8-NEXT: ; implicit-def: $vgpr1_vgpr2 2384; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 2385; GFX8-NEXT: s_cbranch_execz BB12_2 2386; GFX8-NEXT: ; %bb.1: 2387; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 2388; GFX8-NEXT: v_mov_b32_e32 v1, s6 2389; GFX8-NEXT: s_waitcnt lgkmcnt(0) 2390; GFX8-NEXT: v_mul_hi_u32 v1, s2, v1 2391; GFX8-NEXT: s_mul_i32 s7, s3, s6 2392; GFX8-NEXT: s_mul_i32 s6, s2, s6 2393; GFX8-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2394; GFX8-NEXT: v_add_u32_e32 v2, vcc, s7, v1 2395; GFX8-NEXT: v_mov_b32_e32 v1, s6 2396; GFX8-NEXT: s_mov_b32 m0, -1 2397; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2398; GFX8-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2399; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2400; GFX8-NEXT: buffer_wbinvl1_vol 2401; GFX8-NEXT: BB12_2: 2402; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 2403; GFX8-NEXT: s_waitcnt lgkmcnt(0) 2404; GFX8-NEXT: s_mov_b32 s4, s0 2405; GFX8-NEXT: v_readfirstlane_b32 s0, v1 2406; GFX8-NEXT: v_mul_lo_u32 v1, s3, v0 2407; GFX8-NEXT: v_mul_hi_u32 v3, s2, v0 2408; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0 2409; GFX8-NEXT: s_mov_b32 s5, s1 2410; GFX8-NEXT: v_readfirstlane_b32 s1, v2 2411; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1 2412; GFX8-NEXT: v_mov_b32_e32 v2, s1 2413; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0 2414; GFX8-NEXT: s_mov_b32 s7, 0xf000 2415; GFX8-NEXT: s_mov_b32 s6, -1 2416; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc 2417; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 2418; GFX8-NEXT: s_endpgm 2419; 2420; GFX9-LABEL: sub_i64_uniform: 2421; GFX9: ; %bb.0: ; %entry 2422; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 2423; GFX9-NEXT: s_mov_b64 s[6:7], exec 2424; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 2425; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0 2426; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2427; GFX9-NEXT: ; implicit-def: $vgpr1_vgpr2 2428; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 2429; GFX9-NEXT: s_cbranch_execz BB12_2 2430; GFX9-NEXT: ; %bb.1: 2431; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 2432; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2433; GFX9-NEXT: s_mul_i32 s7, s3, s6 2434; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6 2435; GFX9-NEXT: s_add_i32 s8, s8, s7 2436; GFX9-NEXT: s_mul_i32 s6, s2, s6 2437; GFX9-NEXT: v_mov_b32_e32 v1, s6 2438; GFX9-NEXT: v_mov_b32_e32 v2, s8 2439; GFX9-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2440; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2441; GFX9-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2442; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2443; GFX9-NEXT: buffer_wbinvl1_vol 2444; GFX9-NEXT: BB12_2: 2445; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 2446; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2447; GFX9-NEXT: v_mul_lo_u32 v3, s3, v0 2448; GFX9-NEXT: v_mul_hi_u32 v4, s2, v0 2449; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0 2450; GFX9-NEXT: s_mov_b32 s4, s0 2451; GFX9-NEXT: v_readfirstlane_b32 s0, v1 2452; GFX9-NEXT: s_mov_b32 s5, s1 2453; GFX9-NEXT: v_readfirstlane_b32 s1, v2 2454; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 2455; GFX9-NEXT: v_mov_b32_e32 v2, s1 2456; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0 2457; GFX9-NEXT: s_mov_b32 s7, 0xf000 2458; GFX9-NEXT: s_mov_b32 s6, -1 2459; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc 2460; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 2461; GFX9-NEXT: s_endpgm 2462; 2463; GFX1064-LABEL: sub_i64_uniform: 2464; GFX1064: ; %bb.0: ; %entry 2465; GFX1064-NEXT: s_mov_b64 s[6:7], exec 2466; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 2467; GFX1064-NEXT: ; implicit-def: $vgpr1_vgpr2 2468; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 2469; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s7, v0 2470; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2471; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 2472; GFX1064-NEXT: s_cbranch_execz BB12_2 2473; GFX1064-NEXT: ; %bb.1: 2474; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7] 2475; GFX1064-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2476; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 2477; GFX1064-NEXT: s_mul_i32 s7, s3, s6 2478; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6 2479; GFX1064-NEXT: s_mul_i32 s6, s2, s6 2480; GFX1064-NEXT: s_add_i32 s8, s8, s7 2481; GFX1064-NEXT: v_mov_b32_e32 v1, s6 2482; GFX1064-NEXT: v_mov_b32_e32 v2, s8 2483; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2484; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 2485; GFX1064-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2486; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2487; GFX1064-NEXT: buffer_gl0_inv 2488; GFX1064-NEXT: buffer_gl1_inv 2489; GFX1064-NEXT: BB12_2: 2490; GFX1064-NEXT: v_nop 2491; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 2492; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 2493; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v0 2494; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v0 2495; GFX1064-NEXT: v_mul_lo_u32 v0, s2, v0 2496; GFX1064-NEXT: v_readfirstlane_b32 s4, v1 2497; GFX1064-NEXT: v_readfirstlane_b32 s5, v2 2498; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 2499; GFX1064-NEXT: s_mov_b32 s2, -1 2500; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 2501; GFX1064-NEXT: v_sub_co_u32_e64 v0, vcc, s4, v0 2502; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s5, v1, vcc 2503; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2504; GFX1064-NEXT: s_endpgm 2505; 2506; GFX1032-LABEL: sub_i64_uniform: 2507; GFX1032: ; %bb.0: ; %entry 2508; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 2509; GFX1032-NEXT: s_mov_b32 s5, exec_lo 2510; GFX1032-NEXT: ; implicit-def: $vcc_hi 2511; GFX1032-NEXT: ; implicit-def: $vgpr1_vgpr2 2512; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s5, 0 2513; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 2514; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 2515; GFX1032-NEXT: s_cbranch_execz BB12_2 2516; GFX1032-NEXT: ; %bb.1: 2517; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s5 2518; GFX1032-NEXT: v_mov_b32_e32 v3, local_var64@abs32@lo 2519; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 2520; GFX1032-NEXT: s_mul_i32 s6, s3, s5 2521; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5 2522; GFX1032-NEXT: s_mul_i32 s5, s2, s5 2523; GFX1032-NEXT: s_add_i32 s7, s7, s6 2524; GFX1032-NEXT: v_mov_b32_e32 v1, s5 2525; GFX1032-NEXT: v_mov_b32_e32 v2, s7 2526; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2527; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 2528; GFX1032-NEXT: ds_sub_rtn_u64 v[1:2], v3, v[1:2] 2529; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2530; GFX1032-NEXT: buffer_gl0_inv 2531; GFX1032-NEXT: buffer_gl1_inv 2532; GFX1032-NEXT: BB12_2: 2533; GFX1032-NEXT: v_nop 2534; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 2535; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 2536; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v0 2537; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v0 2538; GFX1032-NEXT: v_mul_lo_u32 v0, s2, v0 2539; GFX1032-NEXT: v_readfirstlane_b32 s4, v1 2540; GFX1032-NEXT: v_readfirstlane_b32 s5, v2 2541; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 2542; GFX1032-NEXT: s_mov_b32 s2, -1 2543; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 2544; GFX1032-NEXT: v_sub_co_u32_e64 v0, vcc_lo, s4, v0 2545; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo 2546; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2547; GFX1032-NEXT: s_endpgm 2548entry: 2549 %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %subitive acq_rel 2550 store i64 %old, i64 addrspace(1)* %out 2551 ret void 2552} 2553 2554; GCN-NOT: v_mbcnt_lo_u32_b32 2555; GCN-NOT: v_mbcnt_hi_u32_b32 2556; GCN-NOT: s_bcnt1_i32_b64 2557define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) { 2558; 2559; 2560; GFX7LESS-LABEL: sub_i64_varying: 2561; GFX7LESS: ; %bb.0: ; %entry 2562; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 2563; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 2564; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 2565; GFX7LESS-NEXT: s_mov_b32 m0, -1 2566; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2567; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v2, v[0:1] 2568; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2569; GFX7LESS-NEXT: buffer_wbinvl1 2570; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 2571; GFX7LESS-NEXT: s_mov_b32 s2, -1 2572; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2573; GFX7LESS-NEXT: s_endpgm 2574; 2575; GFX8-LABEL: sub_i64_varying: 2576; GFX8: ; %bb.0: ; %entry 2577; GFX8-NEXT: v_mov_b32_e32 v1, 0 2578; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 2579; GFX8-NEXT: s_mov_b32 m0, -1 2580; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2581; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2582; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v2, v[0:1] 2583; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2584; GFX8-NEXT: buffer_wbinvl1_vol 2585; GFX8-NEXT: s_mov_b32 s3, 0xf000 2586; GFX8-NEXT: s_mov_b32 s2, -1 2587; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2588; GFX8-NEXT: s_endpgm 2589; 2590; GFX9-LABEL: sub_i64_varying: 2591; GFX9: ; %bb.0: ; %entry 2592; GFX9-NEXT: v_mov_b32_e32 v1, 0 2593; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 2594; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2595; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2596; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v2, v[0:1] 2597; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2598; GFX9-NEXT: buffer_wbinvl1_vol 2599; GFX9-NEXT: s_mov_b32 s3, 0xf000 2600; GFX9-NEXT: s_mov_b32 s2, -1 2601; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2602; GFX9-NEXT: s_endpgm 2603; 2604; GFX1064-LABEL: sub_i64_varying: 2605; GFX1064: ; %bb.0: ; %entry 2606; GFX1064-NEXT: v_mov_b32_e32 v1, 0 2607; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 2608; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2609; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 2610; GFX1064-NEXT: s_mov_b32 s2, -1 2611; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2612; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 2613; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v2, v[0:1] 2614; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2615; GFX1064-NEXT: buffer_gl0_inv 2616; GFX1064-NEXT: buffer_gl1_inv 2617; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2618; GFX1064-NEXT: s_endpgm 2619; 2620; GFX1032-LABEL: sub_i64_varying: 2621; GFX1032: ; %bb.0: ; %entry 2622; GFX1032-NEXT: v_mov_b32_e32 v1, 0 2623; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 2624; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2625; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 2626; GFX1032-NEXT: s_mov_b32 s2, -1 2627; GFX1032-NEXT: ; implicit-def: $vcc_hi 2628; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2629; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 2630; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v2, v[0:1] 2631; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2632; GFX1032-NEXT: buffer_gl0_inv 2633; GFX1032-NEXT: buffer_gl1_inv 2634; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 2635; GFX1032-NEXT: s_endpgm 2636entry: 2637 %lane = call i32 @llvm.amdgcn.workitem.id.x() 2638 %zext = zext i32 %lane to i64 2639 %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %zext acq_rel 2640 store i64 %old, i64 addrspace(1)* %out 2641 ret void 2642} 2643 2644define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) { 2645; 2646; 2647; GFX7LESS-LABEL: and_i32_varying: 2648; GFX7LESS: ; %bb.0: ; %entry 2649; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 2650; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 2651; GFX7LESS-NEXT: s_mov_b32 m0, -1 2652; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2653; GFX7LESS-NEXT: ds_and_rtn_b32 v0, v1, v0 2654; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2655; GFX7LESS-NEXT: buffer_wbinvl1 2656; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 2657; GFX7LESS-NEXT: s_mov_b32 s2, -1 2658; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 2659; GFX7LESS-NEXT: s_endpgm 2660; 2661; GFX8-LABEL: and_i32_varying: 2662; GFX8: ; %bb.0: ; %entry 2663; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2664; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 2665; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 2666; GFX8-NEXT: v_mov_b32_e32 v2, v0 2667; GFX8-NEXT: s_or_saveexec_b64 s[2:3], -1 2668; GFX8-NEXT: v_mov_b32_e32 v1, -1 2669; GFX8-NEXT: s_mov_b64 exec, s[2:3] 2670; GFX8-NEXT: s_not_b64 exec, exec 2671; GFX8-NEXT: v_mov_b32_e32 v2, -1 2672; GFX8-NEXT: s_not_b64 exec, exec 2673; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 2674; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2675; GFX8-NEXT: s_nop 1 2676; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 2677; GFX8-NEXT: s_nop 1 2678; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 2679; GFX8-NEXT: s_nop 1 2680; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 2681; GFX8-NEXT: s_nop 1 2682; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 2683; GFX8-NEXT: s_nop 1 2684; GFX8-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 2685; GFX8-NEXT: v_readlane_b32 s2, v2, 63 2686; GFX8-NEXT: s_nop 0 2687; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 2688; GFX8-NEXT: s_mov_b64 exec, s[4:5] 2689; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 2690; GFX8-NEXT: ; implicit-def: $vgpr0 2691; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 2692; GFX8-NEXT: s_cbranch_execz BB14_2 2693; GFX8-NEXT: ; %bb.1: 2694; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2695; GFX8-NEXT: v_mov_b32_e32 v3, s2 2696; GFX8-NEXT: s_mov_b32 m0, -1 2697; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2698; GFX8-NEXT: ds_and_rtn_b32 v0, v0, v3 2699; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2700; GFX8-NEXT: buffer_wbinvl1_vol 2701; GFX8-NEXT: BB14_2: 2702; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 2703; GFX8-NEXT: v_readfirstlane_b32 s2, v0 2704; GFX8-NEXT: v_mov_b32_e32 v0, v1 2705; GFX8-NEXT: v_and_b32_e32 v0, s2, v0 2706; GFX8-NEXT: s_mov_b32 s3, 0xf000 2707; GFX8-NEXT: s_mov_b32 s2, -1 2708; GFX8-NEXT: s_nop 0 2709; GFX8-NEXT: s_waitcnt lgkmcnt(0) 2710; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 2711; GFX8-NEXT: s_endpgm 2712; 2713; GFX9-LABEL: and_i32_varying: 2714; GFX9: ; %bb.0: ; %entry 2715; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2716; GFX9-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 2717; GFX9-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 2718; GFX9-NEXT: v_mov_b32_e32 v2, v0 2719; GFX9-NEXT: s_or_saveexec_b64 s[2:3], -1 2720; GFX9-NEXT: v_mov_b32_e32 v1, -1 2721; GFX9-NEXT: s_mov_b64 exec, s[2:3] 2722; GFX9-NEXT: s_not_b64 exec, exec 2723; GFX9-NEXT: v_mov_b32_e32 v2, -1 2724; GFX9-NEXT: s_not_b64 exec, exec 2725; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 2726; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2727; GFX9-NEXT: s_nop 1 2728; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 2729; GFX9-NEXT: s_nop 1 2730; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 2731; GFX9-NEXT: s_nop 1 2732; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 2733; GFX9-NEXT: s_nop 1 2734; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 2735; GFX9-NEXT: s_nop 1 2736; GFX9-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 2737; GFX9-NEXT: v_readlane_b32 s2, v2, 63 2738; GFX9-NEXT: s_nop 0 2739; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 2740; GFX9-NEXT: s_mov_b64 exec, s[4:5] 2741; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 2742; GFX9-NEXT: ; implicit-def: $vgpr0 2743; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 2744; GFX9-NEXT: s_cbranch_execz BB14_2 2745; GFX9-NEXT: ; %bb.1: 2746; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2747; GFX9-NEXT: v_mov_b32_e32 v3, s2 2748; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2749; GFX9-NEXT: ds_and_rtn_b32 v0, v0, v3 2750; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2751; GFX9-NEXT: buffer_wbinvl1_vol 2752; GFX9-NEXT: BB14_2: 2753; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 2754; GFX9-NEXT: v_readfirstlane_b32 s2, v0 2755; GFX9-NEXT: v_mov_b32_e32 v0, v1 2756; GFX9-NEXT: v_and_b32_e32 v0, s2, v0 2757; GFX9-NEXT: s_mov_b32 s3, 0xf000 2758; GFX9-NEXT: s_mov_b32 s2, -1 2759; GFX9-NEXT: s_nop 0 2760; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2761; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 2762; GFX9-NEXT: s_endpgm 2763; 2764; GFX1064-LABEL: and_i32_varying: 2765; GFX1064: ; %bb.0: ; %entry 2766; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2767; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 2768; GFX1064-NEXT: v_mov_b32_e32 v2, v0 2769; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4 2770; GFX1064-NEXT: s_or_saveexec_b64 s[2:3], -1 2771; GFX1064-NEXT: v_mov_b32_e32 v1, -1 2772; GFX1064-NEXT: s_mov_b64 exec, s[2:3] 2773; GFX1064-NEXT: s_not_b64 exec, exec 2774; GFX1064-NEXT: v_mov_b32_e32 v2, -1 2775; GFX1064-NEXT: s_not_b64 exec, exec 2776; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 2777; GFX1064-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2778; GFX1064-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 2779; GFX1064-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 2780; GFX1064-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 2781; GFX1064-NEXT: v_mov_b32_e32 v3, v2 2782; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 2783; GFX1064-NEXT: v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 2784; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 2785; GFX1064-NEXT: v_mov_b32_e32 v3, s2 2786; GFX1064-NEXT: v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 2787; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 2788; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2789; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 2790; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 2791; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 2792; GFX1064-NEXT: s_mov_b32 s2, -1 2793; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 2794; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 2795; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 2796; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 2797; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 2798; GFX1064-NEXT: ; implicit-def: $vgpr0 2799; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 2800; GFX1064-NEXT: s_cbranch_execz BB14_2 2801; GFX1064-NEXT: ; %bb.1: 2802; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2803; GFX1064-NEXT: v_mov_b32_e32 v7, s3 2804; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2805; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 2806; GFX1064-NEXT: ds_and_rtn_b32 v0, v0, v7 2807; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2808; GFX1064-NEXT: buffer_gl0_inv 2809; GFX1064-NEXT: buffer_gl1_inv 2810; GFX1064-NEXT: BB14_2: 2811; GFX1064-NEXT: v_nop 2812; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 2813; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 2814; GFX1064-NEXT: v_mov_b32_e32 v0, v1 2815; GFX1064-NEXT: v_and_b32_e32 v0, s3, v0 2816; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 2817; GFX1064-NEXT: s_nop 1 2818; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 2819; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 2820; GFX1064-NEXT: s_endpgm 2821; 2822; GFX1032-LABEL: and_i32_varying: 2823; GFX1032: ; %bb.0: ; %entry 2824; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2825; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 2826; GFX1032-NEXT: ; implicit-def: $vcc_hi 2827; GFX1032-NEXT: v_mov_b32_e32 v2, v0 2828; GFX1032-NEXT: s_or_saveexec_b32 s2, -1 2829; GFX1032-NEXT: v_mov_b32_e32 v1, -1 2830; GFX1032-NEXT: s_mov_b32 exec_lo, s2 2831; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 2832; GFX1032-NEXT: v_mov_b32_e32 v2, -1 2833; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 2834; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 2835; GFX1032-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2836; GFX1032-NEXT: s_mov_b32 s2, -1 2837; GFX1032-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 2838; GFX1032-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 2839; GFX1032-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 2840; GFX1032-NEXT: v_mov_b32_e32 v3, v2 2841; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 2842; GFX1032-NEXT: v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 2843; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 2844; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 2845; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 2846; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 2847; GFX1032-NEXT: s_mov_b32 exec_lo, s4 2848; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 2849; GFX1032-NEXT: ; implicit-def: $vgpr0 2850; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 2851; GFX1032-NEXT: s_cbranch_execz BB14_2 2852; GFX1032-NEXT: ; %bb.1: 2853; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2854; GFX1032-NEXT: v_mov_b32_e32 v7, s3 2855; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2856; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 2857; GFX1032-NEXT: ds_and_rtn_b32 v0, v0, v7 2858; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2859; GFX1032-NEXT: buffer_gl0_inv 2860; GFX1032-NEXT: buffer_gl1_inv 2861; GFX1032-NEXT: BB14_2: 2862; GFX1032-NEXT: v_nop 2863; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 2864; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 2865; GFX1032-NEXT: v_mov_b32_e32 v0, v1 2866; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0 2867; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 2868; GFX1032-NEXT: s_nop 1 2869; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 2870; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 2871; GFX1032-NEXT: s_endpgm 2872entry: 2873 %lane = call i32 @llvm.amdgcn.workitem.id.x() 2874 %old = atomicrmw and i32 addrspace(3)* @local_var32, i32 %lane acq_rel 2875 store i32 %old, i32 addrspace(1)* %out 2876 ret void 2877} 2878 2879define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) { 2880; 2881; 2882; GFX7LESS-LABEL: or_i32_varying: 2883; GFX7LESS: ; %bb.0: ; %entry 2884; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 2885; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 2886; GFX7LESS-NEXT: s_mov_b32 m0, -1 2887; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2888; GFX7LESS-NEXT: ds_or_rtn_b32 v0, v1, v0 2889; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2890; GFX7LESS-NEXT: buffer_wbinvl1 2891; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 2892; GFX7LESS-NEXT: s_mov_b32 s2, -1 2893; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 2894; GFX7LESS-NEXT: s_endpgm 2895; 2896; GFX8-LABEL: or_i32_varying: 2897; GFX8: ; %bb.0: ; %entry 2898; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2899; GFX8-NEXT: s_mov_b64 s[2:3], exec 2900; GFX8-NEXT: v_mov_b32_e32 v2, v0 2901; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 2902; GFX8-NEXT: v_mov_b32_e32 v1, 0 2903; GFX8-NEXT: s_mov_b64 exec, s[4:5] 2904; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 2905; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 2906; GFX8-NEXT: s_not_b64 exec, exec 2907; GFX8-NEXT: v_mov_b32_e32 v2, 0 2908; GFX8-NEXT: s_not_b64 exec, exec 2909; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 2910; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 2911; GFX8-NEXT: s_nop 1 2912; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 2913; GFX8-NEXT: s_nop 1 2914; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 2915; GFX8-NEXT: s_nop 1 2916; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 2917; GFX8-NEXT: s_nop 1 2918; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 2919; GFX8-NEXT: s_nop 1 2920; GFX8-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 2921; GFX8-NEXT: v_readlane_b32 s2, v2, 63 2922; GFX8-NEXT: s_nop 0 2923; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 2924; GFX8-NEXT: s_mov_b64 exec, s[4:5] 2925; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2926; GFX8-NEXT: ; implicit-def: $vgpr0 2927; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 2928; GFX8-NEXT: s_cbranch_execz BB15_2 2929; GFX8-NEXT: ; %bb.1: 2930; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2931; GFX8-NEXT: v_mov_b32_e32 v3, s2 2932; GFX8-NEXT: s_mov_b32 m0, -1 2933; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2934; GFX8-NEXT: ds_or_rtn_b32 v0, v0, v3 2935; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2936; GFX8-NEXT: buffer_wbinvl1_vol 2937; GFX8-NEXT: BB15_2: 2938; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 2939; GFX8-NEXT: v_readfirstlane_b32 s2, v0 2940; GFX8-NEXT: v_mov_b32_e32 v0, v1 2941; GFX8-NEXT: v_or_b32_e32 v0, s2, v0 2942; GFX8-NEXT: s_mov_b32 s3, 0xf000 2943; GFX8-NEXT: s_mov_b32 s2, -1 2944; GFX8-NEXT: s_nop 0 2945; GFX8-NEXT: s_waitcnt lgkmcnt(0) 2946; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 2947; GFX8-NEXT: s_endpgm 2948; 2949; GFX9-LABEL: or_i32_varying: 2950; GFX9: ; %bb.0: ; %entry 2951; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 2952; GFX9-NEXT: s_mov_b64 s[2:3], exec 2953; GFX9-NEXT: v_mov_b32_e32 v2, v0 2954; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 2955; GFX9-NEXT: v_mov_b32_e32 v1, 0 2956; GFX9-NEXT: s_mov_b64 exec, s[4:5] 2957; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 2958; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 2959; GFX9-NEXT: s_not_b64 exec, exec 2960; GFX9-NEXT: v_mov_b32_e32 v2, 0 2961; GFX9-NEXT: s_not_b64 exec, exec 2962; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 2963; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 2964; GFX9-NEXT: s_nop 1 2965; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 2966; GFX9-NEXT: s_nop 1 2967; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 2968; GFX9-NEXT: s_nop 1 2969; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 2970; GFX9-NEXT: s_nop 1 2971; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 2972; GFX9-NEXT: s_nop 1 2973; GFX9-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 2974; GFX9-NEXT: v_readlane_b32 s2, v2, 63 2975; GFX9-NEXT: s_nop 0 2976; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 2977; GFX9-NEXT: s_mov_b64 exec, s[4:5] 2978; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 2979; GFX9-NEXT: ; implicit-def: $vgpr0 2980; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 2981; GFX9-NEXT: s_cbranch_execz BB15_2 2982; GFX9-NEXT: ; %bb.1: 2983; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 2984; GFX9-NEXT: v_mov_b32_e32 v3, s2 2985; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2986; GFX9-NEXT: ds_or_rtn_b32 v0, v0, v3 2987; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2988; GFX9-NEXT: buffer_wbinvl1_vol 2989; GFX9-NEXT: BB15_2: 2990; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 2991; GFX9-NEXT: v_readfirstlane_b32 s2, v0 2992; GFX9-NEXT: v_mov_b32_e32 v0, v1 2993; GFX9-NEXT: v_or_b32_e32 v0, s2, v0 2994; GFX9-NEXT: s_mov_b32 s3, 0xf000 2995; GFX9-NEXT: s_mov_b32 s2, -1 2996; GFX9-NEXT: s_nop 0 2997; GFX9-NEXT: s_waitcnt lgkmcnt(0) 2998; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 2999; GFX9-NEXT: s_endpgm 3000; 3001; GFX1064-LABEL: or_i32_varying: 3002; GFX1064: ; %bb.0: ; %entry 3003; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3004; GFX1064-NEXT: s_mov_b64 s[2:3], exec 3005; GFX1064-NEXT: v_mov_b32_e32 v2, v0 3006; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3007; GFX1064-NEXT: v_mov_b32_e32 v1, 0 3008; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3009; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 3010; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 3011; GFX1064-NEXT: s_not_b64 exec, exec 3012; GFX1064-NEXT: v_mov_b32_e32 v2, 0 3013; GFX1064-NEXT: s_not_b64 exec, exec 3014; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3015; GFX1064-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3016; GFX1064-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3017; GFX1064-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3018; GFX1064-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3019; GFX1064-NEXT: v_mov_b32_e32 v3, v2 3020; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3021; GFX1064-NEXT: v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3022; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 3023; GFX1064-NEXT: v_mov_b32_e32 v3, s2 3024; GFX1064-NEXT: v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 3025; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 3026; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3027; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 3028; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 3029; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 3030; GFX1064-NEXT: s_mov_b32 s2, -1 3031; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 3032; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 3033; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 3034; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3035; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3036; GFX1064-NEXT: ; implicit-def: $vgpr0 3037; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 3038; GFX1064-NEXT: s_cbranch_execz BB15_2 3039; GFX1064-NEXT: ; %bb.1: 3040; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3041; GFX1064-NEXT: v_mov_b32_e32 v7, s3 3042; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3043; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 3044; GFX1064-NEXT: ds_or_rtn_b32 v0, v0, v7 3045; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3046; GFX1064-NEXT: buffer_gl0_inv 3047; GFX1064-NEXT: buffer_gl1_inv 3048; GFX1064-NEXT: BB15_2: 3049; GFX1064-NEXT: v_nop 3050; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 3051; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 3052; GFX1064-NEXT: v_mov_b32_e32 v0, v1 3053; GFX1064-NEXT: v_or_b32_e32 v0, s3, v0 3054; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 3055; GFX1064-NEXT: s_nop 1 3056; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 3057; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 3058; GFX1064-NEXT: s_endpgm 3059; 3060; GFX1032-LABEL: or_i32_varying: 3061; GFX1032: ; %bb.0: ; %entry 3062; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3063; GFX1032-NEXT: s_mov_b32 s2, exec_lo 3064; GFX1032-NEXT: ; implicit-def: $vcc_hi 3065; GFX1032-NEXT: v_mov_b32_e32 v2, v0 3066; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 3067; GFX1032-NEXT: v_mov_b32_e32 v1, 0 3068; GFX1032-NEXT: s_mov_b32 exec_lo, s3 3069; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 3070; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3071; GFX1032-NEXT: v_mov_b32_e32 v2, 0 3072; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3073; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 3074; GFX1032-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3075; GFX1032-NEXT: s_mov_b32 s2, -1 3076; GFX1032-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3077; GFX1032-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3078; GFX1032-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3079; GFX1032-NEXT: v_mov_b32_e32 v3, v2 3080; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3081; GFX1032-NEXT: v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3082; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 3083; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3084; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 3085; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 3086; GFX1032-NEXT: s_mov_b32 exec_lo, s4 3087; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 3088; GFX1032-NEXT: ; implicit-def: $vgpr0 3089; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 3090; GFX1032-NEXT: s_cbranch_execz BB15_2 3091; GFX1032-NEXT: ; %bb.1: 3092; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3093; GFX1032-NEXT: v_mov_b32_e32 v7, s3 3094; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3095; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 3096; GFX1032-NEXT: ds_or_rtn_b32 v0, v0, v7 3097; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3098; GFX1032-NEXT: buffer_gl0_inv 3099; GFX1032-NEXT: buffer_gl1_inv 3100; GFX1032-NEXT: BB15_2: 3101; GFX1032-NEXT: v_nop 3102; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 3103; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 3104; GFX1032-NEXT: v_mov_b32_e32 v0, v1 3105; GFX1032-NEXT: v_or_b32_e32 v0, s3, v0 3106; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 3107; GFX1032-NEXT: s_nop 1 3108; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 3109; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 3110; GFX1032-NEXT: s_endpgm 3111entry: 3112 %lane = call i32 @llvm.amdgcn.workitem.id.x() 3113 %old = atomicrmw or i32 addrspace(3)* @local_var32, i32 %lane acq_rel 3114 store i32 %old, i32 addrspace(1)* %out 3115 ret void 3116} 3117 3118define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) { 3119; 3120; 3121; GFX7LESS-LABEL: xor_i32_varying: 3122; GFX7LESS: ; %bb.0: ; %entry 3123; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 3124; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 3125; GFX7LESS-NEXT: s_mov_b32 m0, -1 3126; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3127; GFX7LESS-NEXT: ds_xor_rtn_b32 v0, v1, v0 3128; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3129; GFX7LESS-NEXT: buffer_wbinvl1 3130; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 3131; GFX7LESS-NEXT: s_mov_b32 s2, -1 3132; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 3133; GFX7LESS-NEXT: s_endpgm 3134; 3135; GFX8-LABEL: xor_i32_varying: 3136; GFX8: ; %bb.0: ; %entry 3137; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3138; GFX8-NEXT: s_mov_b64 s[2:3], exec 3139; GFX8-NEXT: v_mov_b32_e32 v2, v0 3140; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 3141; GFX8-NEXT: v_mov_b32_e32 v1, 0 3142; GFX8-NEXT: s_mov_b64 exec, s[4:5] 3143; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 3144; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 3145; GFX8-NEXT: s_not_b64 exec, exec 3146; GFX8-NEXT: v_mov_b32_e32 v2, 0 3147; GFX8-NEXT: s_not_b64 exec, exec 3148; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 3149; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3150; GFX8-NEXT: s_nop 1 3151; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3152; GFX8-NEXT: s_nop 1 3153; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3154; GFX8-NEXT: s_nop 1 3155; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3156; GFX8-NEXT: s_nop 1 3157; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3158; GFX8-NEXT: s_nop 1 3159; GFX8-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3160; GFX8-NEXT: v_readlane_b32 s2, v2, 63 3161; GFX8-NEXT: s_nop 0 3162; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3163; GFX8-NEXT: s_mov_b64 exec, s[4:5] 3164; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3165; GFX8-NEXT: ; implicit-def: $vgpr0 3166; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 3167; GFX8-NEXT: s_cbranch_execz BB16_2 3168; GFX8-NEXT: ; %bb.1: 3169; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3170; GFX8-NEXT: v_mov_b32_e32 v3, s2 3171; GFX8-NEXT: s_mov_b32 m0, -1 3172; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3173; GFX8-NEXT: ds_xor_rtn_b32 v0, v0, v3 3174; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3175; GFX8-NEXT: buffer_wbinvl1_vol 3176; GFX8-NEXT: BB16_2: 3177; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 3178; GFX8-NEXT: v_readfirstlane_b32 s2, v0 3179; GFX8-NEXT: v_mov_b32_e32 v0, v1 3180; GFX8-NEXT: v_xor_b32_e32 v0, s2, v0 3181; GFX8-NEXT: s_mov_b32 s3, 0xf000 3182; GFX8-NEXT: s_mov_b32 s2, -1 3183; GFX8-NEXT: s_nop 0 3184; GFX8-NEXT: s_waitcnt lgkmcnt(0) 3185; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 3186; GFX8-NEXT: s_endpgm 3187; 3188; GFX9-LABEL: xor_i32_varying: 3189; GFX9: ; %bb.0: ; %entry 3190; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3191; GFX9-NEXT: s_mov_b64 s[2:3], exec 3192; GFX9-NEXT: v_mov_b32_e32 v2, v0 3193; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 3194; GFX9-NEXT: v_mov_b32_e32 v1, 0 3195; GFX9-NEXT: s_mov_b64 exec, s[4:5] 3196; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 3197; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 3198; GFX9-NEXT: s_not_b64 exec, exec 3199; GFX9-NEXT: v_mov_b32_e32 v2, 0 3200; GFX9-NEXT: s_not_b64 exec, exec 3201; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 3202; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3203; GFX9-NEXT: s_nop 1 3204; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3205; GFX9-NEXT: s_nop 1 3206; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3207; GFX9-NEXT: s_nop 1 3208; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3209; GFX9-NEXT: s_nop 1 3210; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3211; GFX9-NEXT: s_nop 1 3212; GFX9-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3213; GFX9-NEXT: v_readlane_b32 s2, v2, 63 3214; GFX9-NEXT: s_nop 0 3215; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3216; GFX9-NEXT: s_mov_b64 exec, s[4:5] 3217; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3218; GFX9-NEXT: ; implicit-def: $vgpr0 3219; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 3220; GFX9-NEXT: s_cbranch_execz BB16_2 3221; GFX9-NEXT: ; %bb.1: 3222; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3223; GFX9-NEXT: v_mov_b32_e32 v3, s2 3224; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3225; GFX9-NEXT: ds_xor_rtn_b32 v0, v0, v3 3226; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3227; GFX9-NEXT: buffer_wbinvl1_vol 3228; GFX9-NEXT: BB16_2: 3229; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 3230; GFX9-NEXT: v_readfirstlane_b32 s2, v0 3231; GFX9-NEXT: v_mov_b32_e32 v0, v1 3232; GFX9-NEXT: v_xor_b32_e32 v0, s2, v0 3233; GFX9-NEXT: s_mov_b32 s3, 0xf000 3234; GFX9-NEXT: s_mov_b32 s2, -1 3235; GFX9-NEXT: s_nop 0 3236; GFX9-NEXT: s_waitcnt lgkmcnt(0) 3237; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 3238; GFX9-NEXT: s_endpgm 3239; 3240; GFX1064-LABEL: xor_i32_varying: 3241; GFX1064: ; %bb.0: ; %entry 3242; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3243; GFX1064-NEXT: s_mov_b64 s[2:3], exec 3244; GFX1064-NEXT: v_mov_b32_e32 v2, v0 3245; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3246; GFX1064-NEXT: v_mov_b32_e32 v1, 0 3247; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3248; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 3249; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 3250; GFX1064-NEXT: s_not_b64 exec, exec 3251; GFX1064-NEXT: v_mov_b32_e32 v2, 0 3252; GFX1064-NEXT: s_not_b64 exec, exec 3253; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3254; GFX1064-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3255; GFX1064-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3256; GFX1064-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3257; GFX1064-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3258; GFX1064-NEXT: v_mov_b32_e32 v3, v2 3259; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3260; GFX1064-NEXT: v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3261; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 3262; GFX1064-NEXT: v_mov_b32_e32 v3, s2 3263; GFX1064-NEXT: v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 3264; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 3265; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3266; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 3267; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 3268; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 3269; GFX1064-NEXT: s_mov_b32 s2, -1 3270; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 3271; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 3272; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 3273; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3274; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3275; GFX1064-NEXT: ; implicit-def: $vgpr0 3276; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 3277; GFX1064-NEXT: s_cbranch_execz BB16_2 3278; GFX1064-NEXT: ; %bb.1: 3279; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3280; GFX1064-NEXT: v_mov_b32_e32 v7, s3 3281; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3282; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 3283; GFX1064-NEXT: ds_xor_rtn_b32 v0, v0, v7 3284; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3285; GFX1064-NEXT: buffer_gl0_inv 3286; GFX1064-NEXT: buffer_gl1_inv 3287; GFX1064-NEXT: BB16_2: 3288; GFX1064-NEXT: v_nop 3289; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 3290; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 3291; GFX1064-NEXT: v_mov_b32_e32 v0, v1 3292; GFX1064-NEXT: v_xor_b32_e32 v0, s3, v0 3293; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 3294; GFX1064-NEXT: s_nop 1 3295; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 3296; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 3297; GFX1064-NEXT: s_endpgm 3298; 3299; GFX1032-LABEL: xor_i32_varying: 3300; GFX1032: ; %bb.0: ; %entry 3301; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3302; GFX1032-NEXT: s_mov_b32 s2, exec_lo 3303; GFX1032-NEXT: ; implicit-def: $vcc_hi 3304; GFX1032-NEXT: v_mov_b32_e32 v2, v0 3305; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 3306; GFX1032-NEXT: v_mov_b32_e32 v1, 0 3307; GFX1032-NEXT: s_mov_b32 exec_lo, s3 3308; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 3309; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3310; GFX1032-NEXT: v_mov_b32_e32 v2, 0 3311; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3312; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 3313; GFX1032-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 3314; GFX1032-NEXT: s_mov_b32 s2, -1 3315; GFX1032-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 3316; GFX1032-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 3317; GFX1032-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 3318; GFX1032-NEXT: v_mov_b32_e32 v3, v2 3319; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3320; GFX1032-NEXT: v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3321; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 3322; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3323; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 3324; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 3325; GFX1032-NEXT: s_mov_b32 exec_lo, s4 3326; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 3327; GFX1032-NEXT: ; implicit-def: $vgpr0 3328; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 3329; GFX1032-NEXT: s_cbranch_execz BB16_2 3330; GFX1032-NEXT: ; %bb.1: 3331; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3332; GFX1032-NEXT: v_mov_b32_e32 v7, s3 3333; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3334; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 3335; GFX1032-NEXT: ds_xor_rtn_b32 v0, v0, v7 3336; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3337; GFX1032-NEXT: buffer_gl0_inv 3338; GFX1032-NEXT: buffer_gl1_inv 3339; GFX1032-NEXT: BB16_2: 3340; GFX1032-NEXT: v_nop 3341; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 3342; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 3343; GFX1032-NEXT: v_mov_b32_e32 v0, v1 3344; GFX1032-NEXT: v_xor_b32_e32 v0, s3, v0 3345; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 3346; GFX1032-NEXT: s_nop 1 3347; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 3348; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 3349; GFX1032-NEXT: s_endpgm 3350entry: 3351 %lane = call i32 @llvm.amdgcn.workitem.id.x() 3352 %old = atomicrmw xor i32 addrspace(3)* @local_var32, i32 %lane acq_rel 3353 store i32 %old, i32 addrspace(1)* %out 3354 ret void 3355} 3356 3357define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) { 3358; 3359; 3360; GFX7LESS-LABEL: max_i32_varying: 3361; GFX7LESS: ; %bb.0: ; %entry 3362; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 3363; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 3364; GFX7LESS-NEXT: s_mov_b32 m0, -1 3365; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3366; GFX7LESS-NEXT: ds_max_rtn_i32 v0, v1, v0 3367; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3368; GFX7LESS-NEXT: buffer_wbinvl1 3369; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 3370; GFX7LESS-NEXT: s_mov_b32 s2, -1 3371; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 3372; GFX7LESS-NEXT: s_endpgm 3373; 3374; GFX8-LABEL: max_i32_varying: 3375; GFX8: ; %bb.0: ; %entry 3376; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3377; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 3378; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 3379; GFX8-NEXT: v_mov_b32_e32 v2, v0 3380; GFX8-NEXT: s_or_saveexec_b64 s[2:3], -1 3381; GFX8-NEXT: v_bfrev_b32_e32 v1, 1 3382; GFX8-NEXT: s_mov_b64 exec, s[2:3] 3383; GFX8-NEXT: s_not_b64 exec, exec 3384; GFX8-NEXT: v_mov_b32_e32 v2, v1 3385; GFX8-NEXT: s_not_b64 exec, exec 3386; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 3387; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3388; GFX8-NEXT: s_nop 1 3389; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3390; GFX8-NEXT: s_nop 1 3391; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3392; GFX8-NEXT: s_nop 1 3393; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3394; GFX8-NEXT: s_nop 1 3395; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3396; GFX8-NEXT: s_nop 1 3397; GFX8-NEXT: v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3398; GFX8-NEXT: v_readlane_b32 s2, v2, 63 3399; GFX8-NEXT: s_nop 0 3400; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3401; GFX8-NEXT: s_mov_b64 exec, s[4:5] 3402; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 3403; GFX8-NEXT: ; implicit-def: $vgpr0 3404; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 3405; GFX8-NEXT: s_cbranch_execz BB17_2 3406; GFX8-NEXT: ; %bb.1: 3407; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3408; GFX8-NEXT: v_mov_b32_e32 v3, s2 3409; GFX8-NEXT: s_mov_b32 m0, -1 3410; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3411; GFX8-NEXT: ds_max_rtn_i32 v0, v0, v3 3412; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3413; GFX8-NEXT: buffer_wbinvl1_vol 3414; GFX8-NEXT: BB17_2: 3415; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 3416; GFX8-NEXT: v_readfirstlane_b32 s2, v0 3417; GFX8-NEXT: v_mov_b32_e32 v0, v1 3418; GFX8-NEXT: v_max_i32_e32 v0, s2, v0 3419; GFX8-NEXT: s_mov_b32 s3, 0xf000 3420; GFX8-NEXT: s_mov_b32 s2, -1 3421; GFX8-NEXT: s_nop 0 3422; GFX8-NEXT: s_waitcnt lgkmcnt(0) 3423; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 3424; GFX8-NEXT: s_endpgm 3425; 3426; GFX9-LABEL: max_i32_varying: 3427; GFX9: ; %bb.0: ; %entry 3428; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3429; GFX9-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 3430; GFX9-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 3431; GFX9-NEXT: v_mov_b32_e32 v2, v0 3432; GFX9-NEXT: s_or_saveexec_b64 s[2:3], -1 3433; GFX9-NEXT: v_bfrev_b32_e32 v1, 1 3434; GFX9-NEXT: s_mov_b64 exec, s[2:3] 3435; GFX9-NEXT: s_not_b64 exec, exec 3436; GFX9-NEXT: v_mov_b32_e32 v2, v1 3437; GFX9-NEXT: s_not_b64 exec, exec 3438; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 3439; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3440; GFX9-NEXT: s_nop 1 3441; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3442; GFX9-NEXT: s_nop 1 3443; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3444; GFX9-NEXT: s_nop 1 3445; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3446; GFX9-NEXT: s_nop 1 3447; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3448; GFX9-NEXT: s_nop 1 3449; GFX9-NEXT: v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3450; GFX9-NEXT: v_readlane_b32 s2, v2, 63 3451; GFX9-NEXT: s_nop 0 3452; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3453; GFX9-NEXT: s_mov_b64 exec, s[4:5] 3454; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 3455; GFX9-NEXT: ; implicit-def: $vgpr0 3456; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 3457; GFX9-NEXT: s_cbranch_execz BB17_2 3458; GFX9-NEXT: ; %bb.1: 3459; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3460; GFX9-NEXT: v_mov_b32_e32 v3, s2 3461; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3462; GFX9-NEXT: ds_max_rtn_i32 v0, v0, v3 3463; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3464; GFX9-NEXT: buffer_wbinvl1_vol 3465; GFX9-NEXT: BB17_2: 3466; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 3467; GFX9-NEXT: v_readfirstlane_b32 s2, v0 3468; GFX9-NEXT: v_mov_b32_e32 v0, v1 3469; GFX9-NEXT: v_max_i32_e32 v0, s2, v0 3470; GFX9-NEXT: s_mov_b32 s3, 0xf000 3471; GFX9-NEXT: s_mov_b32 s2, -1 3472; GFX9-NEXT: s_nop 0 3473; GFX9-NEXT: s_waitcnt lgkmcnt(0) 3474; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 3475; GFX9-NEXT: s_endpgm 3476; 3477; GFX1064-LABEL: max_i32_varying: 3478; GFX1064: ; %bb.0: ; %entry 3479; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3480; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 3481; GFX1064-NEXT: v_mov_b32_e32 v2, v0 3482; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4 3483; GFX1064-NEXT: s_or_saveexec_b64 s[2:3], -1 3484; GFX1064-NEXT: v_bfrev_b32_e32 v1, 1 3485; GFX1064-NEXT: s_mov_b64 exec, s[2:3] 3486; GFX1064-NEXT: s_not_b64 exec, exec 3487; GFX1064-NEXT: v_mov_b32_e32 v2, v1 3488; GFX1064-NEXT: s_not_b64 exec, exec 3489; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3490; GFX1064-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3491; GFX1064-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3492; GFX1064-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3493; GFX1064-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3494; GFX1064-NEXT: v_mov_b32_e32 v3, v2 3495; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3496; GFX1064-NEXT: v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3497; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 3498; GFX1064-NEXT: v_mov_b32_e32 v3, s2 3499; GFX1064-NEXT: v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 3500; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 3501; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3502; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 3503; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 3504; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 3505; GFX1064-NEXT: s_mov_b32 s2, -1 3506; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 3507; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 3508; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 3509; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3510; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 3511; GFX1064-NEXT: ; implicit-def: $vgpr0 3512; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 3513; GFX1064-NEXT: s_cbranch_execz BB17_2 3514; GFX1064-NEXT: ; %bb.1: 3515; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3516; GFX1064-NEXT: v_mov_b32_e32 v7, s3 3517; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3518; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 3519; GFX1064-NEXT: ds_max_rtn_i32 v0, v0, v7 3520; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3521; GFX1064-NEXT: buffer_gl0_inv 3522; GFX1064-NEXT: buffer_gl1_inv 3523; GFX1064-NEXT: BB17_2: 3524; GFX1064-NEXT: v_nop 3525; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 3526; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 3527; GFX1064-NEXT: v_mov_b32_e32 v0, v1 3528; GFX1064-NEXT: v_max_i32_e32 v0, s3, v0 3529; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 3530; GFX1064-NEXT: s_nop 1 3531; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 3532; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 3533; GFX1064-NEXT: s_endpgm 3534; 3535; GFX1032-LABEL: max_i32_varying: 3536; GFX1032: ; %bb.0: ; %entry 3537; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3538; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 3539; GFX1032-NEXT: ; implicit-def: $vcc_hi 3540; GFX1032-NEXT: v_mov_b32_e32 v2, v0 3541; GFX1032-NEXT: s_or_saveexec_b32 s2, -1 3542; GFX1032-NEXT: v_bfrev_b32_e32 v1, 1 3543; GFX1032-NEXT: s_mov_b32 exec_lo, s2 3544; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3545; GFX1032-NEXT: v_mov_b32_e32 v2, v1 3546; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3547; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 3548; GFX1032-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3549; GFX1032-NEXT: s_mov_b32 s2, -1 3550; GFX1032-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3551; GFX1032-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3552; GFX1032-NEXT: v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3553; GFX1032-NEXT: v_mov_b32_e32 v3, v2 3554; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3555; GFX1032-NEXT: v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3556; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 3557; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3558; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 3559; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 3560; GFX1032-NEXT: s_mov_b32 exec_lo, s4 3561; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 3562; GFX1032-NEXT: ; implicit-def: $vgpr0 3563; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 3564; GFX1032-NEXT: s_cbranch_execz BB17_2 3565; GFX1032-NEXT: ; %bb.1: 3566; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3567; GFX1032-NEXT: v_mov_b32_e32 v7, s3 3568; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3569; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 3570; GFX1032-NEXT: ds_max_rtn_i32 v0, v0, v7 3571; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3572; GFX1032-NEXT: buffer_gl0_inv 3573; GFX1032-NEXT: buffer_gl1_inv 3574; GFX1032-NEXT: BB17_2: 3575; GFX1032-NEXT: v_nop 3576; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 3577; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 3578; GFX1032-NEXT: v_mov_b32_e32 v0, v1 3579; GFX1032-NEXT: v_max_i32_e32 v0, s3, v0 3580; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 3581; GFX1032-NEXT: s_nop 1 3582; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 3583; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 3584; GFX1032-NEXT: s_endpgm 3585entry: 3586 %lane = call i32 @llvm.amdgcn.workitem.id.x() 3587 %old = atomicrmw max i32 addrspace(3)* @local_var32, i32 %lane acq_rel 3588 store i32 %old, i32 addrspace(1)* %out 3589 ret void 3590} 3591 3592define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) { 3593; 3594; 3595; GFX7LESS-LABEL: max_i64_constant: 3596; GFX7LESS: ; %bb.0: ; %entry 3597; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 3598; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 3599; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 3600; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3601; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1 3602; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 3603; GFX7LESS-NEXT: s_cbranch_execz BB18_2 3604; GFX7LESS-NEXT: ; %bb.1: 3605; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 3606; GFX7LESS-NEXT: v_mov_b32_e32 v0, 5 3607; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 3608; GFX7LESS-NEXT: s_mov_b32 m0, -1 3609; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3610; GFX7LESS-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1] 3611; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3612; GFX7LESS-NEXT: buffer_wbinvl1 3613; GFX7LESS-NEXT: BB18_2: 3614; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 3615; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0 3616; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v1 3617; GFX7LESS-NEXT: v_bfrev_b32_e32 v1, 1 3618; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 3619; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 3620; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc 3621; GFX7LESS-NEXT: v_mov_b32_e32 v2, s5 3622; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4 3623; GFX7LESS-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1] 3624; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 3625; GFX7LESS-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc 3626; GFX7LESS-NEXT: s_mov_b32 s2, -1 3627; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 3628; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 3629; GFX7LESS-NEXT: s_endpgm 3630; 3631; GFX8-LABEL: max_i64_constant: 3632; GFX8: ; %bb.0: ; %entry 3633; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3634; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 3635; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 3636; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3637; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1 3638; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 3639; GFX8-NEXT: s_cbranch_execz BB18_2 3640; GFX8-NEXT: ; %bb.1: 3641; GFX8-NEXT: v_mov_b32_e32 v0, 5 3642; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 3643; GFX8-NEXT: v_mov_b32_e32 v1, 0 3644; GFX8-NEXT: s_mov_b32 m0, -1 3645; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3646; GFX8-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1] 3647; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3648; GFX8-NEXT: buffer_wbinvl1_vol 3649; GFX8-NEXT: BB18_2: 3650; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 3651; GFX8-NEXT: v_readfirstlane_b32 s2, v0 3652; GFX8-NEXT: v_bfrev_b32_e32 v0, 1 3653; GFX8-NEXT: v_readfirstlane_b32 s3, v1 3654; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v0, vcc 3655; GFX8-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 3656; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1] 3657; GFX8-NEXT: v_mov_b32_e32 v2, s3 3658; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 3659; GFX8-NEXT: v_mov_b32_e32 v2, s2 3660; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 3661; GFX8-NEXT: s_mov_b32 s3, 0xf000 3662; GFX8-NEXT: s_mov_b32 s2, -1 3663; GFX8-NEXT: s_waitcnt lgkmcnt(0) 3664; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 3665; GFX8-NEXT: s_endpgm 3666; 3667; GFX9-LABEL: max_i64_constant: 3668; GFX9: ; %bb.0: ; %entry 3669; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3670; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 3671; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 3672; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3673; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 3674; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 3675; GFX9-NEXT: s_cbranch_execz BB18_2 3676; GFX9-NEXT: ; %bb.1: 3677; GFX9-NEXT: v_mov_b32_e32 v0, 5 3678; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 3679; GFX9-NEXT: v_mov_b32_e32 v1, 0 3680; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3681; GFX9-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1] 3682; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3683; GFX9-NEXT: buffer_wbinvl1_vol 3684; GFX9-NEXT: BB18_2: 3685; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 3686; GFX9-NEXT: v_readfirstlane_b32 s2, v0 3687; GFX9-NEXT: v_bfrev_b32_e32 v0, 1 3688; GFX9-NEXT: v_readfirstlane_b32 s3, v1 3689; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v0, vcc 3690; GFX9-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 3691; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1] 3692; GFX9-NEXT: v_mov_b32_e32 v2, s3 3693; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 3694; GFX9-NEXT: v_mov_b32_e32 v2, s2 3695; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 3696; GFX9-NEXT: s_mov_b32 s3, 0xf000 3697; GFX9-NEXT: s_mov_b32 s2, -1 3698; GFX9-NEXT: s_waitcnt lgkmcnt(0) 3699; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 3700; GFX9-NEXT: s_endpgm 3701; 3702; GFX1064-LABEL: max_i64_constant: 3703; GFX1064: ; %bb.0: ; %entry 3704; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3705; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 3706; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0 3707; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 3708; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1 3709; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 3710; GFX1064-NEXT: s_cbranch_execz BB18_2 3711; GFX1064-NEXT: ; %bb.1: 3712; GFX1064-NEXT: v_mov_b32_e32 v0, 5 3713; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 3714; GFX1064-NEXT: v_mov_b32_e32 v1, 0 3715; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3716; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 3717; GFX1064-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1] 3718; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3719; GFX1064-NEXT: buffer_gl0_inv 3720; GFX1064-NEXT: buffer_gl1_inv 3721; GFX1064-NEXT: BB18_2: 3722; GFX1064-NEXT: v_nop 3723; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 3724; GFX1064-NEXT: v_readfirstlane_b32 s4, v0 3725; GFX1064-NEXT: v_readfirstlane_b32 s5, v1 3726; GFX1064-NEXT: v_cndmask_b32_e64 v1, 0, 0x80000000, vcc 3727; GFX1064-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 3728; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 3729; GFX1064-NEXT: s_mov_b32 s2, -1 3730; GFX1064-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1] 3731; GFX1064-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc 3732; GFX1064-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc 3733; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 3734; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 3735; GFX1064-NEXT: s_endpgm 3736; 3737; GFX1032-LABEL: max_i64_constant: 3738; GFX1032: ; %bb.0: ; %entry 3739; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3740; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 3741; GFX1032-NEXT: ; implicit-def: $vcc_hi 3742; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 3743; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1 3744; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 3745; GFX1032-NEXT: s_cbranch_execz BB18_2 3746; GFX1032-NEXT: ; %bb.1: 3747; GFX1032-NEXT: v_mov_b32_e32 v0, 5 3748; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 3749; GFX1032-NEXT: v_mov_b32_e32 v1, 0 3750; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3751; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 3752; GFX1032-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1] 3753; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3754; GFX1032-NEXT: buffer_gl0_inv 3755; GFX1032-NEXT: buffer_gl1_inv 3756; GFX1032-NEXT: BB18_2: 3757; GFX1032-NEXT: v_nop 3758; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 3759; GFX1032-NEXT: v_readfirstlane_b32 s4, v0 3760; GFX1032-NEXT: v_readfirstlane_b32 s5, v1 3761; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, 0x80000000, vcc_lo 3762; GFX1032-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc_lo 3763; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 3764; GFX1032-NEXT: s_mov_b32 s2, -1 3765; GFX1032-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[4:5], v[0:1] 3766; GFX1032-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc_lo 3767; GFX1032-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc_lo 3768; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 3769; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 3770; GFX1032-NEXT: s_endpgm 3771entry: 3772 %old = atomicrmw max i64 addrspace(3)* @local_var64, i64 5 acq_rel 3773 store i64 %old, i64 addrspace(1)* %out 3774 ret void 3775} 3776 3777define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) { 3778; 3779; 3780; GFX7LESS-LABEL: min_i32_varying: 3781; GFX7LESS: ; %bb.0: ; %entry 3782; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 3783; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 3784; GFX7LESS-NEXT: s_mov_b32 m0, -1 3785; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3786; GFX7LESS-NEXT: ds_min_rtn_i32 v0, v1, v0 3787; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3788; GFX7LESS-NEXT: buffer_wbinvl1 3789; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 3790; GFX7LESS-NEXT: s_mov_b32 s2, -1 3791; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 3792; GFX7LESS-NEXT: s_endpgm 3793; 3794; GFX8-LABEL: min_i32_varying: 3795; GFX8: ; %bb.0: ; %entry 3796; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3797; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 3798; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 3799; GFX8-NEXT: v_mov_b32_e32 v2, v0 3800; GFX8-NEXT: s_or_saveexec_b64 s[2:3], -1 3801; GFX8-NEXT: v_bfrev_b32_e32 v1, -2 3802; GFX8-NEXT: s_mov_b64 exec, s[2:3] 3803; GFX8-NEXT: s_not_b64 exec, exec 3804; GFX8-NEXT: v_mov_b32_e32 v2, v1 3805; GFX8-NEXT: s_not_b64 exec, exec 3806; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 3807; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3808; GFX8-NEXT: s_nop 1 3809; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3810; GFX8-NEXT: s_nop 1 3811; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3812; GFX8-NEXT: s_nop 1 3813; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3814; GFX8-NEXT: s_nop 1 3815; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3816; GFX8-NEXT: s_nop 1 3817; GFX8-NEXT: v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3818; GFX8-NEXT: v_readlane_b32 s2, v2, 63 3819; GFX8-NEXT: s_nop 0 3820; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3821; GFX8-NEXT: s_mov_b64 exec, s[4:5] 3822; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 3823; GFX8-NEXT: ; implicit-def: $vgpr0 3824; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 3825; GFX8-NEXT: s_cbranch_execz BB19_2 3826; GFX8-NEXT: ; %bb.1: 3827; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3828; GFX8-NEXT: v_mov_b32_e32 v3, s2 3829; GFX8-NEXT: s_mov_b32 m0, -1 3830; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3831; GFX8-NEXT: ds_min_rtn_i32 v0, v0, v3 3832; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3833; GFX8-NEXT: buffer_wbinvl1_vol 3834; GFX8-NEXT: BB19_2: 3835; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 3836; GFX8-NEXT: v_readfirstlane_b32 s2, v0 3837; GFX8-NEXT: v_mov_b32_e32 v0, v1 3838; GFX8-NEXT: v_min_i32_e32 v0, s2, v0 3839; GFX8-NEXT: s_mov_b32 s3, 0xf000 3840; GFX8-NEXT: s_mov_b32 s2, -1 3841; GFX8-NEXT: s_nop 0 3842; GFX8-NEXT: s_waitcnt lgkmcnt(0) 3843; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 3844; GFX8-NEXT: s_endpgm 3845; 3846; GFX9-LABEL: min_i32_varying: 3847; GFX9: ; %bb.0: ; %entry 3848; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3849; GFX9-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 3850; GFX9-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 3851; GFX9-NEXT: v_mov_b32_e32 v2, v0 3852; GFX9-NEXT: s_or_saveexec_b64 s[2:3], -1 3853; GFX9-NEXT: v_bfrev_b32_e32 v1, -2 3854; GFX9-NEXT: s_mov_b64 exec, s[2:3] 3855; GFX9-NEXT: s_not_b64 exec, exec 3856; GFX9-NEXT: v_mov_b32_e32 v2, v1 3857; GFX9-NEXT: s_not_b64 exec, exec 3858; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 3859; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3860; GFX9-NEXT: s_nop 1 3861; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3862; GFX9-NEXT: s_nop 1 3863; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3864; GFX9-NEXT: s_nop 1 3865; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3866; GFX9-NEXT: s_nop 1 3867; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 3868; GFX9-NEXT: s_nop 1 3869; GFX9-NEXT: v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 3870; GFX9-NEXT: v_readlane_b32 s2, v2, 63 3871; GFX9-NEXT: s_nop 0 3872; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 3873; GFX9-NEXT: s_mov_b64 exec, s[4:5] 3874; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 3875; GFX9-NEXT: ; implicit-def: $vgpr0 3876; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 3877; GFX9-NEXT: s_cbranch_execz BB19_2 3878; GFX9-NEXT: ; %bb.1: 3879; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3880; GFX9-NEXT: v_mov_b32_e32 v3, s2 3881; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3882; GFX9-NEXT: ds_min_rtn_i32 v0, v0, v3 3883; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3884; GFX9-NEXT: buffer_wbinvl1_vol 3885; GFX9-NEXT: BB19_2: 3886; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 3887; GFX9-NEXT: v_readfirstlane_b32 s2, v0 3888; GFX9-NEXT: v_mov_b32_e32 v0, v1 3889; GFX9-NEXT: v_min_i32_e32 v0, s2, v0 3890; GFX9-NEXT: s_mov_b32 s3, 0xf000 3891; GFX9-NEXT: s_mov_b32 s2, -1 3892; GFX9-NEXT: s_nop 0 3893; GFX9-NEXT: s_waitcnt lgkmcnt(0) 3894; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 3895; GFX9-NEXT: s_endpgm 3896; 3897; GFX1064-LABEL: min_i32_varying: 3898; GFX1064: ; %bb.0: ; %entry 3899; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3900; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 3901; GFX1064-NEXT: v_mov_b32_e32 v2, v0 3902; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4 3903; GFX1064-NEXT: s_or_saveexec_b64 s[2:3], -1 3904; GFX1064-NEXT: v_bfrev_b32_e32 v1, -2 3905; GFX1064-NEXT: s_mov_b64 exec, s[2:3] 3906; GFX1064-NEXT: s_not_b64 exec, exec 3907; GFX1064-NEXT: v_mov_b32_e32 v2, v1 3908; GFX1064-NEXT: s_not_b64 exec, exec 3909; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 3910; GFX1064-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3911; GFX1064-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3912; GFX1064-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3913; GFX1064-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3914; GFX1064-NEXT: v_mov_b32_e32 v3, v2 3915; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3916; GFX1064-NEXT: v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3917; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 3918; GFX1064-NEXT: v_mov_b32_e32 v3, s2 3919; GFX1064-NEXT: v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 3920; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 3921; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3922; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 3923; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 3924; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 3925; GFX1064-NEXT: s_mov_b32 s2, -1 3926; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 3927; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 3928; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 3929; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 3930; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 3931; GFX1064-NEXT: ; implicit-def: $vgpr0 3932; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 3933; GFX1064-NEXT: s_cbranch_execz BB19_2 3934; GFX1064-NEXT: ; %bb.1: 3935; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3936; GFX1064-NEXT: v_mov_b32_e32 v7, s3 3937; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3938; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 3939; GFX1064-NEXT: ds_min_rtn_i32 v0, v0, v7 3940; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3941; GFX1064-NEXT: buffer_gl0_inv 3942; GFX1064-NEXT: buffer_gl1_inv 3943; GFX1064-NEXT: BB19_2: 3944; GFX1064-NEXT: v_nop 3945; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 3946; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 3947; GFX1064-NEXT: v_mov_b32_e32 v0, v1 3948; GFX1064-NEXT: v_min_i32_e32 v0, s3, v0 3949; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 3950; GFX1064-NEXT: s_nop 1 3951; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 3952; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 3953; GFX1064-NEXT: s_endpgm 3954; 3955; GFX1032-LABEL: min_i32_varying: 3956; GFX1032: ; %bb.0: ; %entry 3957; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 3958; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 3959; GFX1032-NEXT: ; implicit-def: $vcc_hi 3960; GFX1032-NEXT: v_mov_b32_e32 v2, v0 3961; GFX1032-NEXT: s_or_saveexec_b32 s2, -1 3962; GFX1032-NEXT: v_bfrev_b32_e32 v1, -2 3963; GFX1032-NEXT: s_mov_b32 exec_lo, s2 3964; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3965; GFX1032-NEXT: v_mov_b32_e32 v2, v1 3966; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 3967; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 3968; GFX1032-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3969; GFX1032-NEXT: s_mov_b32 s2, -1 3970; GFX1032-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 3971; GFX1032-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 3972; GFX1032-NEXT: v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 3973; GFX1032-NEXT: v_mov_b32_e32 v3, v2 3974; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 3975; GFX1032-NEXT: v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 3976; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 3977; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 3978; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 3979; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 3980; GFX1032-NEXT: s_mov_b32 exec_lo, s4 3981; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 3982; GFX1032-NEXT: ; implicit-def: $vgpr0 3983; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 3984; GFX1032-NEXT: s_cbranch_execz BB19_2 3985; GFX1032-NEXT: ; %bb.1: 3986; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 3987; GFX1032-NEXT: v_mov_b32_e32 v7, s3 3988; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3989; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 3990; GFX1032-NEXT: ds_min_rtn_i32 v0, v0, v7 3991; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3992; GFX1032-NEXT: buffer_gl0_inv 3993; GFX1032-NEXT: buffer_gl1_inv 3994; GFX1032-NEXT: BB19_2: 3995; GFX1032-NEXT: v_nop 3996; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 3997; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 3998; GFX1032-NEXT: v_mov_b32_e32 v0, v1 3999; GFX1032-NEXT: v_min_i32_e32 v0, s3, v0 4000; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 4001; GFX1032-NEXT: s_nop 1 4002; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 4003; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 4004; GFX1032-NEXT: s_endpgm 4005entry: 4006 %lane = call i32 @llvm.amdgcn.workitem.id.x() 4007 %old = atomicrmw min i32 addrspace(3)* @local_var32, i32 %lane acq_rel 4008 store i32 %old, i32 addrspace(1)* %out 4009 ret void 4010} 4011 4012define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) { 4013; 4014; 4015; GFX7LESS-LABEL: min_i64_constant: 4016; GFX7LESS: ; %bb.0: ; %entry 4017; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 4018; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4019; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 4020; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4021; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1 4022; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 4023; GFX7LESS-NEXT: s_cbranch_execz BB20_2 4024; GFX7LESS-NEXT: ; %bb.1: 4025; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4026; GFX7LESS-NEXT: v_mov_b32_e32 v0, 5 4027; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 4028; GFX7LESS-NEXT: s_mov_b32 m0, -1 4029; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4030; GFX7LESS-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1] 4031; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4032; GFX7LESS-NEXT: buffer_wbinvl1 4033; GFX7LESS-NEXT: BB20_2: 4034; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 4035; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0 4036; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v1 4037; GFX7LESS-NEXT: v_bfrev_b32_e32 v1, -2 4038; GFX7LESS-NEXT: s_mov_b32 s2, -1 4039; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4040; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc 4041; GFX7LESS-NEXT: v_mov_b32_e32 v2, s5 4042; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4 4043; GFX7LESS-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1] 4044; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4045; GFX7LESS-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc 4046; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 4047; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 4048; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4049; GFX7LESS-NEXT: s_endpgm 4050; 4051; GFX8-LABEL: min_i64_constant: 4052; GFX8: ; %bb.0: ; %entry 4053; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4054; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4055; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4056; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4057; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1 4058; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 4059; GFX8-NEXT: s_cbranch_execz BB20_2 4060; GFX8-NEXT: ; %bb.1: 4061; GFX8-NEXT: v_mov_b32_e32 v0, 5 4062; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4063; GFX8-NEXT: v_mov_b32_e32 v1, 0 4064; GFX8-NEXT: s_mov_b32 m0, -1 4065; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4066; GFX8-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1] 4067; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4068; GFX8-NEXT: buffer_wbinvl1_vol 4069; GFX8-NEXT: BB20_2: 4070; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 4071; GFX8-NEXT: v_readfirstlane_b32 s4, v0 4072; GFX8-NEXT: v_bfrev_b32_e32 v0, -2 4073; GFX8-NEXT: v_readfirstlane_b32 s5, v1 4074; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v0, vcc 4075; GFX8-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4076; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1] 4077; GFX8-NEXT: v_mov_b32_e32 v2, s5 4078; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4079; GFX8-NEXT: v_mov_b32_e32 v2, s4 4080; GFX8-NEXT: s_mov_b32 s2, -1 4081; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4082; GFX8-NEXT: s_mov_b32 s3, 0xf000 4083; GFX8-NEXT: s_waitcnt lgkmcnt(0) 4084; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4085; GFX8-NEXT: s_endpgm 4086; 4087; GFX9-LABEL: min_i64_constant: 4088; GFX9: ; %bb.0: ; %entry 4089; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4090; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4091; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4092; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4093; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 4094; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 4095; GFX9-NEXT: s_cbranch_execz BB20_2 4096; GFX9-NEXT: ; %bb.1: 4097; GFX9-NEXT: v_mov_b32_e32 v0, 5 4098; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4099; GFX9-NEXT: v_mov_b32_e32 v1, 0 4100; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4101; GFX9-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1] 4102; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4103; GFX9-NEXT: buffer_wbinvl1_vol 4104; GFX9-NEXT: BB20_2: 4105; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 4106; GFX9-NEXT: v_readfirstlane_b32 s4, v0 4107; GFX9-NEXT: v_bfrev_b32_e32 v0, -2 4108; GFX9-NEXT: v_readfirstlane_b32 s5, v1 4109; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v0, vcc 4110; GFX9-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4111; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1] 4112; GFX9-NEXT: v_mov_b32_e32 v2, s5 4113; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4114; GFX9-NEXT: v_mov_b32_e32 v2, s4 4115; GFX9-NEXT: s_mov_b32 s2, -1 4116; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4117; GFX9-NEXT: s_mov_b32 s3, 0xf000 4118; GFX9-NEXT: s_waitcnt lgkmcnt(0) 4119; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4120; GFX9-NEXT: s_endpgm 4121; 4122; GFX1064-LABEL: min_i64_constant: 4123; GFX1064: ; %bb.0: ; %entry 4124; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4125; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4126; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0 4127; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4128; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1 4129; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 4130; GFX1064-NEXT: s_cbranch_execz BB20_2 4131; GFX1064-NEXT: ; %bb.1: 4132; GFX1064-NEXT: v_mov_b32_e32 v0, 5 4133; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4134; GFX1064-NEXT: v_mov_b32_e32 v1, 0 4135; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4136; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 4137; GFX1064-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1] 4138; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4139; GFX1064-NEXT: buffer_gl0_inv 4140; GFX1064-NEXT: buffer_gl1_inv 4141; GFX1064-NEXT: BB20_2: 4142; GFX1064-NEXT: v_nop 4143; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 4144; GFX1064-NEXT: v_readfirstlane_b32 s4, v0 4145; GFX1064-NEXT: v_readfirstlane_b32 s5, v1 4146; GFX1064-NEXT: v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc 4147; GFX1064-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4148; GFX1064-NEXT: s_mov_b32 s2, -1 4149; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 4150; GFX1064-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1] 4151; GFX1064-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc 4152; GFX1064-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc 4153; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 4154; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4155; GFX1064-NEXT: s_endpgm 4156; 4157; GFX1032-LABEL: min_i64_constant: 4158; GFX1032: ; %bb.0: ; %entry 4159; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4160; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4161; GFX1032-NEXT: ; implicit-def: $vcc_hi 4162; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 4163; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1 4164; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 4165; GFX1032-NEXT: s_cbranch_execz BB20_2 4166; GFX1032-NEXT: ; %bb.1: 4167; GFX1032-NEXT: v_mov_b32_e32 v0, 5 4168; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4169; GFX1032-NEXT: v_mov_b32_e32 v1, 0 4170; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4171; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 4172; GFX1032-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1] 4173; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4174; GFX1032-NEXT: buffer_gl0_inv 4175; GFX1032-NEXT: buffer_gl1_inv 4176; GFX1032-NEXT: BB20_2: 4177; GFX1032-NEXT: v_nop 4178; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 4179; GFX1032-NEXT: v_readfirstlane_b32 s4, v0 4180; GFX1032-NEXT: v_readfirstlane_b32 s5, v1 4181; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc_lo 4182; GFX1032-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc_lo 4183; GFX1032-NEXT: s_mov_b32 s2, -1 4184; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 4185; GFX1032-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1] 4186; GFX1032-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc_lo 4187; GFX1032-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc_lo 4188; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 4189; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4190; GFX1032-NEXT: s_endpgm 4191entry: 4192 %old = atomicrmw min i64 addrspace(3)* @local_var64, i64 5 acq_rel 4193 store i64 %old, i64 addrspace(1)* %out 4194 ret void 4195} 4196 4197define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) { 4198; 4199; 4200; GFX7LESS-LABEL: umax_i32_varying: 4201; GFX7LESS: ; %bb.0: ; %entry 4202; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 4203; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 4204; GFX7LESS-NEXT: s_mov_b32 m0, -1 4205; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4206; GFX7LESS-NEXT: ds_max_rtn_u32 v0, v1, v0 4207; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4208; GFX7LESS-NEXT: buffer_wbinvl1 4209; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 4210; GFX7LESS-NEXT: s_mov_b32 s2, -1 4211; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 4212; GFX7LESS-NEXT: s_endpgm 4213; 4214; GFX8-LABEL: umax_i32_varying: 4215; GFX8: ; %bb.0: ; %entry 4216; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4217; GFX8-NEXT: s_mov_b64 s[2:3], exec 4218; GFX8-NEXT: v_mov_b32_e32 v2, v0 4219; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 4220; GFX8-NEXT: v_mov_b32_e32 v1, 0 4221; GFX8-NEXT: s_mov_b64 exec, s[4:5] 4222; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 4223; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 4224; GFX8-NEXT: s_not_b64 exec, exec 4225; GFX8-NEXT: v_mov_b32_e32 v2, 0 4226; GFX8-NEXT: s_not_b64 exec, exec 4227; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 4228; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 4229; GFX8-NEXT: s_nop 1 4230; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 4231; GFX8-NEXT: s_nop 1 4232; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 4233; GFX8-NEXT: s_nop 1 4234; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 4235; GFX8-NEXT: s_nop 1 4236; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 4237; GFX8-NEXT: s_nop 1 4238; GFX8-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 4239; GFX8-NEXT: v_readlane_b32 s2, v2, 63 4240; GFX8-NEXT: s_nop 0 4241; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 4242; GFX8-NEXT: s_mov_b64 exec, s[4:5] 4243; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4244; GFX8-NEXT: ; implicit-def: $vgpr0 4245; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 4246; GFX8-NEXT: s_cbranch_execz BB21_2 4247; GFX8-NEXT: ; %bb.1: 4248; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4249; GFX8-NEXT: v_mov_b32_e32 v3, s2 4250; GFX8-NEXT: s_mov_b32 m0, -1 4251; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4252; GFX8-NEXT: ds_max_rtn_u32 v0, v0, v3 4253; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4254; GFX8-NEXT: buffer_wbinvl1_vol 4255; GFX8-NEXT: BB21_2: 4256; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 4257; GFX8-NEXT: v_readfirstlane_b32 s2, v0 4258; GFX8-NEXT: v_mov_b32_e32 v0, v1 4259; GFX8-NEXT: v_max_u32_e32 v0, s2, v0 4260; GFX8-NEXT: s_mov_b32 s3, 0xf000 4261; GFX8-NEXT: s_mov_b32 s2, -1 4262; GFX8-NEXT: s_nop 0 4263; GFX8-NEXT: s_waitcnt lgkmcnt(0) 4264; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 4265; GFX8-NEXT: s_endpgm 4266; 4267; GFX9-LABEL: umax_i32_varying: 4268; GFX9: ; %bb.0: ; %entry 4269; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4270; GFX9-NEXT: s_mov_b64 s[2:3], exec 4271; GFX9-NEXT: v_mov_b32_e32 v2, v0 4272; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 4273; GFX9-NEXT: v_mov_b32_e32 v1, 0 4274; GFX9-NEXT: s_mov_b64 exec, s[4:5] 4275; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 4276; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 4277; GFX9-NEXT: s_not_b64 exec, exec 4278; GFX9-NEXT: v_mov_b32_e32 v2, 0 4279; GFX9-NEXT: s_not_b64 exec, exec 4280; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 4281; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 4282; GFX9-NEXT: s_nop 1 4283; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 4284; GFX9-NEXT: s_nop 1 4285; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 4286; GFX9-NEXT: s_nop 1 4287; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 4288; GFX9-NEXT: s_nop 1 4289; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 4290; GFX9-NEXT: s_nop 1 4291; GFX9-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 4292; GFX9-NEXT: v_readlane_b32 s2, v2, 63 4293; GFX9-NEXT: s_nop 0 4294; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 4295; GFX9-NEXT: s_mov_b64 exec, s[4:5] 4296; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4297; GFX9-NEXT: ; implicit-def: $vgpr0 4298; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 4299; GFX9-NEXT: s_cbranch_execz BB21_2 4300; GFX9-NEXT: ; %bb.1: 4301; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4302; GFX9-NEXT: v_mov_b32_e32 v3, s2 4303; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4304; GFX9-NEXT: ds_max_rtn_u32 v0, v0, v3 4305; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4306; GFX9-NEXT: buffer_wbinvl1_vol 4307; GFX9-NEXT: BB21_2: 4308; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 4309; GFX9-NEXT: v_readfirstlane_b32 s2, v0 4310; GFX9-NEXT: v_mov_b32_e32 v0, v1 4311; GFX9-NEXT: v_max_u32_e32 v0, s2, v0 4312; GFX9-NEXT: s_mov_b32 s3, 0xf000 4313; GFX9-NEXT: s_mov_b32 s2, -1 4314; GFX9-NEXT: s_nop 0 4315; GFX9-NEXT: s_waitcnt lgkmcnt(0) 4316; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 4317; GFX9-NEXT: s_endpgm 4318; 4319; GFX1064-LABEL: umax_i32_varying: 4320; GFX1064: ; %bb.0: ; %entry 4321; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4322; GFX1064-NEXT: s_mov_b64 s[2:3], exec 4323; GFX1064-NEXT: v_mov_b32_e32 v2, v0 4324; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 4325; GFX1064-NEXT: v_mov_b32_e32 v1, 0 4326; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 4327; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 4328; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, s3, v0 4329; GFX1064-NEXT: s_not_b64 exec, exec 4330; GFX1064-NEXT: v_mov_b32_e32 v2, 0 4331; GFX1064-NEXT: s_not_b64 exec, exec 4332; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 4333; GFX1064-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 4334; GFX1064-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 4335; GFX1064-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 4336; GFX1064-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 4337; GFX1064-NEXT: v_mov_b32_e32 v3, v2 4338; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 4339; GFX1064-NEXT: v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 4340; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 4341; GFX1064-NEXT: v_mov_b32_e32 v3, s2 4342; GFX1064-NEXT: v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 4343; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 4344; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4345; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 4346; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 4347; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 4348; GFX1064-NEXT: s_mov_b32 s2, -1 4349; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 4350; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 4351; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 4352; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 4353; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4354; GFX1064-NEXT: ; implicit-def: $vgpr0 4355; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 4356; GFX1064-NEXT: s_cbranch_execz BB21_2 4357; GFX1064-NEXT: ; %bb.1: 4358; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4359; GFX1064-NEXT: v_mov_b32_e32 v7, s3 4360; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4361; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 4362; GFX1064-NEXT: ds_max_rtn_u32 v0, v0, v7 4363; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4364; GFX1064-NEXT: buffer_gl0_inv 4365; GFX1064-NEXT: buffer_gl1_inv 4366; GFX1064-NEXT: BB21_2: 4367; GFX1064-NEXT: v_nop 4368; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 4369; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 4370; GFX1064-NEXT: v_mov_b32_e32 v0, v1 4371; GFX1064-NEXT: v_max_u32_e32 v0, s3, v0 4372; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 4373; GFX1064-NEXT: s_nop 1 4374; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 4375; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 4376; GFX1064-NEXT: s_endpgm 4377; 4378; GFX1032-LABEL: umax_i32_varying: 4379; GFX1032: ; %bb.0: ; %entry 4380; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4381; GFX1032-NEXT: s_mov_b32 s2, exec_lo 4382; GFX1032-NEXT: ; implicit-def: $vcc_hi 4383; GFX1032-NEXT: v_mov_b32_e32 v2, v0 4384; GFX1032-NEXT: s_or_saveexec_b32 s3, -1 4385; GFX1032-NEXT: v_mov_b32_e32 v1, 0 4386; GFX1032-NEXT: s_mov_b32 exec_lo, s3 4387; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 4388; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 4389; GFX1032-NEXT: v_mov_b32_e32 v2, 0 4390; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 4391; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 4392; GFX1032-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0 4393; GFX1032-NEXT: s_mov_b32 s2, -1 4394; GFX1032-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0 4395; GFX1032-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0 4396; GFX1032-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0 4397; GFX1032-NEXT: v_mov_b32_e32 v3, v2 4398; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 4399; GFX1032-NEXT: v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 4400; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 4401; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4402; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 4403; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 4404; GFX1032-NEXT: s_mov_b32 exec_lo, s4 4405; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 4406; GFX1032-NEXT: ; implicit-def: $vgpr0 4407; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 4408; GFX1032-NEXT: s_cbranch_execz BB21_2 4409; GFX1032-NEXT: ; %bb.1: 4410; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4411; GFX1032-NEXT: v_mov_b32_e32 v7, s3 4412; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4413; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 4414; GFX1032-NEXT: ds_max_rtn_u32 v0, v0, v7 4415; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4416; GFX1032-NEXT: buffer_gl0_inv 4417; GFX1032-NEXT: buffer_gl1_inv 4418; GFX1032-NEXT: BB21_2: 4419; GFX1032-NEXT: v_nop 4420; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 4421; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 4422; GFX1032-NEXT: v_mov_b32_e32 v0, v1 4423; GFX1032-NEXT: v_max_u32_e32 v0, s3, v0 4424; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 4425; GFX1032-NEXT: s_nop 1 4426; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 4427; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 4428; GFX1032-NEXT: s_endpgm 4429entry: 4430 %lane = call i32 @llvm.amdgcn.workitem.id.x() 4431 %old = atomicrmw umax i32 addrspace(3)* @local_var32, i32 %lane acq_rel 4432 store i32 %old, i32 addrspace(1)* %out 4433 ret void 4434} 4435 4436define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) { 4437; 4438; 4439; GFX7LESS-LABEL: umax_i64_constant: 4440; GFX7LESS: ; %bb.0: ; %entry 4441; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 4442; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4443; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 4444; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4445; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1 4446; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 4447; GFX7LESS-NEXT: s_cbranch_execz BB22_2 4448; GFX7LESS-NEXT: ; %bb.1: 4449; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4450; GFX7LESS-NEXT: v_mov_b32_e32 v0, 5 4451; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 4452; GFX7LESS-NEXT: s_mov_b32 m0, -1 4453; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4454; GFX7LESS-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1] 4455; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4456; GFX7LESS-NEXT: buffer_wbinvl1 4457; GFX7LESS-NEXT: BB22_2: 4458; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 4459; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0 4460; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v1 4461; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 4462; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 4463; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 4464; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4 4465; GFX7LESS-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1] 4466; GFX7LESS-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4467; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5 4468; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc 4469; GFX7LESS-NEXT: s_mov_b32 s2, -1 4470; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 4471; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4472; GFX7LESS-NEXT: s_endpgm 4473; 4474; GFX8-LABEL: umax_i64_constant: 4475; GFX8: ; %bb.0: ; %entry 4476; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4477; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4478; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4479; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4480; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1 4481; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 4482; GFX8-NEXT: s_cbranch_execz BB22_2 4483; GFX8-NEXT: ; %bb.1: 4484; GFX8-NEXT: v_mov_b32_e32 v0, 5 4485; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4486; GFX8-NEXT: v_mov_b32_e32 v1, 0 4487; GFX8-NEXT: s_mov_b32 m0, -1 4488; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4489; GFX8-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1] 4490; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4491; GFX8-NEXT: buffer_wbinvl1_vol 4492; GFX8-NEXT: BB22_2: 4493; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 4494; GFX8-NEXT: v_readfirstlane_b32 s2, v0 4495; GFX8-NEXT: v_readfirstlane_b32 s3, v1 4496; GFX8-NEXT: v_mov_b32_e32 v1, 0 4497; GFX8-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 4498; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1] 4499; GFX8-NEXT: v_mov_b32_e32 v1, s3 4500; GFX8-NEXT: v_mov_b32_e32 v2, s2 4501; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4502; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc 4503; GFX8-NEXT: s_mov_b32 s3, 0xf000 4504; GFX8-NEXT: s_mov_b32 s2, -1 4505; GFX8-NEXT: s_waitcnt lgkmcnt(0) 4506; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4507; GFX8-NEXT: s_endpgm 4508; 4509; GFX9-LABEL: umax_i64_constant: 4510; GFX9: ; %bb.0: ; %entry 4511; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4512; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4513; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4514; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4515; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 4516; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 4517; GFX9-NEXT: s_cbranch_execz BB22_2 4518; GFX9-NEXT: ; %bb.1: 4519; GFX9-NEXT: v_mov_b32_e32 v0, 5 4520; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4521; GFX9-NEXT: v_mov_b32_e32 v1, 0 4522; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4523; GFX9-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1] 4524; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4525; GFX9-NEXT: buffer_wbinvl1_vol 4526; GFX9-NEXT: BB22_2: 4527; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 4528; GFX9-NEXT: v_readfirstlane_b32 s2, v0 4529; GFX9-NEXT: v_readfirstlane_b32 s3, v1 4530; GFX9-NEXT: v_mov_b32_e32 v1, 0 4531; GFX9-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 4532; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1] 4533; GFX9-NEXT: v_mov_b32_e32 v1, s3 4534; GFX9-NEXT: v_mov_b32_e32 v2, s2 4535; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4536; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc 4537; GFX9-NEXT: s_mov_b32 s3, 0xf000 4538; GFX9-NEXT: s_mov_b32 s2, -1 4539; GFX9-NEXT: s_waitcnt lgkmcnt(0) 4540; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4541; GFX9-NEXT: s_endpgm 4542; 4543; GFX1064-LABEL: umax_i64_constant: 4544; GFX1064: ; %bb.0: ; %entry 4545; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4546; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4547; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0 4548; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4549; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1 4550; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 4551; GFX1064-NEXT: s_cbranch_execz BB22_2 4552; GFX1064-NEXT: ; %bb.1: 4553; GFX1064-NEXT: v_mov_b32_e32 v0, 5 4554; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4555; GFX1064-NEXT: v_mov_b32_e32 v1, 0 4556; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4557; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 4558; GFX1064-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1] 4559; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4560; GFX1064-NEXT: buffer_gl0_inv 4561; GFX1064-NEXT: buffer_gl1_inv 4562; GFX1064-NEXT: BB22_2: 4563; GFX1064-NEXT: v_nop 4564; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 4565; GFX1064-NEXT: v_readfirstlane_b32 s4, v0 4566; GFX1064-NEXT: v_readfirstlane_b32 s5, v1 4567; GFX1064-NEXT: v_mov_b32_e32 v1, 0 4568; GFX1064-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc 4569; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 4570; GFX1064-NEXT: s_mov_b32 s2, -1 4571; GFX1064-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1] 4572; GFX1064-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc 4573; GFX1064-NEXT: v_cndmask_b32_e64 v1, 0, s5, vcc 4574; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 4575; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4576; GFX1064-NEXT: s_endpgm 4577; 4578; GFX1032-LABEL: umax_i64_constant: 4579; GFX1032: ; %bb.0: ; %entry 4580; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4581; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4582; GFX1032-NEXT: ; implicit-def: $vcc_hi 4583; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 4584; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1 4585; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 4586; GFX1032-NEXT: s_cbranch_execz BB22_2 4587; GFX1032-NEXT: ; %bb.1: 4588; GFX1032-NEXT: v_mov_b32_e32 v0, 5 4589; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4590; GFX1032-NEXT: v_mov_b32_e32 v1, 0 4591; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4592; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 4593; GFX1032-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1] 4594; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4595; GFX1032-NEXT: buffer_gl0_inv 4596; GFX1032-NEXT: buffer_gl1_inv 4597; GFX1032-NEXT: BB22_2: 4598; GFX1032-NEXT: v_nop 4599; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 4600; GFX1032-NEXT: v_readfirstlane_b32 s4, v0 4601; GFX1032-NEXT: v_readfirstlane_b32 s5, v1 4602; GFX1032-NEXT: v_mov_b32_e32 v1, 0 4603; GFX1032-NEXT: v_cndmask_b32_e64 v0, 5, 0, vcc_lo 4604; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 4605; GFX1032-NEXT: s_mov_b32 s2, -1 4606; GFX1032-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1] 4607; GFX1032-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc_lo 4608; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, s5, vcc_lo 4609; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 4610; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4611; GFX1032-NEXT: s_endpgm 4612entry: 4613 %old = atomicrmw umax i64 addrspace(3)* @local_var64, i64 5 acq_rel 4614 store i64 %old, i64 addrspace(1)* %out 4615 ret void 4616} 4617 4618define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) { 4619; 4620; 4621; GFX7LESS-LABEL: umin_i32_varying: 4622; GFX7LESS: ; %bb.0: ; %entry 4623; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 4624; GFX7LESS-NEXT: v_mov_b32_e32 v1, local_var32@abs32@lo 4625; GFX7LESS-NEXT: s_mov_b32 m0, -1 4626; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4627; GFX7LESS-NEXT: ds_min_rtn_u32 v0, v1, v0 4628; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4629; GFX7LESS-NEXT: buffer_wbinvl1 4630; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 4631; GFX7LESS-NEXT: s_mov_b32 s2, -1 4632; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0 4633; GFX7LESS-NEXT: s_endpgm 4634; 4635; GFX8-LABEL: umin_i32_varying: 4636; GFX8: ; %bb.0: ; %entry 4637; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4638; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 4639; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 4640; GFX8-NEXT: v_mov_b32_e32 v2, v0 4641; GFX8-NEXT: s_or_saveexec_b64 s[2:3], -1 4642; GFX8-NEXT: v_mov_b32_e32 v1, -1 4643; GFX8-NEXT: s_mov_b64 exec, s[2:3] 4644; GFX8-NEXT: s_not_b64 exec, exec 4645; GFX8-NEXT: v_mov_b32_e32 v2, -1 4646; GFX8-NEXT: s_not_b64 exec, exec 4647; GFX8-NEXT: s_or_saveexec_b64 s[4:5], -1 4648; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4649; GFX8-NEXT: s_nop 1 4650; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 4651; GFX8-NEXT: s_nop 1 4652; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 4653; GFX8-NEXT: s_nop 1 4654; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 4655; GFX8-NEXT: s_nop 1 4656; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 4657; GFX8-NEXT: s_nop 1 4658; GFX8-NEXT: v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 4659; GFX8-NEXT: v_readlane_b32 s2, v2, 63 4660; GFX8-NEXT: s_nop 0 4661; GFX8-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 4662; GFX8-NEXT: s_mov_b64 exec, s[4:5] 4663; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 4664; GFX8-NEXT: ; implicit-def: $vgpr0 4665; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc 4666; GFX8-NEXT: s_cbranch_execz BB23_2 4667; GFX8-NEXT: ; %bb.1: 4668; GFX8-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4669; GFX8-NEXT: v_mov_b32_e32 v3, s2 4670; GFX8-NEXT: s_mov_b32 m0, -1 4671; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4672; GFX8-NEXT: ds_min_rtn_u32 v0, v0, v3 4673; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4674; GFX8-NEXT: buffer_wbinvl1_vol 4675; GFX8-NEXT: BB23_2: 4676; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] 4677; GFX8-NEXT: v_readfirstlane_b32 s2, v0 4678; GFX8-NEXT: v_mov_b32_e32 v0, v1 4679; GFX8-NEXT: v_min_u32_e32 v0, s2, v0 4680; GFX8-NEXT: s_mov_b32 s3, 0xf000 4681; GFX8-NEXT: s_mov_b32 s2, -1 4682; GFX8-NEXT: s_nop 0 4683; GFX8-NEXT: s_waitcnt lgkmcnt(0) 4684; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0 4685; GFX8-NEXT: s_endpgm 4686; 4687; GFX9-LABEL: umin_i32_varying: 4688; GFX9: ; %bb.0: ; %entry 4689; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4690; GFX9-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 4691; GFX9-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 4692; GFX9-NEXT: v_mov_b32_e32 v2, v0 4693; GFX9-NEXT: s_or_saveexec_b64 s[2:3], -1 4694; GFX9-NEXT: v_mov_b32_e32 v1, -1 4695; GFX9-NEXT: s_mov_b64 exec, s[2:3] 4696; GFX9-NEXT: s_not_b64 exec, exec 4697; GFX9-NEXT: v_mov_b32_e32 v2, -1 4698; GFX9-NEXT: s_not_b64 exec, exec 4699; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1 4700; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4701; GFX9-NEXT: s_nop 1 4702; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 4703; GFX9-NEXT: s_nop 1 4704; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 4705; GFX9-NEXT: s_nop 1 4706; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 4707; GFX9-NEXT: s_nop 1 4708; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf 4709; GFX9-NEXT: s_nop 1 4710; GFX9-NEXT: v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf 4711; GFX9-NEXT: v_readlane_b32 s2, v2, 63 4712; GFX9-NEXT: s_nop 0 4713; GFX9-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf 4714; GFX9-NEXT: s_mov_b64 exec, s[4:5] 4715; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 4716; GFX9-NEXT: ; implicit-def: $vgpr0 4717; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc 4718; GFX9-NEXT: s_cbranch_execz BB23_2 4719; GFX9-NEXT: ; %bb.1: 4720; GFX9-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4721; GFX9-NEXT: v_mov_b32_e32 v3, s2 4722; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4723; GFX9-NEXT: ds_min_rtn_u32 v0, v0, v3 4724; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4725; GFX9-NEXT: buffer_wbinvl1_vol 4726; GFX9-NEXT: BB23_2: 4727; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] 4728; GFX9-NEXT: v_readfirstlane_b32 s2, v0 4729; GFX9-NEXT: v_mov_b32_e32 v0, v1 4730; GFX9-NEXT: v_min_u32_e32 v0, s2, v0 4731; GFX9-NEXT: s_mov_b32 s3, 0xf000 4732; GFX9-NEXT: s_mov_b32 s2, -1 4733; GFX9-NEXT: s_nop 0 4734; GFX9-NEXT: s_waitcnt lgkmcnt(0) 4735; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 4736; GFX9-NEXT: s_endpgm 4737; 4738; GFX1064-LABEL: umin_i32_varying: 4739; GFX1064: ; %bb.0: ; %entry 4740; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4741; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 4742; GFX1064-NEXT: v_mov_b32_e32 v2, v0 4743; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4 4744; GFX1064-NEXT: s_or_saveexec_b64 s[2:3], -1 4745; GFX1064-NEXT: v_mov_b32_e32 v1, -1 4746; GFX1064-NEXT: s_mov_b64 exec, s[2:3] 4747; GFX1064-NEXT: s_not_b64 exec, exec 4748; GFX1064-NEXT: v_mov_b32_e32 v2, -1 4749; GFX1064-NEXT: s_not_b64 exec, exec 4750; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1 4751; GFX1064-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4752; GFX1064-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 4753; GFX1064-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 4754; GFX1064-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 4755; GFX1064-NEXT: v_mov_b32_e32 v3, v2 4756; GFX1064-NEXT: v_permlanex16_b32 v3, v3, -1, -1 4757; GFX1064-NEXT: v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 4758; GFX1064-NEXT: v_readlane_b32 s2, v2, 31 4759; GFX1064-NEXT: v_mov_b32_e32 v3, s2 4760; GFX1064-NEXT: v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf 4761; GFX1064-NEXT: v_readlane_b32 s2, v2, 15 4762; GFX1064-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4763; GFX1064-NEXT: v_readlane_b32 s3, v2, 31 4764; GFX1064-NEXT: v_readlane_b32 s6, v2, 47 4765; GFX1064-NEXT: v_writelane_b32 v1, s2, 16 4766; GFX1064-NEXT: s_mov_b32 s2, -1 4767; GFX1064-NEXT: v_writelane_b32 v1, s3, 32 4768; GFX1064-NEXT: v_readlane_b32 s3, v2, 63 4769; GFX1064-NEXT: v_writelane_b32 v1, s6, 48 4770; GFX1064-NEXT: s_mov_b64 exec, s[4:5] 4771; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 4772; GFX1064-NEXT: ; implicit-def: $vgpr0 4773; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc 4774; GFX1064-NEXT: s_cbranch_execz BB23_2 4775; GFX1064-NEXT: ; %bb.1: 4776; GFX1064-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4777; GFX1064-NEXT: v_mov_b32_e32 v7, s3 4778; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4779; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 4780; GFX1064-NEXT: ds_min_rtn_u32 v0, v0, v7 4781; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4782; GFX1064-NEXT: buffer_gl0_inv 4783; GFX1064-NEXT: buffer_gl1_inv 4784; GFX1064-NEXT: BB23_2: 4785; GFX1064-NEXT: v_nop 4786; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] 4787; GFX1064-NEXT: v_readfirstlane_b32 s3, v0 4788; GFX1064-NEXT: v_mov_b32_e32 v0, v1 4789; GFX1064-NEXT: v_min_u32_e32 v0, s3, v0 4790; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 4791; GFX1064-NEXT: s_nop 1 4792; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 4793; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0 4794; GFX1064-NEXT: s_endpgm 4795; 4796; GFX1032-LABEL: umin_i32_varying: 4797; GFX1032: ; %bb.0: ; %entry 4798; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4799; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0 4800; GFX1032-NEXT: ; implicit-def: $vcc_hi 4801; GFX1032-NEXT: v_mov_b32_e32 v2, v0 4802; GFX1032-NEXT: s_or_saveexec_b32 s2, -1 4803; GFX1032-NEXT: v_mov_b32_e32 v1, -1 4804; GFX1032-NEXT: s_mov_b32 exec_lo, s2 4805; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 4806; GFX1032-NEXT: v_mov_b32_e32 v2, -1 4807; GFX1032-NEXT: s_not_b32 exec_lo, exec_lo 4808; GFX1032-NEXT: s_or_saveexec_b32 s4, -1 4809; GFX1032-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4810; GFX1032-NEXT: s_mov_b32 s2, -1 4811; GFX1032-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf 4812; GFX1032-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf 4813; GFX1032-NEXT: v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf 4814; GFX1032-NEXT: v_mov_b32_e32 v3, v2 4815; GFX1032-NEXT: v_permlanex16_b32 v3, v3, -1, -1 4816; GFX1032-NEXT: v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf 4817; GFX1032-NEXT: v_readlane_b32 s3, v2, 31 4818; GFX1032-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf 4819; GFX1032-NEXT: v_readlane_b32 s5, v2, 15 4820; GFX1032-NEXT: v_writelane_b32 v1, s5, 16 4821; GFX1032-NEXT: s_mov_b32 exec_lo, s4 4822; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 4823; GFX1032-NEXT: ; implicit-def: $vgpr0 4824; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo 4825; GFX1032-NEXT: s_cbranch_execz BB23_2 4826; GFX1032-NEXT: ; %bb.1: 4827; GFX1032-NEXT: v_mov_b32_e32 v0, local_var32@abs32@lo 4828; GFX1032-NEXT: v_mov_b32_e32 v7, s3 4829; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4830; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 4831; GFX1032-NEXT: ds_min_rtn_u32 v0, v0, v7 4832; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4833; GFX1032-NEXT: buffer_gl0_inv 4834; GFX1032-NEXT: buffer_gl1_inv 4835; GFX1032-NEXT: BB23_2: 4836; GFX1032-NEXT: v_nop 4837; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 4838; GFX1032-NEXT: v_readfirstlane_b32 s3, v0 4839; GFX1032-NEXT: v_mov_b32_e32 v0, v1 4840; GFX1032-NEXT: v_min_u32_e32 v0, s3, v0 4841; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 4842; GFX1032-NEXT: s_nop 1 4843; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 4844; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0 4845; GFX1032-NEXT: s_endpgm 4846entry: 4847 %lane = call i32 @llvm.amdgcn.workitem.id.x() 4848 %old = atomicrmw umin i32 addrspace(3)* @local_var32, i32 %lane acq_rel 4849 store i32 %old, i32 addrspace(1)* %out 4850 ret void 4851} 4852 4853define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) { 4854; 4855; 4856; GFX7LESS-LABEL: umin_i64_constant: 4857; GFX7LESS: ; %bb.0: ; %entry 4858; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 4859; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4860; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 4861; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4862; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1 4863; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc 4864; GFX7LESS-NEXT: s_cbranch_execz BB24_2 4865; GFX7LESS-NEXT: ; %bb.1: 4866; GFX7LESS-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4867; GFX7LESS-NEXT: v_mov_b32_e32 v0, 5 4868; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 4869; GFX7LESS-NEXT: s_mov_b32 m0, -1 4870; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4871; GFX7LESS-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1] 4872; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4873; GFX7LESS-NEXT: buffer_wbinvl1 4874; GFX7LESS-NEXT: BB24_2: 4875; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3] 4876; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0 4877; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v1 4878; GFX7LESS-NEXT: s_mov_b32 s2, -1 4879; GFX7LESS-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc 4880; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4881; GFX7LESS-NEXT: v_mov_b32_e32 v2, s5 4882; GFX7LESS-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1] 4883; GFX7LESS-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4884; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4 4885; GFX7LESS-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4886; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 4887; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) 4888; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4889; GFX7LESS-NEXT: s_endpgm 4890; 4891; GFX8-LABEL: umin_i64_constant: 4892; GFX8: ; %bb.0: ; %entry 4893; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4894; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4895; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4896; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4897; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1 4898; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc 4899; GFX8-NEXT: s_cbranch_execz BB24_2 4900; GFX8-NEXT: ; %bb.1: 4901; GFX8-NEXT: v_mov_b32_e32 v0, 5 4902; GFX8-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4903; GFX8-NEXT: v_mov_b32_e32 v1, 0 4904; GFX8-NEXT: s_mov_b32 m0, -1 4905; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4906; GFX8-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1] 4907; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4908; GFX8-NEXT: buffer_wbinvl1_vol 4909; GFX8-NEXT: BB24_2: 4910; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] 4911; GFX8-NEXT: v_readfirstlane_b32 s5, v1 4912; GFX8-NEXT: v_readfirstlane_b32 s4, v0 4913; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc 4914; GFX8-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4915; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1] 4916; GFX8-NEXT: v_mov_b32_e32 v2, s5 4917; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4918; GFX8-NEXT: v_mov_b32_e32 v2, s4 4919; GFX8-NEXT: s_mov_b32 s2, -1 4920; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4921; GFX8-NEXT: s_mov_b32 s3, 0xf000 4922; GFX8-NEXT: s_waitcnt lgkmcnt(0) 4923; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4924; GFX8-NEXT: s_endpgm 4925; 4926; GFX9-LABEL: umin_i64_constant: 4927; GFX9: ; %bb.0: ; %entry 4928; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4929; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 4930; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 4931; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4932; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 4933; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc 4934; GFX9-NEXT: s_cbranch_execz BB24_2 4935; GFX9-NEXT: ; %bb.1: 4936; GFX9-NEXT: v_mov_b32_e32 v0, 5 4937; GFX9-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4938; GFX9-NEXT: v_mov_b32_e32 v1, 0 4939; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4940; GFX9-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1] 4941; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4942; GFX9-NEXT: buffer_wbinvl1_vol 4943; GFX9-NEXT: BB24_2: 4944; GFX9-NEXT: s_or_b64 exec, exec, s[2:3] 4945; GFX9-NEXT: v_readfirstlane_b32 s5, v1 4946; GFX9-NEXT: v_readfirstlane_b32 s4, v0 4947; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc 4948; GFX9-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4949; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1] 4950; GFX9-NEXT: v_mov_b32_e32 v2, s5 4951; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc 4952; GFX9-NEXT: v_mov_b32_e32 v2, s4 4953; GFX9-NEXT: s_mov_b32 s2, -1 4954; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc 4955; GFX9-NEXT: s_mov_b32 s3, 0xf000 4956; GFX9-NEXT: s_waitcnt lgkmcnt(0) 4957; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4958; GFX9-NEXT: s_endpgm 4959; 4960; GFX1064-LABEL: umin_i64_constant: 4961; GFX1064: ; %bb.0: ; %entry 4962; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4963; GFX1064-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4964; GFX1064-NEXT: v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0 4965; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 4966; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1 4967; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc 4968; GFX1064-NEXT: s_cbranch_execz BB24_2 4969; GFX1064-NEXT: ; %bb.1: 4970; GFX1064-NEXT: v_mov_b32_e32 v0, 5 4971; GFX1064-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 4972; GFX1064-NEXT: v_mov_b32_e32 v1, 0 4973; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4974; GFX1064-NEXT: s_waitcnt_vscnt null, 0x0 4975; GFX1064-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1] 4976; GFX1064-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4977; GFX1064-NEXT: buffer_gl0_inv 4978; GFX1064-NEXT: buffer_gl1_inv 4979; GFX1064-NEXT: BB24_2: 4980; GFX1064-NEXT: v_nop 4981; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3] 4982; GFX1064-NEXT: v_readfirstlane_b32 s4, v0 4983; GFX1064-NEXT: v_readfirstlane_b32 s5, v1 4984; GFX1064-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc 4985; GFX1064-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc 4986; GFX1064-NEXT: s_mov_b32 s2, -1 4987; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 4988; GFX1064-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1] 4989; GFX1064-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc 4990; GFX1064-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc 4991; GFX1064-NEXT: s_waitcnt lgkmcnt(0) 4992; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 4993; GFX1064-NEXT: s_endpgm 4994; 4995; GFX1032-LABEL: umin_i64_constant: 4996; GFX1032: ; %bb.0: ; %entry 4997; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 4998; GFX1032-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 4999; GFX1032-NEXT: ; implicit-def: $vcc_hi 5000; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 5001; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1 5002; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo 5003; GFX1032-NEXT: s_cbranch_execz BB24_2 5004; GFX1032-NEXT: ; %bb.1: 5005; GFX1032-NEXT: v_mov_b32_e32 v0, 5 5006; GFX1032-NEXT: v_mov_b32_e32 v2, local_var64@abs32@lo 5007; GFX1032-NEXT: v_mov_b32_e32 v1, 0 5008; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5009; GFX1032-NEXT: s_waitcnt_vscnt null, 0x0 5010; GFX1032-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1] 5011; GFX1032-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5012; GFX1032-NEXT: buffer_gl0_inv 5013; GFX1032-NEXT: buffer_gl1_inv 5014; GFX1032-NEXT: BB24_2: 5015; GFX1032-NEXT: v_nop 5016; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2 5017; GFX1032-NEXT: v_readfirstlane_b32 s4, v0 5018; GFX1032-NEXT: v_readfirstlane_b32 s5, v1 5019; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc_lo 5020; GFX1032-NEXT: v_cndmask_b32_e64 v0, 5, -1, vcc_lo 5021; GFX1032-NEXT: s_mov_b32 s2, -1 5022; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 5023; GFX1032-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1] 5024; GFX1032-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc_lo 5025; GFX1032-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc_lo 5026; GFX1032-NEXT: s_waitcnt lgkmcnt(0) 5027; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 5028; GFX1032-NEXT: s_endpgm 5029entry: 5030 %old = atomicrmw umin i64 addrspace(3)* @local_var64, i64 5 acq_rel 5031 store i64 %old, i64 addrspace(1)* %out 5032 ret void 5033} 5034