1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s 3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s 4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s 5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s 6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s 7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s 8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s 9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s 10 11define amdgpu_kernel void @flat_workgroup_unordered_load( 12; GFX7-LABEL: flat_workgroup_unordered_load: 13; GFX7: ; %bb.0: ; %entry 14; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 15; GFX7-NEXT: s_waitcnt lgkmcnt(0) 16; GFX7-NEXT: v_mov_b32_e32 v0, s0 17; GFX7-NEXT: v_mov_b32_e32 v1, s1 18; GFX7-NEXT: flat_load_dword v2, v[0:1] 19; GFX7-NEXT: v_mov_b32_e32 v0, s2 20; GFX7-NEXT: v_mov_b32_e32 v1, s3 21; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 22; GFX7-NEXT: flat_store_dword v[0:1], v2 23; GFX7-NEXT: s_endpgm 24; 25; GFX10-WGP-LABEL: flat_workgroup_unordered_load: 26; GFX10-WGP: ; %bb.0: ; %entry 27; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 28; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 29; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 30; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 31; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] 32; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 33; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 34; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 35; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 36; GFX10-WGP-NEXT: s_endpgm 37; 38; GFX10-CU-LABEL: flat_workgroup_unordered_load: 39; GFX10-CU: ; %bb.0: ; %entry 40; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 41; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 42; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 43; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 44; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 45; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 46; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 47; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 48; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 49; GFX10-CU-NEXT: s_endpgm 50; 51; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load: 52; SKIP-CACHE-INV: ; %bb.0: ; %entry 53; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 54; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 55; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 56; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 57; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 58; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 59; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 60; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 61; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 62; SKIP-CACHE-INV-NEXT: s_endpgm 63; 64; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load: 65; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 66; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 67; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 68; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 69; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 70; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 71; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 72; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 73; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 74; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 75; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 76; 77; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load: 78; GFX90A-TGSPLIT: ; %bb.0: ; %entry 79; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 80; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 81; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 82; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 83; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] 84; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 85; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 86; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 87; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 88; GFX90A-TGSPLIT-NEXT: s_endpgm 89; 90; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load: 91; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 92; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 93; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 94; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 95; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 96; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 97; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 98; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 99; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 100; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 101; GFX940-NOTTGSPLIT-NEXT: s_endpgm 102; 103; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_load: 104; GFX940-TGSPLIT: ; %bb.0: ; %entry 105; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 106; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 107; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 108; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 109; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] 110; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 111; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 112; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 113; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 114; GFX940-TGSPLIT-NEXT: s_endpgm 115 i32* %in, i32* %out) { 116entry: 117 %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4 118 store i32 %val, i32* %out 119 ret void 120} 121 122define amdgpu_kernel void @flat_workgroup_monotonic_load( 123; GFX7-LABEL: flat_workgroup_monotonic_load: 124; GFX7: ; %bb.0: ; %entry 125; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 126; GFX7-NEXT: s_waitcnt lgkmcnt(0) 127; GFX7-NEXT: v_mov_b32_e32 v0, s0 128; GFX7-NEXT: v_mov_b32_e32 v1, s1 129; GFX7-NEXT: flat_load_dword v2, v[0:1] 130; GFX7-NEXT: v_mov_b32_e32 v0, s2 131; GFX7-NEXT: v_mov_b32_e32 v1, s3 132; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 133; GFX7-NEXT: flat_store_dword v[0:1], v2 134; GFX7-NEXT: s_endpgm 135; 136; GFX10-WGP-LABEL: flat_workgroup_monotonic_load: 137; GFX10-WGP: ; %bb.0: ; %entry 138; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 139; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 140; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 141; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 142; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 143; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 144; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 145; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 146; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 147; GFX10-WGP-NEXT: s_endpgm 148; 149; GFX10-CU-LABEL: flat_workgroup_monotonic_load: 150; GFX10-CU: ; %bb.0: ; %entry 151; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 152; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 153; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 154; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 155; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 156; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 157; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 158; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 159; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 160; GFX10-CU-NEXT: s_endpgm 161; 162; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load: 163; SKIP-CACHE-INV: ; %bb.0: ; %entry 164; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 165; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 166; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 167; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 168; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 169; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 170; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 171; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 172; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 173; SKIP-CACHE-INV-NEXT: s_endpgm 174; 175; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load: 176; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 177; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 178; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 179; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 180; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 181; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 182; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 183; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 184; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 185; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 186; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 187; 188; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load: 189; GFX90A-TGSPLIT: ; %bb.0: ; %entry 190; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 191; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 192; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 193; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 194; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 195; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 196; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 197; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 198; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 199; GFX90A-TGSPLIT-NEXT: s_endpgm 200; 201; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load: 202; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 203; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 204; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 205; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 206; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 207; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 208; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 209; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 210; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 211; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 212; GFX940-NOTTGSPLIT-NEXT: s_endpgm 213; 214; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_load: 215; GFX940-TGSPLIT: ; %bb.0: ; %entry 216; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 217; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 218; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 219; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 220; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 221; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 222; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 223; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 224; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 225; GFX940-TGSPLIT-NEXT: s_endpgm 226 i32* %in, i32* %out) { 227entry: 228 %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4 229 store i32 %val, i32* %out 230 ret void 231} 232 233define amdgpu_kernel void @flat_workgroup_acquire_load( 234; GFX7-LABEL: flat_workgroup_acquire_load: 235; GFX7: ; %bb.0: ; %entry 236; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 237; GFX7-NEXT: s_waitcnt lgkmcnt(0) 238; GFX7-NEXT: v_mov_b32_e32 v0, s0 239; GFX7-NEXT: v_mov_b32_e32 v1, s1 240; GFX7-NEXT: flat_load_dword v2, v[0:1] 241; GFX7-NEXT: s_waitcnt lgkmcnt(0) 242; GFX7-NEXT: v_mov_b32_e32 v0, s2 243; GFX7-NEXT: v_mov_b32_e32 v1, s3 244; GFX7-NEXT: s_waitcnt vmcnt(0) 245; GFX7-NEXT: flat_store_dword v[0:1], v2 246; GFX7-NEXT: s_endpgm 247; 248; GFX10-WGP-LABEL: flat_workgroup_acquire_load: 249; GFX10-WGP: ; %bb.0: ; %entry 250; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 251; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 252; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 253; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 254; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 255; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 256; GFX10-WGP-NEXT: buffer_gl0_inv 257; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 258; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 259; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 260; GFX10-WGP-NEXT: s_endpgm 261; 262; GFX10-CU-LABEL: flat_workgroup_acquire_load: 263; GFX10-CU: ; %bb.0: ; %entry 264; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 265; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 266; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 267; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 268; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 269; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 270; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 271; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 272; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 273; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 274; GFX10-CU-NEXT: s_endpgm 275; 276; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load: 277; SKIP-CACHE-INV: ; %bb.0: ; %entry 278; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 279; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 280; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 281; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 282; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 283; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 284; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 285; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 286; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 287; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 288; SKIP-CACHE-INV-NEXT: s_endpgm 289; 290; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load: 291; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 292; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 293; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 294; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 295; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 296; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 297; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 298; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 299; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 300; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) 301; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 302; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 303; 304; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load: 305; GFX90A-TGSPLIT: ; %bb.0: ; %entry 306; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 307; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 308; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 309; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 310; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 311; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 312; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 313; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 314; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 315; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 316; GFX90A-TGSPLIT-NEXT: s_endpgm 317; 318; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load: 319; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 320; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 321; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 322; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 323; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 324; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 325; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 326; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 327; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 328; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) 329; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 330; GFX940-NOTTGSPLIT-NEXT: s_endpgm 331; 332; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_load: 333; GFX940-TGSPLIT: ; %bb.0: ; %entry 334; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 335; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 336; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 337; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 338; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 339; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 340; GFX940-TGSPLIT-NEXT: buffer_inv sc0 341; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 342; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 343; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 344; GFX940-TGSPLIT-NEXT: s_endpgm 345 i32* %in, i32* %out) { 346entry: 347 %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4 348 store i32 %val, i32* %out 349 ret void 350} 351 352define amdgpu_kernel void @flat_workgroup_seq_cst_load( 353; GFX7-LABEL: flat_workgroup_seq_cst_load: 354; GFX7: ; %bb.0: ; %entry 355; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 356; GFX7-NEXT: s_waitcnt lgkmcnt(0) 357; GFX7-NEXT: v_mov_b32_e32 v0, s0 358; GFX7-NEXT: v_mov_b32_e32 v1, s1 359; GFX7-NEXT: s_waitcnt lgkmcnt(0) 360; GFX7-NEXT: flat_load_dword v2, v[0:1] 361; GFX7-NEXT: s_waitcnt lgkmcnt(0) 362; GFX7-NEXT: v_mov_b32_e32 v0, s2 363; GFX7-NEXT: v_mov_b32_e32 v1, s3 364; GFX7-NEXT: s_waitcnt vmcnt(0) 365; GFX7-NEXT: flat_store_dword v[0:1], v2 366; GFX7-NEXT: s_endpgm 367; 368; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load: 369; GFX10-WGP: ; %bb.0: ; %entry 370; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 371; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 372; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 373; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 374; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 375; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 376; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 377; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 378; GFX10-WGP-NEXT: buffer_gl0_inv 379; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 380; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 381; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 382; GFX10-WGP-NEXT: s_endpgm 383; 384; GFX10-CU-LABEL: flat_workgroup_seq_cst_load: 385; GFX10-CU: ; %bb.0: ; %entry 386; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 387; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 388; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 389; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 390; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 391; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 392; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 393; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 394; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 395; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 396; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 397; GFX10-CU-NEXT: s_endpgm 398; 399; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load: 400; SKIP-CACHE-INV: ; %bb.0: ; %entry 401; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 402; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 403; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 404; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 405; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 406; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 407; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 408; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 409; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 410; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 411; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 412; SKIP-CACHE-INV-NEXT: s_endpgm 413; 414; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load: 415; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 416; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 417; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 418; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 419; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 420; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 421; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 422; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 423; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 424; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 425; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) 426; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 427; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 428; 429; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load: 430; GFX90A-TGSPLIT: ; %bb.0: ; %entry 431; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 432; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 433; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 434; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 435; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 436; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 437; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 438; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 439; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 440; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 441; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 442; GFX90A-TGSPLIT-NEXT: s_endpgm 443; 444; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load: 445; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 446; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 447; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 448; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 449; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 450; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 451; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 452; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 453; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 454; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 455; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) 456; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 457; GFX940-NOTTGSPLIT-NEXT: s_endpgm 458; 459; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_load: 460; GFX940-TGSPLIT: ; %bb.0: ; %entry 461; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 462; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 463; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 464; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 465; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 466; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 467; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 468; GFX940-TGSPLIT-NEXT: buffer_inv sc0 469; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 470; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 471; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 472; GFX940-TGSPLIT-NEXT: s_endpgm 473 i32* %in, i32* %out) { 474entry: 475 %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4 476 store i32 %val, i32* %out 477 ret void 478} 479 480define amdgpu_kernel void @flat_workgroup_unordered_store( 481; GFX7-LABEL: flat_workgroup_unordered_store: 482; GFX7: ; %bb.0: ; %entry 483; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 484; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 485; GFX7-NEXT: s_waitcnt lgkmcnt(0) 486; GFX7-NEXT: v_mov_b32_e32 v0, s0 487; GFX7-NEXT: v_mov_b32_e32 v1, s1 488; GFX7-NEXT: v_mov_b32_e32 v2, s2 489; GFX7-NEXT: flat_store_dword v[0:1], v2 490; GFX7-NEXT: s_endpgm 491; 492; GFX10-WGP-LABEL: flat_workgroup_unordered_store: 493; GFX10-WGP: ; %bb.0: ; %entry 494; GFX10-WGP-NEXT: s_clause 0x1 495; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 496; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 497; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 498; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 499; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 500; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 501; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 502; GFX10-WGP-NEXT: s_endpgm 503; 504; GFX10-CU-LABEL: flat_workgroup_unordered_store: 505; GFX10-CU: ; %bb.0: ; %entry 506; GFX10-CU-NEXT: s_clause 0x1 507; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 508; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 509; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 510; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 511; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 512; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 513; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 514; GFX10-CU-NEXT: s_endpgm 515; 516; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store: 517; SKIP-CACHE-INV: ; %bb.0: ; %entry 518; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 519; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 520; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 521; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 522; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 523; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 524; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 525; SKIP-CACHE-INV-NEXT: s_endpgm 526; 527; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store: 528; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 529; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 530; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 531; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 532; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 533; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 534; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 535; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 536; 537; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store: 538; GFX90A-TGSPLIT: ; %bb.0: ; %entry 539; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 540; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 541; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 542; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 543; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 544; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 545; GFX90A-TGSPLIT-NEXT: s_endpgm 546; 547; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store: 548; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 549; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 550; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 551; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 552; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 553; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 554; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 555; GFX940-NOTTGSPLIT-NEXT: s_endpgm 556; 557; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_store: 558; GFX940-TGSPLIT: ; %bb.0: ; %entry 559; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 560; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 561; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 562; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 563; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 564; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 565; GFX940-TGSPLIT-NEXT: s_endpgm 566 i32 %in, i32* %out) { 567entry: 568 store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4 569 ret void 570} 571 572define amdgpu_kernel void @flat_workgroup_monotonic_store( 573; GFX7-LABEL: flat_workgroup_monotonic_store: 574; GFX7: ; %bb.0: ; %entry 575; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 576; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 577; GFX7-NEXT: s_waitcnt lgkmcnt(0) 578; GFX7-NEXT: v_mov_b32_e32 v0, s0 579; GFX7-NEXT: v_mov_b32_e32 v1, s1 580; GFX7-NEXT: v_mov_b32_e32 v2, s2 581; GFX7-NEXT: flat_store_dword v[0:1], v2 582; GFX7-NEXT: s_endpgm 583; 584; GFX10-WGP-LABEL: flat_workgroup_monotonic_store: 585; GFX10-WGP: ; %bb.0: ; %entry 586; GFX10-WGP-NEXT: s_clause 0x1 587; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 588; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 589; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 590; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 591; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 592; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 593; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 594; GFX10-WGP-NEXT: s_endpgm 595; 596; GFX10-CU-LABEL: flat_workgroup_monotonic_store: 597; GFX10-CU: ; %bb.0: ; %entry 598; GFX10-CU-NEXT: s_clause 0x1 599; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 600; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 601; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 602; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 603; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 604; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 605; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 606; GFX10-CU-NEXT: s_endpgm 607; 608; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store: 609; SKIP-CACHE-INV: ; %bb.0: ; %entry 610; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 611; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 612; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 613; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 614; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 615; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 616; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 617; SKIP-CACHE-INV-NEXT: s_endpgm 618; 619; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store: 620; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 621; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 622; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 623; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 624; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 625; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 626; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 627; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 628; 629; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store: 630; GFX90A-TGSPLIT: ; %bb.0: ; %entry 631; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 632; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 633; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 634; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 635; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 636; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 637; GFX90A-TGSPLIT-NEXT: s_endpgm 638; 639; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store: 640; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 641; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 642; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 643; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 644; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 645; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 646; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 647; GFX940-NOTTGSPLIT-NEXT: s_endpgm 648; 649; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_store: 650; GFX940-TGSPLIT: ; %bb.0: ; %entry 651; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 652; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 653; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 654; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 655; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 656; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 657; GFX940-TGSPLIT-NEXT: s_endpgm 658 i32 %in, i32* %out) { 659entry: 660 store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4 661 ret void 662} 663 664define amdgpu_kernel void @flat_workgroup_release_store( 665; GFX7-LABEL: flat_workgroup_release_store: 666; GFX7: ; %bb.0: ; %entry 667; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 668; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 669; GFX7-NEXT: s_waitcnt lgkmcnt(0) 670; GFX7-NEXT: v_mov_b32_e32 v0, s0 671; GFX7-NEXT: v_mov_b32_e32 v1, s1 672; GFX7-NEXT: v_mov_b32_e32 v2, s2 673; GFX7-NEXT: s_waitcnt lgkmcnt(0) 674; GFX7-NEXT: flat_store_dword v[0:1], v2 675; GFX7-NEXT: s_endpgm 676; 677; GFX10-WGP-LABEL: flat_workgroup_release_store: 678; GFX10-WGP: ; %bb.0: ; %entry 679; GFX10-WGP-NEXT: s_clause 0x1 680; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 681; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 682; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 683; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 684; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 685; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 686; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 687; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 688; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 689; GFX10-WGP-NEXT: s_endpgm 690; 691; GFX10-CU-LABEL: flat_workgroup_release_store: 692; GFX10-CU: ; %bb.0: ; %entry 693; GFX10-CU-NEXT: s_clause 0x1 694; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 695; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 696; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 697; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 698; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 699; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 700; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 701; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 702; GFX10-CU-NEXT: s_endpgm 703; 704; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store: 705; SKIP-CACHE-INV: ; %bb.0: ; %entry 706; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 707; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 708; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 709; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 710; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 711; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 712; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 713; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 714; SKIP-CACHE-INV-NEXT: s_endpgm 715; 716; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store: 717; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 718; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 719; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 720; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 721; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 722; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 723; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 724; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 725; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 726; 727; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store: 728; GFX90A-TGSPLIT: ; %bb.0: ; %entry 729; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 730; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 731; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 732; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 733; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 734; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 735; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 736; GFX90A-TGSPLIT-NEXT: s_endpgm 737; 738; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_store: 739; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 740; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 741; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 742; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 743; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 744; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 745; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 746; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 747; GFX940-NOTTGSPLIT-NEXT: s_endpgm 748; 749; GFX940-TGSPLIT-LABEL: flat_workgroup_release_store: 750; GFX940-TGSPLIT: ; %bb.0: ; %entry 751; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 752; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 753; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 754; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 755; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 756; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 757; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 758; GFX940-TGSPLIT-NEXT: s_endpgm 759 i32 %in, i32* %out) { 760entry: 761 store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4 762 ret void 763} 764 765define amdgpu_kernel void @flat_workgroup_seq_cst_store( 766; GFX7-LABEL: flat_workgroup_seq_cst_store: 767; GFX7: ; %bb.0: ; %entry 768; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 769; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 770; GFX7-NEXT: s_waitcnt lgkmcnt(0) 771; GFX7-NEXT: v_mov_b32_e32 v0, s0 772; GFX7-NEXT: v_mov_b32_e32 v1, s1 773; GFX7-NEXT: v_mov_b32_e32 v2, s2 774; GFX7-NEXT: s_waitcnt lgkmcnt(0) 775; GFX7-NEXT: flat_store_dword v[0:1], v2 776; GFX7-NEXT: s_endpgm 777; 778; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store: 779; GFX10-WGP: ; %bb.0: ; %entry 780; GFX10-WGP-NEXT: s_clause 0x1 781; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 782; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 783; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 784; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 785; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 786; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 787; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 788; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 789; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 790; GFX10-WGP-NEXT: s_endpgm 791; 792; GFX10-CU-LABEL: flat_workgroup_seq_cst_store: 793; GFX10-CU: ; %bb.0: ; %entry 794; GFX10-CU-NEXT: s_clause 0x1 795; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 796; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 797; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 798; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 799; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 800; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 801; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 802; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 803; GFX10-CU-NEXT: s_endpgm 804; 805; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store: 806; SKIP-CACHE-INV: ; %bb.0: ; %entry 807; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 808; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 809; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 810; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 811; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 812; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 813; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 814; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 815; SKIP-CACHE-INV-NEXT: s_endpgm 816; 817; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store: 818; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 819; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 820; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 821; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 822; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 823; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 824; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 825; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 826; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 827; 828; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store: 829; GFX90A-TGSPLIT: ; %bb.0: ; %entry 830; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 831; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 832; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 833; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 834; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 835; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 836; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 837; GFX90A-TGSPLIT-NEXT: s_endpgm 838; 839; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store: 840; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 841; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 842; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 843; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 844; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 845; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 846; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 847; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 848; GFX940-NOTTGSPLIT-NEXT: s_endpgm 849; 850; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_store: 851; GFX940-TGSPLIT: ; %bb.0: ; %entry 852; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 853; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 854; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 855; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 856; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 857; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 858; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 859; GFX940-TGSPLIT-NEXT: s_endpgm 860 i32 %in, i32* %out) { 861entry: 862 store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4 863 ret void 864} 865 866define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw( 867; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw: 868; GFX7: ; %bb.0: ; %entry 869; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 870; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 871; GFX7-NEXT: s_waitcnt lgkmcnt(0) 872; GFX7-NEXT: v_mov_b32_e32 v0, s0 873; GFX7-NEXT: v_mov_b32_e32 v1, s1 874; GFX7-NEXT: v_mov_b32_e32 v2, s2 875; GFX7-NEXT: flat_atomic_swap v[0:1], v2 876; GFX7-NEXT: s_endpgm 877; 878; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw: 879; GFX10-WGP: ; %bb.0: ; %entry 880; GFX10-WGP-NEXT: s_clause 0x1 881; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 882; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 883; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 884; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 885; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 886; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 887; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 888; GFX10-WGP-NEXT: s_endpgm 889; 890; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw: 891; GFX10-CU: ; %bb.0: ; %entry 892; GFX10-CU-NEXT: s_clause 0x1 893; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 894; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 895; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 896; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 897; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 898; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 899; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 900; GFX10-CU-NEXT: s_endpgm 901; 902; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw: 903; SKIP-CACHE-INV: ; %bb.0: ; %entry 904; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 905; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 906; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 907; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 908; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 909; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 910; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 911; SKIP-CACHE-INV-NEXT: s_endpgm 912; 913; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw: 914; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 915; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 916; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 917; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 918; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 919; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 920; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 921; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 922; 923; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw: 924; GFX90A-TGSPLIT: ; %bb.0: ; %entry 925; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 926; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 927; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 928; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 929; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 930; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 931; GFX90A-TGSPLIT-NEXT: s_endpgm 932; 933; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw: 934; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 935; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 936; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 937; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 938; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 939; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 940; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 941; GFX940-NOTTGSPLIT-NEXT: s_endpgm 942; 943; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw: 944; GFX940-TGSPLIT: ; %bb.0: ; %entry 945; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 946; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 947; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 948; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 949; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 950; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 951; GFX940-TGSPLIT-NEXT: s_endpgm 952 i32* %out, i32 %in) { 953entry: 954 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic 955 ret void 956} 957 958define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw( 959; GFX7-LABEL: flat_workgroup_acquire_atomicrmw: 960; GFX7: ; %bb.0: ; %entry 961; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 962; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 963; GFX7-NEXT: s_waitcnt lgkmcnt(0) 964; GFX7-NEXT: v_mov_b32_e32 v0, s0 965; GFX7-NEXT: v_mov_b32_e32 v1, s1 966; GFX7-NEXT: v_mov_b32_e32 v2, s2 967; GFX7-NEXT: flat_atomic_swap v[0:1], v2 968; GFX7-NEXT: s_waitcnt lgkmcnt(0) 969; GFX7-NEXT: s_endpgm 970; 971; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw: 972; GFX10-WGP: ; %bb.0: ; %entry 973; GFX10-WGP-NEXT: s_clause 0x1 974; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 975; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 976; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 977; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 978; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 979; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 980; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 981; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 982; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 983; GFX10-WGP-NEXT: buffer_gl0_inv 984; GFX10-WGP-NEXT: s_endpgm 985; 986; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw: 987; GFX10-CU: ; %bb.0: ; %entry 988; GFX10-CU-NEXT: s_clause 0x1 989; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 990; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 991; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 992; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 993; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 994; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 995; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 996; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 997; GFX10-CU-NEXT: s_endpgm 998; 999; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw: 1000; SKIP-CACHE-INV: ; %bb.0: ; %entry 1001; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1002; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1003; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1004; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1005; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1006; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1007; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 1008; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1009; SKIP-CACHE-INV-NEXT: s_endpgm 1010; 1011; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw: 1012; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1013; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1014; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1015; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1016; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1017; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1018; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1019; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1020; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1021; 1022; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw: 1023; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1024; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1025; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1026; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1027; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1028; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1029; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1030; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1031; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1032; GFX90A-TGSPLIT-NEXT: s_endpgm 1033; 1034; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw: 1035; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1036; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1037; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1038; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1039; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1040; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1041; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1042; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1043; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1044; 1045; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw: 1046; GFX940-TGSPLIT: ; %bb.0: ; %entry 1047; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1048; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1049; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1050; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1051; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1052; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1053; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1054; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1055; GFX940-TGSPLIT-NEXT: s_endpgm 1056 i32* %out, i32 %in) { 1057entry: 1058 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire 1059 ret void 1060} 1061 1062define amdgpu_kernel void @flat_workgroup_release_atomicrmw( 1063; GFX7-LABEL: flat_workgroup_release_atomicrmw: 1064; GFX7: ; %bb.0: ; %entry 1065; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1066; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1067; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1068; GFX7-NEXT: v_mov_b32_e32 v0, s0 1069; GFX7-NEXT: v_mov_b32_e32 v1, s1 1070; GFX7-NEXT: v_mov_b32_e32 v2, s2 1071; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1072; GFX7-NEXT: flat_atomic_swap v[0:1], v2 1073; GFX7-NEXT: s_endpgm 1074; 1075; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw: 1076; GFX10-WGP: ; %bb.0: ; %entry 1077; GFX10-WGP-NEXT: s_clause 0x1 1078; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1079; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1080; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1081; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1082; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1083; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1084; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1085; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1086; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 1087; GFX10-WGP-NEXT: s_endpgm 1088; 1089; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw: 1090; GFX10-CU: ; %bb.0: ; %entry 1091; GFX10-CU-NEXT: s_clause 0x1 1092; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1093; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1094; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1095; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1096; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1097; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1098; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1099; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 1100; GFX10-CU-NEXT: s_endpgm 1101; 1102; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw: 1103; SKIP-CACHE-INV: ; %bb.0: ; %entry 1104; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1105; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1106; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1107; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1108; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1109; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1110; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1111; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 1112; SKIP-CACHE-INV-NEXT: s_endpgm 1113; 1114; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw: 1115; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1116; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1117; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1118; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1119; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1120; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1121; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1122; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1123; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1124; 1125; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw: 1126; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1127; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1128; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1129; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1130; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1131; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1132; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1133; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1134; GFX90A-TGSPLIT-NEXT: s_endpgm 1135; 1136; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw: 1137; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1138; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1139; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1140; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1141; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1142; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1143; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1144; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1145; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1146; 1147; GFX940-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw: 1148; GFX940-TGSPLIT: ; %bb.0: ; %entry 1149; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1150; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1151; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1152; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1153; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1154; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1155; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1156; GFX940-TGSPLIT-NEXT: s_endpgm 1157 i32* %out, i32 %in) { 1158entry: 1159 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release 1160 ret void 1161} 1162 1163define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw( 1164; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw: 1165; GFX7: ; %bb.0: ; %entry 1166; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1167; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1168; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1169; GFX7-NEXT: v_mov_b32_e32 v0, s0 1170; GFX7-NEXT: v_mov_b32_e32 v1, s1 1171; GFX7-NEXT: v_mov_b32_e32 v2, s2 1172; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1173; GFX7-NEXT: flat_atomic_swap v[0:1], v2 1174; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1175; GFX7-NEXT: s_endpgm 1176; 1177; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw: 1178; GFX10-WGP: ; %bb.0: ; %entry 1179; GFX10-WGP-NEXT: s_clause 0x1 1180; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1181; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1182; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1183; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1184; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1185; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1186; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1187; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1188; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 1189; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1190; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1191; GFX10-WGP-NEXT: buffer_gl0_inv 1192; GFX10-WGP-NEXT: s_endpgm 1193; 1194; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw: 1195; GFX10-CU: ; %bb.0: ; %entry 1196; GFX10-CU-NEXT: s_clause 0x1 1197; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1198; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1199; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1200; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1201; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1202; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1203; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1204; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 1205; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1206; GFX10-CU-NEXT: s_endpgm 1207; 1208; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw: 1209; SKIP-CACHE-INV: ; %bb.0: ; %entry 1210; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1211; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1212; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1213; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1214; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1215; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1216; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1217; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 1218; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1219; SKIP-CACHE-INV-NEXT: s_endpgm 1220; 1221; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw: 1222; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1223; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1224; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1225; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1226; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1227; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1228; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1229; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1230; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1231; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1232; 1233; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw: 1234; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1235; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1236; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1237; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1238; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1239; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1240; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1241; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1242; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1243; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1244; GFX90A-TGSPLIT-NEXT: s_endpgm 1245; 1246; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw: 1247; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1248; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1249; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1250; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1251; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1252; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1253; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1254; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1255; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1256; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1257; 1258; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw: 1259; GFX940-TGSPLIT: ; %bb.0: ; %entry 1260; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1261; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1262; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1263; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1264; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1265; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1266; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1267; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1268; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1269; GFX940-TGSPLIT-NEXT: s_endpgm 1270 i32* %out, i32 %in) { 1271entry: 1272 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel 1273 ret void 1274} 1275 1276define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw( 1277; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw: 1278; GFX7: ; %bb.0: ; %entry 1279; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1280; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1281; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1282; GFX7-NEXT: v_mov_b32_e32 v0, s0 1283; GFX7-NEXT: v_mov_b32_e32 v1, s1 1284; GFX7-NEXT: v_mov_b32_e32 v2, s2 1285; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1286; GFX7-NEXT: flat_atomic_swap v[0:1], v2 1287; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1288; GFX7-NEXT: s_endpgm 1289; 1290; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw: 1291; GFX10-WGP: ; %bb.0: ; %entry 1292; GFX10-WGP-NEXT: s_clause 0x1 1293; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1294; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1295; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1296; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1297; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1298; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1299; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1300; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1301; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 1302; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1303; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1304; GFX10-WGP-NEXT: buffer_gl0_inv 1305; GFX10-WGP-NEXT: s_endpgm 1306; 1307; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw: 1308; GFX10-CU: ; %bb.0: ; %entry 1309; GFX10-CU-NEXT: s_clause 0x1 1310; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1311; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1312; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1313; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1314; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1315; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1316; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1317; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 1318; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1319; GFX10-CU-NEXT: s_endpgm 1320; 1321; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw: 1322; SKIP-CACHE-INV: ; %bb.0: ; %entry 1323; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1324; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1325; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1326; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1327; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1328; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1329; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1330; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 1331; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1332; SKIP-CACHE-INV-NEXT: s_endpgm 1333; 1334; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw: 1335; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1336; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1337; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1338; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1339; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1340; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1341; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1342; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1343; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1344; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1345; 1346; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw: 1347; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1348; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1349; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1350; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1351; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1352; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1353; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1354; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1355; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1356; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1357; GFX90A-TGSPLIT-NEXT: s_endpgm 1358; 1359; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw: 1360; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1361; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1362; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1363; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1364; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1365; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1366; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1367; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1368; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1369; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1370; 1371; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw: 1372; GFX940-TGSPLIT: ; %bb.0: ; %entry 1373; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1374; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1375; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1376; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1377; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1378; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1379; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 1380; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1381; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1382; GFX940-TGSPLIT-NEXT: s_endpgm 1383 i32* %out, i32 %in) { 1384entry: 1385 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst 1386 ret void 1387} 1388 1389define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw( 1390; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1391; GFX7: ; %bb.0: ; %entry 1392; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1393; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1394; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1395; GFX7-NEXT: v_mov_b32_e32 v0, s0 1396; GFX7-NEXT: v_mov_b32_e32 v1, s1 1397; GFX7-NEXT: v_mov_b32_e32 v2, s2 1398; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1399; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1400; GFX7-NEXT: flat_store_dword v[0:1], v2 1401; GFX7-NEXT: s_endpgm 1402; 1403; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1404; GFX10-WGP: ; %bb.0: ; %entry 1405; GFX10-WGP-NEXT: s_clause 0x1 1406; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1407; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1408; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1409; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1410; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1411; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1412; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1413; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1414; GFX10-WGP-NEXT: buffer_gl0_inv 1415; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 1416; GFX10-WGP-NEXT: s_endpgm 1417; 1418; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1419; GFX10-CU: ; %bb.0: ; %entry 1420; GFX10-CU-NEXT: s_clause 0x1 1421; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1422; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1423; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1424; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1425; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1426; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1427; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1428; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1429; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 1430; GFX10-CU-NEXT: s_endpgm 1431; 1432; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1433; SKIP-CACHE-INV: ; %bb.0: ; %entry 1434; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1435; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1436; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1437; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1438; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1439; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1440; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1441; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1442; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 1443; SKIP-CACHE-INV-NEXT: s_endpgm 1444; 1445; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1446; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1447; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1448; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1449; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1450; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1451; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1452; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1453; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1454; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1455; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1456; 1457; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1458; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1459; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1460; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1461; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1462; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1463; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1464; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1465; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1466; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1467; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1468; GFX90A-TGSPLIT-NEXT: s_endpgm 1469; 1470; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1471; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1472; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1473; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1474; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1475; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1476; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1477; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1478; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1479; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1480; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1481; 1482; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw: 1483; GFX940-TGSPLIT: ; %bb.0: ; %entry 1484; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1485; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1486; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1487; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1488; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1489; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1490; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1491; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1492; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1493; GFX940-TGSPLIT-NEXT: s_endpgm 1494 i32* %out, i32 %in) { 1495entry: 1496 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire 1497 store i32 %val, i32* %out, align 4 1498 ret void 1499} 1500 1501define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw( 1502; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1503; GFX7: ; %bb.0: ; %entry 1504; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1505; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1506; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1507; GFX7-NEXT: v_mov_b32_e32 v0, s0 1508; GFX7-NEXT: v_mov_b32_e32 v1, s1 1509; GFX7-NEXT: v_mov_b32_e32 v2, s2 1510; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1511; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1512; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1513; GFX7-NEXT: flat_store_dword v[0:1], v2 1514; GFX7-NEXT: s_endpgm 1515; 1516; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1517; GFX10-WGP: ; %bb.0: ; %entry 1518; GFX10-WGP-NEXT: s_clause 0x1 1519; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1520; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1521; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1522; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1523; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1524; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1525; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1526; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1527; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1528; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1529; GFX10-WGP-NEXT: buffer_gl0_inv 1530; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 1531; GFX10-WGP-NEXT: s_endpgm 1532; 1533; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1534; GFX10-CU: ; %bb.0: ; %entry 1535; GFX10-CU-NEXT: s_clause 0x1 1536; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1537; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1538; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1539; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1540; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1541; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1542; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1543; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1544; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1545; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 1546; GFX10-CU-NEXT: s_endpgm 1547; 1548; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1549; SKIP-CACHE-INV: ; %bb.0: ; %entry 1550; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1551; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1552; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1553; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1554; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1555; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1556; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1557; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1558; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1559; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 1560; SKIP-CACHE-INV-NEXT: s_endpgm 1561; 1562; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1563; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1564; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1565; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1566; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1567; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1568; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1569; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1570; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1571; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1572; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1573; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1574; 1575; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1576; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1577; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1578; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1579; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1580; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1581; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1582; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1583; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1584; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1585; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1586; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1587; GFX90A-TGSPLIT-NEXT: s_endpgm 1588; 1589; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1590; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1591; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1592; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1593; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1594; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1595; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1596; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1597; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1598; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1599; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1600; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1601; 1602; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw: 1603; GFX940-TGSPLIT: ; %bb.0: ; %entry 1604; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1605; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1606; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1607; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1608; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1609; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1610; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1611; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1612; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1613; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1614; GFX940-TGSPLIT-NEXT: s_endpgm 1615 i32* %out, i32 %in) { 1616entry: 1617 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel 1618 store i32 %val, i32* %out, align 4 1619 ret void 1620} 1621 1622define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw( 1623; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1624; GFX7: ; %bb.0: ; %entry 1625; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1626; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 1627; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1628; GFX7-NEXT: v_mov_b32_e32 v0, s0 1629; GFX7-NEXT: v_mov_b32_e32 v1, s1 1630; GFX7-NEXT: v_mov_b32_e32 v2, s2 1631; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1632; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1633; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1634; GFX7-NEXT: flat_store_dword v[0:1], v2 1635; GFX7-NEXT: s_endpgm 1636; 1637; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1638; GFX10-WGP: ; %bb.0: ; %entry 1639; GFX10-WGP-NEXT: s_clause 0x1 1640; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1641; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 1642; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1643; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1644; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1645; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1646; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1647; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1648; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1649; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1650; GFX10-WGP-NEXT: buffer_gl0_inv 1651; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 1652; GFX10-WGP-NEXT: s_endpgm 1653; 1654; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1655; GFX10-CU: ; %bb.0: ; %entry 1656; GFX10-CU-NEXT: s_clause 0x1 1657; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1658; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 1659; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1660; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1661; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1662; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1663; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1664; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1665; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1666; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 1667; GFX10-CU-NEXT: s_endpgm 1668; 1669; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1670; SKIP-CACHE-INV: ; %bb.0: ; %entry 1671; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1672; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 1673; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1674; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1675; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1676; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1677; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1678; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1679; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1680; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 1681; SKIP-CACHE-INV-NEXT: s_endpgm 1682; 1683; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1684; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1685; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1686; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1687; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1688; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1689; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1690; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1691; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1692; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1693; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1694; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1695; 1696; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1697; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1698; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1699; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 1700; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1701; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1702; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 1703; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1704; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 1705; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1706; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1707; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1708; GFX90A-TGSPLIT-NEXT: s_endpgm 1709; 1710; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1711; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1712; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1713; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1714; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1715; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1716; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1717; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1718; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1719; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1720; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 1721; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1722; 1723; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw: 1724; GFX940-TGSPLIT: ; %bb.0: ; %entry 1725; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1726; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 1727; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1728; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1729; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 1730; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1731; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 1732; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1733; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1734; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 1735; GFX940-TGSPLIT-NEXT: s_endpgm 1736 i32* %out, i32 %in) { 1737entry: 1738 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst 1739 store i32 %val, i32* %out, align 4 1740 ret void 1741} 1742 1743define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg( 1744; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1745; GFX7: ; %bb.0: ; %entry 1746; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1747; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 1748; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1749; GFX7-NEXT: s_add_u32 s0, s0, 16 1750; GFX7-NEXT: s_addc_u32 s1, s1, 0 1751; GFX7-NEXT: v_mov_b32_e32 v0, s0 1752; GFX7-NEXT: v_mov_b32_e32 v2, s2 1753; GFX7-NEXT: v_mov_b32_e32 v1, s1 1754; GFX7-NEXT: v_mov_b32_e32 v3, s3 1755; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1756; GFX7-NEXT: s_endpgm 1757; 1758; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1759; GFX10-WGP: ; %bb.0: ; %entry 1760; GFX10-WGP-NEXT: s_clause 0x1 1761; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1762; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1763; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1764; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 1765; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 1766; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1767; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1768; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1769; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 1770; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1771; GFX10-WGP-NEXT: s_endpgm 1772; 1773; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1774; GFX10-CU: ; %bb.0: ; %entry 1775; GFX10-CU-NEXT: s_clause 0x1 1776; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1777; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1778; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1779; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 1780; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 1781; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1782; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1783; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1784; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 1785; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1786; GFX10-CU-NEXT: s_endpgm 1787; 1788; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1789; SKIP-CACHE-INV: ; %bb.0: ; %entry 1790; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1791; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 1792; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1793; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 1794; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 1795; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1796; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 1797; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1798; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1799; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1800; SKIP-CACHE-INV-NEXT: s_endpgm 1801; 1802; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1803; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1804; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1805; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1806; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1807; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1808; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 1809; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1810; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1811; 1812; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1813; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1814; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1815; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1816; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1817; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1818; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 1819; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1820; GFX90A-TGSPLIT-NEXT: s_endpgm 1821; 1822; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1823; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1824; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1825; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 1826; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1827; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1828; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 1829; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1830; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1831; 1832; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg: 1833; GFX940-TGSPLIT: ; %bb.0: ; %entry 1834; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1835; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 1836; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1837; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1838; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 1839; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1840; GFX940-TGSPLIT-NEXT: s_endpgm 1841 i32* %out, i32 %in, i32 %old) { 1842entry: 1843 %gep = getelementptr i32, i32* %out, i32 4 1844 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic 1845 ret void 1846} 1847 1848define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg( 1849; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1850; GFX7: ; %bb.0: ; %entry 1851; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1852; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 1853; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1854; GFX7-NEXT: s_add_u32 s0, s0, 16 1855; GFX7-NEXT: s_addc_u32 s1, s1, 0 1856; GFX7-NEXT: v_mov_b32_e32 v0, s0 1857; GFX7-NEXT: v_mov_b32_e32 v2, s2 1858; GFX7-NEXT: v_mov_b32_e32 v1, s1 1859; GFX7-NEXT: v_mov_b32_e32 v3, s3 1860; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1861; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1862; GFX7-NEXT: s_endpgm 1863; 1864; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1865; GFX10-WGP: ; %bb.0: ; %entry 1866; GFX10-WGP-NEXT: s_clause 0x1 1867; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1868; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1869; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1870; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 1871; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 1872; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1873; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1874; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1875; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 1876; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1877; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1878; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1879; GFX10-WGP-NEXT: buffer_gl0_inv 1880; GFX10-WGP-NEXT: s_endpgm 1881; 1882; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1883; GFX10-CU: ; %bb.0: ; %entry 1884; GFX10-CU-NEXT: s_clause 0x1 1885; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1886; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1887; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1888; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 1889; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 1890; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 1891; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 1892; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 1893; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 1894; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1895; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 1896; GFX10-CU-NEXT: s_endpgm 1897; 1898; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1899; SKIP-CACHE-INV: ; %bb.0: ; %entry 1900; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1901; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 1902; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1903; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 1904; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 1905; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 1906; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 1907; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 1908; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 1909; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1910; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 1911; SKIP-CACHE-INV-NEXT: s_endpgm 1912; 1913; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1914; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 1915; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1916; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1917; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1918; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1919; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 1920; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1921; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1922; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 1923; 1924; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1925; GFX90A-TGSPLIT: ; %bb.0: ; %entry 1926; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1927; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1928; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1929; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 1930; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 1931; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1932; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1933; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 1934; GFX90A-TGSPLIT-NEXT: s_endpgm 1935; 1936; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1937; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 1938; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1939; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 1940; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1941; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1942; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 1943; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1944; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1945; GFX940-NOTTGSPLIT-NEXT: s_endpgm 1946; 1947; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg: 1948; GFX940-TGSPLIT: ; %bb.0: ; %entry 1949; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 1950; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 1951; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 1952; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 1953; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 1954; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 1955; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 1956; GFX940-TGSPLIT-NEXT: buffer_inv sc0 1957; GFX940-TGSPLIT-NEXT: s_endpgm 1958 i32* %out, i32 %in, i32 %old) { 1959entry: 1960 %gep = getelementptr i32, i32* %out, i32 4 1961 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic 1962 ret void 1963} 1964 1965define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg( 1966; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg: 1967; GFX7: ; %bb.0: ; %entry 1968; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1969; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 1970; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1971; GFX7-NEXT: s_add_u32 s0, s0, 16 1972; GFX7-NEXT: s_addc_u32 s1, s1, 0 1973; GFX7-NEXT: v_mov_b32_e32 v0, s0 1974; GFX7-NEXT: v_mov_b32_e32 v2, s2 1975; GFX7-NEXT: v_mov_b32_e32 v1, s1 1976; GFX7-NEXT: v_mov_b32_e32 v3, s3 1977; GFX7-NEXT: s_waitcnt lgkmcnt(0) 1978; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1979; GFX7-NEXT: s_endpgm 1980; 1981; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg: 1982; GFX10-WGP: ; %bb.0: ; %entry 1983; GFX10-WGP-NEXT: s_clause 0x1 1984; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 1985; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 1986; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 1987; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 1988; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 1989; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 1990; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 1991; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 1992; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 1993; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 1994; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 1995; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 1996; GFX10-WGP-NEXT: s_endpgm 1997; 1998; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg: 1999; GFX10-CU: ; %bb.0: ; %entry 2000; GFX10-CU-NEXT: s_clause 0x1 2001; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2002; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2003; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2004; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2005; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2006; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2007; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2008; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2009; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2010; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2011; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2012; GFX10-CU-NEXT: s_endpgm 2013; 2014; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg: 2015; SKIP-CACHE-INV: ; %bb.0: ; %entry 2016; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2017; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2018; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2019; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2020; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2021; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2022; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2023; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2024; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2025; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2026; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2027; SKIP-CACHE-INV-NEXT: s_endpgm 2028; 2029; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg: 2030; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2031; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2032; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2033; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2034; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2035; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2036; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2037; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2038; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2039; 2040; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg: 2041; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2042; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2043; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2044; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2045; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2046; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2047; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2048; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2049; GFX90A-TGSPLIT-NEXT: s_endpgm 2050; 2051; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg: 2052; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2053; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2054; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2055; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2056; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2057; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2058; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2059; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2060; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2061; 2062; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg: 2063; GFX940-TGSPLIT: ; %bb.0: ; %entry 2064; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2065; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2066; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2067; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2068; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2069; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2070; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2071; GFX940-TGSPLIT-NEXT: s_endpgm 2072 i32* %out, i32 %in, i32 %old) { 2073entry: 2074 %gep = getelementptr i32, i32* %out, i32 4 2075 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic 2076 ret void 2077} 2078 2079define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg( 2080; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2081; GFX7: ; %bb.0: ; %entry 2082; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2083; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2084; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2085; GFX7-NEXT: s_add_u32 s0, s0, 16 2086; GFX7-NEXT: s_addc_u32 s1, s1, 0 2087; GFX7-NEXT: v_mov_b32_e32 v0, s0 2088; GFX7-NEXT: v_mov_b32_e32 v2, s2 2089; GFX7-NEXT: v_mov_b32_e32 v1, s1 2090; GFX7-NEXT: v_mov_b32_e32 v3, s3 2091; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2092; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2093; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2094; GFX7-NEXT: s_endpgm 2095; 2096; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2097; GFX10-WGP: ; %bb.0: ; %entry 2098; GFX10-WGP-NEXT: s_clause 0x1 2099; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2100; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2101; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2102; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2103; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2104; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2105; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2106; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2107; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2108; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2109; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2110; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2111; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2112; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2113; GFX10-WGP-NEXT: buffer_gl0_inv 2114; GFX10-WGP-NEXT: s_endpgm 2115; 2116; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2117; GFX10-CU: ; %bb.0: ; %entry 2118; GFX10-CU-NEXT: s_clause 0x1 2119; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2120; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2121; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2122; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2123; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2124; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2125; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2126; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2127; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2128; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2129; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2130; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2131; GFX10-CU-NEXT: s_endpgm 2132; 2133; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2134; SKIP-CACHE-INV: ; %bb.0: ; %entry 2135; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2136; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2137; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2138; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2139; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2140; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2141; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2142; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2143; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2144; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2145; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2146; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2147; SKIP-CACHE-INV-NEXT: s_endpgm 2148; 2149; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2150; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2151; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2152; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2153; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2154; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2155; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2156; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2157; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2158; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2159; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2160; 2161; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2162; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2163; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2164; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2165; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2166; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2167; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2168; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2169; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2170; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2171; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2172; GFX90A-TGSPLIT-NEXT: s_endpgm 2173; 2174; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2175; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2176; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2177; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2178; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2179; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2180; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2181; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2182; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2183; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2184; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2185; 2186; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg: 2187; GFX940-TGSPLIT: ; %bb.0: ; %entry 2188; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2189; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2190; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2191; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2192; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2193; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2194; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2195; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2196; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2197; GFX940-TGSPLIT-NEXT: s_endpgm 2198 i32* %out, i32 %in, i32 %old) { 2199entry: 2200 %gep = getelementptr i32, i32* %out, i32 4 2201 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic 2202 ret void 2203} 2204 2205define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg( 2206; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2207; GFX7: ; %bb.0: ; %entry 2208; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2209; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2210; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2211; GFX7-NEXT: s_add_u32 s0, s0, 16 2212; GFX7-NEXT: s_addc_u32 s1, s1, 0 2213; GFX7-NEXT: v_mov_b32_e32 v0, s0 2214; GFX7-NEXT: v_mov_b32_e32 v2, s2 2215; GFX7-NEXT: v_mov_b32_e32 v1, s1 2216; GFX7-NEXT: v_mov_b32_e32 v3, s3 2217; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2218; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2219; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2220; GFX7-NEXT: s_endpgm 2221; 2222; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2223; GFX10-WGP: ; %bb.0: ; %entry 2224; GFX10-WGP-NEXT: s_clause 0x1 2225; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2226; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2227; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2228; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2229; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2230; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2231; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2232; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2233; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2234; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2235; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2236; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2237; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2238; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2239; GFX10-WGP-NEXT: buffer_gl0_inv 2240; GFX10-WGP-NEXT: s_endpgm 2241; 2242; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2243; GFX10-CU: ; %bb.0: ; %entry 2244; GFX10-CU-NEXT: s_clause 0x1 2245; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2246; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2247; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2248; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2249; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2250; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2251; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2252; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2253; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2254; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2255; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2256; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2257; GFX10-CU-NEXT: s_endpgm 2258; 2259; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2260; SKIP-CACHE-INV: ; %bb.0: ; %entry 2261; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2262; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2263; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2264; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2265; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2266; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2267; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2268; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2269; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2270; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2271; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2272; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2273; SKIP-CACHE-INV-NEXT: s_endpgm 2274; 2275; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2276; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2277; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2278; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2279; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2280; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2281; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2282; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2283; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2284; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2285; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2286; 2287; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2288; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2289; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2290; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2291; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2292; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2293; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2294; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2295; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2296; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2297; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2298; GFX90A-TGSPLIT-NEXT: s_endpgm 2299; 2300; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2301; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2302; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2303; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2304; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2305; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2306; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2307; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2308; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2309; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2310; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2311; 2312; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg: 2313; GFX940-TGSPLIT: ; %bb.0: ; %entry 2314; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2315; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2316; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2317; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2318; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2319; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2320; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2321; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2322; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2323; GFX940-TGSPLIT-NEXT: s_endpgm 2324 i32* %out, i32 %in, i32 %old) { 2325entry: 2326 %gep = getelementptr i32, i32* %out, i32 4 2327 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic 2328 ret void 2329} 2330 2331define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg( 2332; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2333; GFX7: ; %bb.0: ; %entry 2334; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2335; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2336; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2337; GFX7-NEXT: s_add_u32 s0, s0, 16 2338; GFX7-NEXT: s_addc_u32 s1, s1, 0 2339; GFX7-NEXT: v_mov_b32_e32 v0, s0 2340; GFX7-NEXT: v_mov_b32_e32 v2, s2 2341; GFX7-NEXT: v_mov_b32_e32 v1, s1 2342; GFX7-NEXT: v_mov_b32_e32 v3, s3 2343; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2344; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2345; GFX7-NEXT: s_endpgm 2346; 2347; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2348; GFX10-WGP: ; %bb.0: ; %entry 2349; GFX10-WGP-NEXT: s_clause 0x1 2350; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2351; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2352; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2353; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2354; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2355; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2356; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2357; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2358; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2359; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2360; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2361; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2362; GFX10-WGP-NEXT: buffer_gl0_inv 2363; GFX10-WGP-NEXT: s_endpgm 2364; 2365; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2366; GFX10-CU: ; %bb.0: ; %entry 2367; GFX10-CU-NEXT: s_clause 0x1 2368; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2369; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2370; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2371; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2372; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2373; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2374; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2375; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2376; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2377; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2378; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2379; GFX10-CU-NEXT: s_endpgm 2380; 2381; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2382; SKIP-CACHE-INV: ; %bb.0: ; %entry 2383; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2384; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2385; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2386; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2387; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2388; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2389; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2390; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2391; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2392; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2393; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2394; SKIP-CACHE-INV-NEXT: s_endpgm 2395; 2396; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2397; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2398; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2399; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2400; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2401; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2402; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2403; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2404; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2405; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2406; 2407; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2408; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2409; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2410; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2411; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2412; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2413; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2414; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2415; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2416; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2417; GFX90A-TGSPLIT-NEXT: s_endpgm 2418; 2419; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2420; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2421; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2422; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2423; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2424; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2425; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2426; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2427; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2428; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2429; 2430; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg: 2431; GFX940-TGSPLIT: ; %bb.0: ; %entry 2432; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2433; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2434; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2435; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2436; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2437; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2438; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2439; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2440; GFX940-TGSPLIT-NEXT: s_endpgm 2441 i32* %out, i32 %in, i32 %old) { 2442entry: 2443 %gep = getelementptr i32, i32* %out, i32 4 2444 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire 2445 ret void 2446} 2447 2448define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg( 2449; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2450; GFX7: ; %bb.0: ; %entry 2451; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2452; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2453; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2454; GFX7-NEXT: s_add_u32 s0, s0, 16 2455; GFX7-NEXT: s_addc_u32 s1, s1, 0 2456; GFX7-NEXT: v_mov_b32_e32 v0, s0 2457; GFX7-NEXT: v_mov_b32_e32 v2, s2 2458; GFX7-NEXT: v_mov_b32_e32 v1, s1 2459; GFX7-NEXT: v_mov_b32_e32 v3, s3 2460; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2461; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2462; GFX7-NEXT: s_endpgm 2463; 2464; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2465; GFX10-WGP: ; %bb.0: ; %entry 2466; GFX10-WGP-NEXT: s_clause 0x1 2467; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2468; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2469; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2470; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2471; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2472; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2473; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2474; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2475; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2476; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2477; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2478; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2479; GFX10-WGP-NEXT: buffer_gl0_inv 2480; GFX10-WGP-NEXT: s_endpgm 2481; 2482; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2483; GFX10-CU: ; %bb.0: ; %entry 2484; GFX10-CU-NEXT: s_clause 0x1 2485; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2486; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2487; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2488; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2489; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2490; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2491; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2492; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2493; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2494; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2495; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2496; GFX10-CU-NEXT: s_endpgm 2497; 2498; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2499; SKIP-CACHE-INV: ; %bb.0: ; %entry 2500; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2501; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2502; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2503; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2504; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2505; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2506; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2507; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2508; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2509; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2510; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2511; SKIP-CACHE-INV-NEXT: s_endpgm 2512; 2513; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2514; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2515; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2516; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2517; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2518; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2519; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2520; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2521; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2522; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2523; 2524; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2525; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2526; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2527; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2528; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2529; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2530; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2531; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2532; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2533; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2534; GFX90A-TGSPLIT-NEXT: s_endpgm 2535; 2536; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2537; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2538; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2539; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2540; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2541; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2542; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2543; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2544; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2545; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2546; 2547; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg: 2548; GFX940-TGSPLIT: ; %bb.0: ; %entry 2549; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2550; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2551; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2552; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2553; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2554; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2555; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2556; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2557; GFX940-TGSPLIT-NEXT: s_endpgm 2558 i32* %out, i32 %in, i32 %old) { 2559entry: 2560 %gep = getelementptr i32, i32* %out, i32 4 2561 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire 2562 ret void 2563} 2564 2565define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg( 2566; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg: 2567; GFX7: ; %bb.0: ; %entry 2568; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2569; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2570; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2571; GFX7-NEXT: s_add_u32 s0, s0, 16 2572; GFX7-NEXT: s_addc_u32 s1, s1, 0 2573; GFX7-NEXT: v_mov_b32_e32 v0, s0 2574; GFX7-NEXT: v_mov_b32_e32 v2, s2 2575; GFX7-NEXT: v_mov_b32_e32 v1, s1 2576; GFX7-NEXT: v_mov_b32_e32 v3, s3 2577; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2578; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2579; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2580; GFX7-NEXT: s_endpgm 2581; 2582; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg: 2583; GFX10-WGP: ; %bb.0: ; %entry 2584; GFX10-WGP-NEXT: s_clause 0x1 2585; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2586; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2587; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2588; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2589; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2590; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2591; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2592; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2593; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2594; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2595; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2596; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2597; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2598; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2599; GFX10-WGP-NEXT: buffer_gl0_inv 2600; GFX10-WGP-NEXT: s_endpgm 2601; 2602; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg: 2603; GFX10-CU: ; %bb.0: ; %entry 2604; GFX10-CU-NEXT: s_clause 0x1 2605; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2606; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2607; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2608; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2609; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2610; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2611; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2612; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2613; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2614; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2615; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2616; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2617; GFX10-CU-NEXT: s_endpgm 2618; 2619; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg: 2620; SKIP-CACHE-INV: ; %bb.0: ; %entry 2621; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2622; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2623; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2624; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2625; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2626; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2627; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2628; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2629; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2630; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2631; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2632; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2633; SKIP-CACHE-INV-NEXT: s_endpgm 2634; 2635; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg: 2636; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2637; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2638; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2639; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2640; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2641; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2642; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2643; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2644; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2645; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2646; 2647; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg: 2648; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2649; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2650; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2651; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2652; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2653; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2654; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2655; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2656; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2657; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2658; GFX90A-TGSPLIT-NEXT: s_endpgm 2659; 2660; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg: 2661; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2662; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2663; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2664; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2665; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2666; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2667; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2668; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2669; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2670; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2671; 2672; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg: 2673; GFX940-TGSPLIT: ; %bb.0: ; %entry 2674; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2675; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2676; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2677; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2678; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2679; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2680; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2681; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2682; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2683; GFX940-TGSPLIT-NEXT: s_endpgm 2684 i32* %out, i32 %in, i32 %old) { 2685entry: 2686 %gep = getelementptr i32, i32* %out, i32 4 2687 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire 2688 ret void 2689} 2690 2691define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg( 2692; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2693; GFX7: ; %bb.0: ; %entry 2694; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2695; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2696; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2697; GFX7-NEXT: s_add_u32 s0, s0, 16 2698; GFX7-NEXT: s_addc_u32 s1, s1, 0 2699; GFX7-NEXT: v_mov_b32_e32 v0, s0 2700; GFX7-NEXT: v_mov_b32_e32 v2, s2 2701; GFX7-NEXT: v_mov_b32_e32 v1, s1 2702; GFX7-NEXT: v_mov_b32_e32 v3, s3 2703; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2704; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2705; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2706; GFX7-NEXT: s_endpgm 2707; 2708; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2709; GFX10-WGP: ; %bb.0: ; %entry 2710; GFX10-WGP-NEXT: s_clause 0x1 2711; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2712; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2713; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2714; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2715; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2716; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2717; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2718; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2719; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2720; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2721; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2722; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2723; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2724; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2725; GFX10-WGP-NEXT: buffer_gl0_inv 2726; GFX10-WGP-NEXT: s_endpgm 2727; 2728; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2729; GFX10-CU: ; %bb.0: ; %entry 2730; GFX10-CU-NEXT: s_clause 0x1 2731; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2732; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2733; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2734; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2735; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2736; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2737; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2738; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2739; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2740; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2741; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2742; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2743; GFX10-CU-NEXT: s_endpgm 2744; 2745; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2746; SKIP-CACHE-INV: ; %bb.0: ; %entry 2747; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2748; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2749; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2750; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2751; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2752; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2753; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2754; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2755; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2756; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2757; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2758; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2759; SKIP-CACHE-INV-NEXT: s_endpgm 2760; 2761; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2762; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2763; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2764; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2765; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2766; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2767; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2768; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2769; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2770; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2771; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2772; 2773; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2774; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2775; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2776; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2777; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2778; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2779; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2780; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2781; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2782; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2783; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2784; GFX90A-TGSPLIT-NEXT: s_endpgm 2785; 2786; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2787; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2788; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2789; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2790; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2791; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2792; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2793; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2794; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2795; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2796; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2797; 2798; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg: 2799; GFX940-TGSPLIT: ; %bb.0: ; %entry 2800; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2801; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2802; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2803; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2804; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2805; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2806; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2807; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2808; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2809; GFX940-TGSPLIT-NEXT: s_endpgm 2810 i32* %out, i32 %in, i32 %old) { 2811entry: 2812 %gep = getelementptr i32, i32* %out, i32 4 2813 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire 2814 ret void 2815} 2816 2817define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg( 2818; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2819; GFX7: ; %bb.0: ; %entry 2820; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2821; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2822; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2823; GFX7-NEXT: s_add_u32 s0, s0, 16 2824; GFX7-NEXT: s_addc_u32 s1, s1, 0 2825; GFX7-NEXT: v_mov_b32_e32 v0, s0 2826; GFX7-NEXT: v_mov_b32_e32 v2, s2 2827; GFX7-NEXT: v_mov_b32_e32 v1, s1 2828; GFX7-NEXT: v_mov_b32_e32 v3, s3 2829; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2830; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2831; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2832; GFX7-NEXT: s_endpgm 2833; 2834; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2835; GFX10-WGP: ; %bb.0: ; %entry 2836; GFX10-WGP-NEXT: s_clause 0x1 2837; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2838; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2839; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2840; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2841; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2842; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2843; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2844; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2845; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2846; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2847; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2848; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2849; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2850; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2851; GFX10-WGP-NEXT: buffer_gl0_inv 2852; GFX10-WGP-NEXT: s_endpgm 2853; 2854; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2855; GFX10-CU: ; %bb.0: ; %entry 2856; GFX10-CU-NEXT: s_clause 0x1 2857; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2858; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2859; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2860; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2861; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2862; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2863; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2864; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2865; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2866; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2867; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2868; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2869; GFX10-CU-NEXT: s_endpgm 2870; 2871; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2872; SKIP-CACHE-INV: ; %bb.0: ; %entry 2873; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2874; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 2875; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2876; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 2877; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 2878; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 2879; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 2880; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 2881; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 2882; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2883; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2884; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 2885; SKIP-CACHE-INV-NEXT: s_endpgm 2886; 2887; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2888; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 2889; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2890; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2891; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2892; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2893; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2894; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2895; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2896; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2897; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 2898; 2899; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2900; GFX90A-TGSPLIT: ; %bb.0: ; %entry 2901; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2902; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2903; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2904; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 2905; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 2906; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2907; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2908; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2909; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 2910; GFX90A-TGSPLIT-NEXT: s_endpgm 2911; 2912; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2913; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 2914; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2915; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2916; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2917; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2918; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2919; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2920; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2921; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2922; GFX940-NOTTGSPLIT-NEXT: s_endpgm 2923; 2924; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg: 2925; GFX940-TGSPLIT: ; %bb.0: ; %entry 2926; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 2927; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 2928; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 2929; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 2930; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 2931; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2932; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 2933; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 2934; GFX940-TGSPLIT-NEXT: buffer_inv sc0 2935; GFX940-TGSPLIT-NEXT: s_endpgm 2936 i32* %out, i32 %in, i32 %old) { 2937entry: 2938 %gep = getelementptr i32, i32* %out, i32 4 2939 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire 2940 ret void 2941} 2942 2943define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg( 2944; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 2945; GFX7: ; %bb.0: ; %entry 2946; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2947; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 2948; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2949; GFX7-NEXT: s_add_u32 s0, s0, 16 2950; GFX7-NEXT: s_addc_u32 s1, s1, 0 2951; GFX7-NEXT: v_mov_b32_e32 v0, s0 2952; GFX7-NEXT: v_mov_b32_e32 v2, s2 2953; GFX7-NEXT: v_mov_b32_e32 v1, s1 2954; GFX7-NEXT: v_mov_b32_e32 v3, s3 2955; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2956; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2957; GFX7-NEXT: s_waitcnt lgkmcnt(0) 2958; GFX7-NEXT: s_endpgm 2959; 2960; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 2961; GFX10-WGP: ; %bb.0: ; %entry 2962; GFX10-WGP-NEXT: s_clause 0x1 2963; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2964; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2965; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2966; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 2967; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 2968; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 2969; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 2970; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 2971; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 2972; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 2973; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2974; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2975; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 2976; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 2977; GFX10-WGP-NEXT: buffer_gl0_inv 2978; GFX10-WGP-NEXT: s_endpgm 2979; 2980; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 2981; GFX10-CU: ; %bb.0: ; %entry 2982; GFX10-CU-NEXT: s_clause 0x1 2983; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 2984; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 2985; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2986; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 2987; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 2988; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 2989; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 2990; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 2991; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 2992; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2993; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 2994; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 2995; GFX10-CU-NEXT: s_endpgm 2996; 2997; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 2998; SKIP-CACHE-INV: ; %bb.0: ; %entry 2999; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3000; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3001; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3002; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 3003; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 3004; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3005; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3006; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3007; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3008; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3009; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 3010; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3011; SKIP-CACHE-INV-NEXT: s_endpgm 3012; 3013; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 3014; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3015; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3016; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3017; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3018; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3019; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3020; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3021; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 3022; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3023; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3024; 3025; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 3026; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3027; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3028; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3029; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3030; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3031; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3032; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3033; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 3034; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3035; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 3036; GFX90A-TGSPLIT-NEXT: s_endpgm 3037; 3038; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 3039; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3040; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3041; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3042; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3043; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3044; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3045; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3046; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 3047; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3048; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3049; 3050; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg: 3051; GFX940-TGSPLIT: ; %bb.0: ; %entry 3052; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3053; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3054; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3055; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3056; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3057; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3058; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 3059; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3060; GFX940-TGSPLIT-NEXT: buffer_inv sc0 3061; GFX940-TGSPLIT-NEXT: s_endpgm 3062 i32* %out, i32 %in, i32 %old) { 3063entry: 3064 %gep = getelementptr i32, i32* %out, i32 4 3065 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst 3066 ret void 3067} 3068 3069define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg( 3070; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3071; GFX7: ; %bb.0: ; %entry 3072; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3073; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3074; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3075; GFX7-NEXT: s_add_u32 s4, s0, 16 3076; GFX7-NEXT: s_addc_u32 s5, s1, 0 3077; GFX7-NEXT: v_mov_b32_e32 v0, s4 3078; GFX7-NEXT: v_mov_b32_e32 v2, s2 3079; GFX7-NEXT: v_mov_b32_e32 v1, s5 3080; GFX7-NEXT: v_mov_b32_e32 v3, s3 3081; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3082; GFX7-NEXT: v_mov_b32_e32 v0, s0 3083; GFX7-NEXT: v_mov_b32_e32 v1, s1 3084; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3085; GFX7-NEXT: flat_store_dword v[0:1], v2 3086; GFX7-NEXT: s_endpgm 3087; 3088; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3089; GFX10-WGP: ; %bb.0: ; %entry 3090; GFX10-WGP-NEXT: s_clause 0x1 3091; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3092; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3093; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3094; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3095; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3096; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3097; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3098; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3099; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3100; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3101; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3102; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3103; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3104; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3105; GFX10-WGP-NEXT: s_endpgm 3106; 3107; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3108; GFX10-CU: ; %bb.0: ; %entry 3109; GFX10-CU-NEXT: s_clause 0x1 3110; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3111; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3112; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3113; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3114; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3115; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3116; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3117; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3118; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3119; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3120; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3121; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3122; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3123; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3124; GFX10-CU-NEXT: s_endpgm 3125; 3126; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3127; SKIP-CACHE-INV: ; %bb.0: ; %entry 3128; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3129; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3130; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3131; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3132; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3133; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3134; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3135; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3136; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3137; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3138; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3139; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3140; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3141; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3142; SKIP-CACHE-INV-NEXT: s_endpgm 3143; 3144; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3145; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3146; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3147; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3148; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3149; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3150; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3151; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3152; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3153; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3154; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3155; 3156; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3157; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3158; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3159; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3160; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3161; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3162; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3163; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3164; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3165; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3166; GFX90A-TGSPLIT-NEXT: s_endpgm 3167; 3168; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3169; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3170; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3171; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3172; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3173; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3174; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3175; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3176; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3177; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3178; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3179; 3180; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg: 3181; GFX940-TGSPLIT: ; %bb.0: ; %entry 3182; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3183; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3184; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3185; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3186; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3187; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3188; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3189; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3190; GFX940-TGSPLIT-NEXT: s_endpgm 3191 i32* %out, i32 %in, i32 %old) { 3192entry: 3193 %gep = getelementptr i32, i32* %out, i32 4 3194 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic 3195 %val0 = extractvalue { i32, i1 } %val, 0 3196 store i32 %val0, i32* %out, align 4 3197 ret void 3198} 3199 3200define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg( 3201; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3202; GFX7: ; %bb.0: ; %entry 3203; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3204; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3205; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3206; GFX7-NEXT: s_add_u32 s4, s0, 16 3207; GFX7-NEXT: s_addc_u32 s5, s1, 0 3208; GFX7-NEXT: v_mov_b32_e32 v0, s4 3209; GFX7-NEXT: v_mov_b32_e32 v2, s2 3210; GFX7-NEXT: v_mov_b32_e32 v1, s5 3211; GFX7-NEXT: v_mov_b32_e32 v3, s3 3212; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3213; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3214; GFX7-NEXT: v_mov_b32_e32 v0, s0 3215; GFX7-NEXT: v_mov_b32_e32 v1, s1 3216; GFX7-NEXT: s_waitcnt vmcnt(0) 3217; GFX7-NEXT: flat_store_dword v[0:1], v2 3218; GFX7-NEXT: s_endpgm 3219; 3220; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3221; GFX10-WGP: ; %bb.0: ; %entry 3222; GFX10-WGP-NEXT: s_clause 0x1 3223; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3224; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3225; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3226; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3227; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3228; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3229; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3230; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3231; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3232; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3233; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3234; GFX10-WGP-NEXT: buffer_gl0_inv 3235; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3236; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3237; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3238; GFX10-WGP-NEXT: s_endpgm 3239; 3240; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3241; GFX10-CU: ; %bb.0: ; %entry 3242; GFX10-CU-NEXT: s_clause 0x1 3243; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3244; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3245; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3246; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3247; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3248; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3249; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3250; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3251; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3252; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3253; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3254; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3255; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3256; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 3257; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3258; GFX10-CU-NEXT: s_endpgm 3259; 3260; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3261; SKIP-CACHE-INV: ; %bb.0: ; %entry 3262; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3263; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3264; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3265; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3266; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3267; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3268; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3269; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3270; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3271; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3272; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3273; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3274; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3275; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 3276; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3277; SKIP-CACHE-INV-NEXT: s_endpgm 3278; 3279; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3280; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3281; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3282; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3283; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3284; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3285; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3286; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3287; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3288; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3289; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3290; 3291; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3292; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3293; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3294; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3295; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3296; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3297; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3298; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3299; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3300; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 3301; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3302; GFX90A-TGSPLIT-NEXT: s_endpgm 3303; 3304; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3305; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3306; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3307; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3308; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3309; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3310; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3311; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3312; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3313; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3314; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3315; 3316; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg: 3317; GFX940-TGSPLIT: ; %bb.0: ; %entry 3318; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3319; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3320; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3321; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3322; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3323; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3324; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3325; GFX940-TGSPLIT-NEXT: buffer_inv sc0 3326; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3327; GFX940-TGSPLIT-NEXT: s_endpgm 3328 i32* %out, i32 %in, i32 %old) { 3329entry: 3330 %gep = getelementptr i32, i32* %out, i32 4 3331 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic 3332 %val0 = extractvalue { i32, i1 } %val, 0 3333 store i32 %val0, i32* %out, align 4 3334 ret void 3335} 3336 3337define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg( 3338; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3339; GFX7: ; %bb.0: ; %entry 3340; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3341; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3342; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3343; GFX7-NEXT: s_add_u32 s4, s0, 16 3344; GFX7-NEXT: s_addc_u32 s5, s1, 0 3345; GFX7-NEXT: v_mov_b32_e32 v0, s4 3346; GFX7-NEXT: v_mov_b32_e32 v2, s2 3347; GFX7-NEXT: v_mov_b32_e32 v1, s5 3348; GFX7-NEXT: v_mov_b32_e32 v3, s3 3349; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3350; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3351; GFX7-NEXT: v_mov_b32_e32 v0, s0 3352; GFX7-NEXT: v_mov_b32_e32 v1, s1 3353; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3354; GFX7-NEXT: flat_store_dword v[0:1], v2 3355; GFX7-NEXT: s_endpgm 3356; 3357; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3358; GFX10-WGP: ; %bb.0: ; %entry 3359; GFX10-WGP-NEXT: s_clause 0x1 3360; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3361; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3362; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3363; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3364; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3365; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3366; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3367; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3368; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3369; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3370; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 3371; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3372; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3373; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3374; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3375; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3376; GFX10-WGP-NEXT: s_endpgm 3377; 3378; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3379; GFX10-CU: ; %bb.0: ; %entry 3380; GFX10-CU-NEXT: s_clause 0x1 3381; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3382; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3383; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3384; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3385; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3386; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3387; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3388; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3389; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3390; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3391; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3392; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3393; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3394; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3395; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3396; GFX10-CU-NEXT: s_endpgm 3397; 3398; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3399; SKIP-CACHE-INV: ; %bb.0: ; %entry 3400; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3401; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3402; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3403; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3404; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3405; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3406; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3407; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3408; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3409; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3410; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3411; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3412; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3413; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3414; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3415; SKIP-CACHE-INV-NEXT: s_endpgm 3416; 3417; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3418; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3419; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3420; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3421; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3422; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3423; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3424; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3425; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3426; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3427; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3428; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3429; 3430; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3431; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3432; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3433; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3434; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3435; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3436; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3437; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3438; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3439; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3440; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3441; GFX90A-TGSPLIT-NEXT: s_endpgm 3442; 3443; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3444; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3445; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3446; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3447; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3448; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3449; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3450; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3451; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3452; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3453; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3454; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3455; 3456; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg: 3457; GFX940-TGSPLIT: ; %bb.0: ; %entry 3458; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3459; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3460; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3461; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3462; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3463; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3464; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3465; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3466; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3467; GFX940-TGSPLIT-NEXT: s_endpgm 3468 i32* %out, i32 %in, i32 %old) { 3469entry: 3470 %gep = getelementptr i32, i32* %out, i32 4 3471 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic 3472 %val0 = extractvalue { i32, i1 } %val, 0 3473 store i32 %val0, i32* %out, align 4 3474 ret void 3475} 3476 3477define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg( 3478; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3479; GFX7: ; %bb.0: ; %entry 3480; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3481; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3482; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3483; GFX7-NEXT: s_add_u32 s4, s0, 16 3484; GFX7-NEXT: s_addc_u32 s5, s1, 0 3485; GFX7-NEXT: v_mov_b32_e32 v0, s4 3486; GFX7-NEXT: v_mov_b32_e32 v2, s2 3487; GFX7-NEXT: v_mov_b32_e32 v1, s5 3488; GFX7-NEXT: v_mov_b32_e32 v3, s3 3489; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3490; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3491; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3492; GFX7-NEXT: v_mov_b32_e32 v0, s0 3493; GFX7-NEXT: v_mov_b32_e32 v1, s1 3494; GFX7-NEXT: s_waitcnt vmcnt(0) 3495; GFX7-NEXT: flat_store_dword v[0:1], v2 3496; GFX7-NEXT: s_endpgm 3497; 3498; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3499; GFX10-WGP: ; %bb.0: ; %entry 3500; GFX10-WGP-NEXT: s_clause 0x1 3501; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3502; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3503; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3504; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3505; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3506; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3507; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3508; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3509; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3510; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3511; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 3512; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3513; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3514; GFX10-WGP-NEXT: buffer_gl0_inv 3515; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3516; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3517; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3518; GFX10-WGP-NEXT: s_endpgm 3519; 3520; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3521; GFX10-CU: ; %bb.0: ; %entry 3522; GFX10-CU-NEXT: s_clause 0x1 3523; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3524; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3525; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3526; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3527; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3528; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3529; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3530; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3531; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3532; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3533; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3534; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3535; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3536; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3537; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 3538; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3539; GFX10-CU-NEXT: s_endpgm 3540; 3541; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3542; SKIP-CACHE-INV: ; %bb.0: ; %entry 3543; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3544; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3545; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3546; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3547; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3548; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3549; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3550; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3551; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3552; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3553; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3554; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3555; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3556; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3557; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 3558; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3559; SKIP-CACHE-INV-NEXT: s_endpgm 3560; 3561; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3562; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3563; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3564; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3565; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3566; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3567; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3568; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3569; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3570; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3571; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3572; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3573; 3574; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3575; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3576; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3577; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3578; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3579; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3580; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3581; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3582; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3583; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3584; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 3585; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3586; GFX90A-TGSPLIT-NEXT: s_endpgm 3587; 3588; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3589; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3590; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3591; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3592; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3593; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3594; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3595; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3596; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3597; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3598; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3599; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3600; 3601; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg: 3602; GFX940-TGSPLIT: ; %bb.0: ; %entry 3603; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3604; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3605; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3606; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3607; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3608; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3609; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3610; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3611; GFX940-TGSPLIT-NEXT: buffer_inv sc0 3612; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3613; GFX940-TGSPLIT-NEXT: s_endpgm 3614 i32* %out, i32 %in, i32 %old) { 3615entry: 3616 %gep = getelementptr i32, i32* %out, i32 4 3617 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic 3618 %val0 = extractvalue { i32, i1 } %val, 0 3619 store i32 %val0, i32* %out, align 4 3620 ret void 3621} 3622 3623define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg( 3624; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3625; GFX7: ; %bb.0: ; %entry 3626; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3627; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3628; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3629; GFX7-NEXT: s_add_u32 s4, s0, 16 3630; GFX7-NEXT: s_addc_u32 s5, s1, 0 3631; GFX7-NEXT: v_mov_b32_e32 v0, s4 3632; GFX7-NEXT: v_mov_b32_e32 v2, s2 3633; GFX7-NEXT: v_mov_b32_e32 v1, s5 3634; GFX7-NEXT: v_mov_b32_e32 v3, s3 3635; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3636; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3637; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3638; GFX7-NEXT: v_mov_b32_e32 v0, s0 3639; GFX7-NEXT: v_mov_b32_e32 v1, s1 3640; GFX7-NEXT: s_waitcnt vmcnt(0) 3641; GFX7-NEXT: flat_store_dword v[0:1], v2 3642; GFX7-NEXT: s_endpgm 3643; 3644; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3645; GFX10-WGP: ; %bb.0: ; %entry 3646; GFX10-WGP-NEXT: s_clause 0x1 3647; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3648; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3649; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3650; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3651; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3652; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3653; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3654; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3655; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3656; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3657; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 3658; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3659; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3660; GFX10-WGP-NEXT: buffer_gl0_inv 3661; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3662; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3663; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3664; GFX10-WGP-NEXT: s_endpgm 3665; 3666; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3667; GFX10-CU: ; %bb.0: ; %entry 3668; GFX10-CU-NEXT: s_clause 0x1 3669; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3670; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3671; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3672; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3673; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3674; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3675; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3676; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3677; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3678; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3679; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3680; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3681; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3682; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3683; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 3684; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3685; GFX10-CU-NEXT: s_endpgm 3686; 3687; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3688; SKIP-CACHE-INV: ; %bb.0: ; %entry 3689; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3690; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3691; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3692; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3693; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3694; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3695; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3696; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3697; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3698; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3699; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3700; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3701; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3702; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3703; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 3704; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3705; SKIP-CACHE-INV-NEXT: s_endpgm 3706; 3707; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3708; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3709; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3710; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3711; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3712; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3713; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3714; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3715; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3716; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3717; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3718; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3719; 3720; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3721; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3722; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3723; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3724; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3725; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3726; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3727; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3728; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3729; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3730; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 3731; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3732; GFX90A-TGSPLIT-NEXT: s_endpgm 3733; 3734; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3735; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3736; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3737; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3738; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3739; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3740; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3741; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3742; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3743; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3744; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3745; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3746; 3747; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg: 3748; GFX940-TGSPLIT: ; %bb.0: ; %entry 3749; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3750; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3751; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3752; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3753; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3754; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3755; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3756; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3757; GFX940-TGSPLIT-NEXT: buffer_inv sc0 3758; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3759; GFX940-TGSPLIT-NEXT: s_endpgm 3760 i32* %out, i32 %in, i32 %old) { 3761entry: 3762 %gep = getelementptr i32, i32* %out, i32 4 3763 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic 3764 %val0 = extractvalue { i32, i1 } %val, 0 3765 store i32 %val0, i32* %out, align 4 3766 ret void 3767} 3768 3769define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg( 3770; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3771; GFX7: ; %bb.0: ; %entry 3772; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3773; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3774; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3775; GFX7-NEXT: s_add_u32 s4, s0, 16 3776; GFX7-NEXT: s_addc_u32 s5, s1, 0 3777; GFX7-NEXT: v_mov_b32_e32 v0, s4 3778; GFX7-NEXT: v_mov_b32_e32 v2, s2 3779; GFX7-NEXT: v_mov_b32_e32 v1, s5 3780; GFX7-NEXT: v_mov_b32_e32 v3, s3 3781; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3782; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3783; GFX7-NEXT: v_mov_b32_e32 v0, s0 3784; GFX7-NEXT: v_mov_b32_e32 v1, s1 3785; GFX7-NEXT: s_waitcnt vmcnt(0) 3786; GFX7-NEXT: flat_store_dword v[0:1], v2 3787; GFX7-NEXT: s_endpgm 3788; 3789; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3790; GFX10-WGP: ; %bb.0: ; %entry 3791; GFX10-WGP-NEXT: s_clause 0x1 3792; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3793; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3794; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3795; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3796; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3797; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3798; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3799; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3800; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3801; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3802; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3803; GFX10-WGP-NEXT: buffer_gl0_inv 3804; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3805; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3806; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3807; GFX10-WGP-NEXT: s_endpgm 3808; 3809; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3810; GFX10-CU: ; %bb.0: ; %entry 3811; GFX10-CU-NEXT: s_clause 0x1 3812; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3813; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3814; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3815; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3816; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3817; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3818; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3819; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3820; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3821; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3822; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3823; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3824; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3825; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 3826; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3827; GFX10-CU-NEXT: s_endpgm 3828; 3829; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3830; SKIP-CACHE-INV: ; %bb.0: ; %entry 3831; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3832; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3833; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3834; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3835; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3836; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3837; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3838; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3839; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3840; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3841; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3842; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3843; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3844; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 3845; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3846; SKIP-CACHE-INV-NEXT: s_endpgm 3847; 3848; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3849; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3850; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3851; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3852; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3853; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3854; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3855; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3856; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3857; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3858; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3859; 3860; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3861; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3862; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3863; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3864; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3865; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3866; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3867; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3868; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3869; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 3870; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3871; GFX90A-TGSPLIT-NEXT: s_endpgm 3872; 3873; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3874; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 3875; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3876; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3877; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3878; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3879; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3880; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3881; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3882; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3883; GFX940-NOTTGSPLIT-NEXT: s_endpgm 3884; 3885; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg: 3886; GFX940-TGSPLIT: ; %bb.0: ; %entry 3887; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3888; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 3889; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3890; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 3891; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 3892; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 3893; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 3894; GFX940-TGSPLIT-NEXT: buffer_inv sc0 3895; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 3896; GFX940-TGSPLIT-NEXT: s_endpgm 3897 i32* %out, i32 %in, i32 %old) { 3898entry: 3899 %gep = getelementptr i32, i32* %out, i32 4 3900 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire 3901 %val0 = extractvalue { i32, i1 } %val, 0 3902 store i32 %val0, i32* %out, align 4 3903 ret void 3904} 3905 3906define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg( 3907; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3908; GFX7: ; %bb.0: ; %entry 3909; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3910; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 3911; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3912; GFX7-NEXT: s_add_u32 s4, s0, 16 3913; GFX7-NEXT: s_addc_u32 s5, s1, 0 3914; GFX7-NEXT: v_mov_b32_e32 v0, s4 3915; GFX7-NEXT: v_mov_b32_e32 v2, s2 3916; GFX7-NEXT: v_mov_b32_e32 v1, s5 3917; GFX7-NEXT: v_mov_b32_e32 v3, s3 3918; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3919; GFX7-NEXT: s_waitcnt lgkmcnt(0) 3920; GFX7-NEXT: v_mov_b32_e32 v0, s0 3921; GFX7-NEXT: v_mov_b32_e32 v1, s1 3922; GFX7-NEXT: s_waitcnt vmcnt(0) 3923; GFX7-NEXT: flat_store_dword v[0:1], v2 3924; GFX7-NEXT: s_endpgm 3925; 3926; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3927; GFX10-WGP: ; %bb.0: ; %entry 3928; GFX10-WGP-NEXT: s_clause 0x1 3929; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3930; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3931; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 3932; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 3933; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 3934; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 3935; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 3936; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 3937; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 3938; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3939; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3940; GFX10-WGP-NEXT: buffer_gl0_inv 3941; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 3942; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 3943; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 3944; GFX10-WGP-NEXT: s_endpgm 3945; 3946; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3947; GFX10-CU: ; %bb.0: ; %entry 3948; GFX10-CU-NEXT: s_clause 0x1 3949; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3950; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3951; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3952; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 3953; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 3954; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 3955; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 3956; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 3957; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 3958; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3959; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 3960; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 3961; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 3962; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 3963; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 3964; GFX10-CU-NEXT: s_endpgm 3965; 3966; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3967; SKIP-CACHE-INV: ; %bb.0: ; %entry 3968; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 3969; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 3970; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3971; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 3972; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 3973; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 3974; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 3975; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 3976; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 3977; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 3978; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 3979; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 3980; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 3981; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 3982; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 3983; SKIP-CACHE-INV-NEXT: s_endpgm 3984; 3985; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3986; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 3987; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 3988; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 3989; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 3990; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 3991; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 3992; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 3993; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 3994; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 3995; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 3996; 3997; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 3998; GFX90A-TGSPLIT: ; %bb.0: ; %entry 3999; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4000; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4001; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4002; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4003; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4004; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4005; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4006; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4007; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4008; GFX90A-TGSPLIT-NEXT: s_endpgm 4009; 4010; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 4011; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4012; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4013; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4014; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4015; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4016; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4017; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4018; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4019; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4020; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4021; 4022; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg: 4023; GFX940-TGSPLIT: ; %bb.0: ; %entry 4024; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4025; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4026; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4027; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4028; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4029; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4030; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4031; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4032; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4033; GFX940-TGSPLIT-NEXT: s_endpgm 4034 i32* %out, i32 %in, i32 %old) { 4035entry: 4036 %gep = getelementptr i32, i32* %out, i32 4 4037 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire 4038 %val0 = extractvalue { i32, i1 } %val, 0 4039 store i32 %val0, i32* %out, align 4 4040 ret void 4041} 4042 4043define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg( 4044; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4045; GFX7: ; %bb.0: ; %entry 4046; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4047; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4048; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4049; GFX7-NEXT: s_add_u32 s4, s0, 16 4050; GFX7-NEXT: s_addc_u32 s5, s1, 0 4051; GFX7-NEXT: v_mov_b32_e32 v0, s4 4052; GFX7-NEXT: v_mov_b32_e32 v2, s2 4053; GFX7-NEXT: v_mov_b32_e32 v1, s5 4054; GFX7-NEXT: v_mov_b32_e32 v3, s3 4055; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4056; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4057; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4058; GFX7-NEXT: v_mov_b32_e32 v0, s0 4059; GFX7-NEXT: v_mov_b32_e32 v1, s1 4060; GFX7-NEXT: s_waitcnt vmcnt(0) 4061; GFX7-NEXT: flat_store_dword v[0:1], v2 4062; GFX7-NEXT: s_endpgm 4063; 4064; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4065; GFX10-WGP: ; %bb.0: ; %entry 4066; GFX10-WGP-NEXT: s_clause 0x1 4067; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4068; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4069; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4070; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4071; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4072; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4073; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4074; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4075; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4076; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4077; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4078; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4079; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4080; GFX10-WGP-NEXT: buffer_gl0_inv 4081; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4082; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4083; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4084; GFX10-WGP-NEXT: s_endpgm 4085; 4086; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4087; GFX10-CU: ; %bb.0: ; %entry 4088; GFX10-CU-NEXT: s_clause 0x1 4089; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4090; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4091; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4092; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4093; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4094; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4095; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4096; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4097; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4098; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4099; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4100; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4101; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4102; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4103; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4104; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4105; GFX10-CU-NEXT: s_endpgm 4106; 4107; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4108; SKIP-CACHE-INV: ; %bb.0: ; %entry 4109; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4110; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4111; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4112; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4113; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4114; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4115; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4116; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4117; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4118; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4119; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4120; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4121; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4122; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4123; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4124; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4125; SKIP-CACHE-INV-NEXT: s_endpgm 4126; 4127; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4128; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4129; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4130; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4131; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4132; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4133; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4134; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4135; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4136; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4137; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4138; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4139; 4140; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4141; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4142; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4143; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4144; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4145; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4146; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4147; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4148; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4149; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4150; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4151; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4152; GFX90A-TGSPLIT-NEXT: s_endpgm 4153; 4154; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4155; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4156; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4157; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4158; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4159; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4160; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4161; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4162; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4163; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4164; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4165; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4166; 4167; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg: 4168; GFX940-TGSPLIT: ; %bb.0: ; %entry 4169; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4170; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4171; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4172; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4173; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4174; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4175; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4176; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4177; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4178; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4179; GFX940-TGSPLIT-NEXT: s_endpgm 4180 i32* %out, i32 %in, i32 %old) { 4181entry: 4182 %gep = getelementptr i32, i32* %out, i32 4 4183 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire 4184 %val0 = extractvalue { i32, i1 } %val, 0 4185 store i32 %val0, i32* %out, align 4 4186 ret void 4187} 4188 4189define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg( 4190; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4191; GFX7: ; %bb.0: ; %entry 4192; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4193; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4194; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4195; GFX7-NEXT: s_add_u32 s4, s0, 16 4196; GFX7-NEXT: s_addc_u32 s5, s1, 0 4197; GFX7-NEXT: v_mov_b32_e32 v0, s4 4198; GFX7-NEXT: v_mov_b32_e32 v2, s2 4199; GFX7-NEXT: v_mov_b32_e32 v1, s5 4200; GFX7-NEXT: v_mov_b32_e32 v3, s3 4201; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4202; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4203; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4204; GFX7-NEXT: v_mov_b32_e32 v0, s0 4205; GFX7-NEXT: v_mov_b32_e32 v1, s1 4206; GFX7-NEXT: s_waitcnt vmcnt(0) 4207; GFX7-NEXT: flat_store_dword v[0:1], v2 4208; GFX7-NEXT: s_endpgm 4209; 4210; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4211; GFX10-WGP: ; %bb.0: ; %entry 4212; GFX10-WGP-NEXT: s_clause 0x1 4213; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4214; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4215; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4216; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4217; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4218; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4219; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4220; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4221; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4222; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4223; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4224; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4225; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4226; GFX10-WGP-NEXT: buffer_gl0_inv 4227; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4228; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4229; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4230; GFX10-WGP-NEXT: s_endpgm 4231; 4232; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4233; GFX10-CU: ; %bb.0: ; %entry 4234; GFX10-CU-NEXT: s_clause 0x1 4235; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4236; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4237; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4238; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4239; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4240; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4241; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4242; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4243; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4244; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4245; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4246; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4247; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4248; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4249; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4250; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4251; GFX10-CU-NEXT: s_endpgm 4252; 4253; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4254; SKIP-CACHE-INV: ; %bb.0: ; %entry 4255; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4256; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4257; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4258; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4259; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4260; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4261; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4262; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4263; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4264; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4265; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4266; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4267; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4268; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4269; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4270; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4271; SKIP-CACHE-INV-NEXT: s_endpgm 4272; 4273; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4274; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4275; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4276; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4277; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4278; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4279; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4280; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4281; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4282; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4283; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4284; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4285; 4286; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4287; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4288; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4289; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4290; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4291; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4292; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4293; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4294; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4295; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4296; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4297; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4298; GFX90A-TGSPLIT-NEXT: s_endpgm 4299; 4300; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4301; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4302; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4303; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4304; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4305; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4306; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4307; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4308; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4309; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4310; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4311; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4312; 4313; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg: 4314; GFX940-TGSPLIT: ; %bb.0: ; %entry 4315; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4316; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4317; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4318; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4319; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4320; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4321; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4322; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4323; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4324; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4325; GFX940-TGSPLIT-NEXT: s_endpgm 4326 i32* %out, i32 %in, i32 %old) { 4327entry: 4328 %gep = getelementptr i32, i32* %out, i32 4 4329 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire 4330 %val0 = extractvalue { i32, i1 } %val, 0 4331 store i32 %val0, i32* %out, align 4 4332 ret void 4333} 4334 4335define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg( 4336; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4337; GFX7: ; %bb.0: ; %entry 4338; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4339; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4340; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4341; GFX7-NEXT: s_add_u32 s4, s0, 16 4342; GFX7-NEXT: s_addc_u32 s5, s1, 0 4343; GFX7-NEXT: v_mov_b32_e32 v0, s4 4344; GFX7-NEXT: v_mov_b32_e32 v2, s2 4345; GFX7-NEXT: v_mov_b32_e32 v1, s5 4346; GFX7-NEXT: v_mov_b32_e32 v3, s3 4347; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4348; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4349; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4350; GFX7-NEXT: v_mov_b32_e32 v0, s0 4351; GFX7-NEXT: v_mov_b32_e32 v1, s1 4352; GFX7-NEXT: s_waitcnt vmcnt(0) 4353; GFX7-NEXT: flat_store_dword v[0:1], v2 4354; GFX7-NEXT: s_endpgm 4355; 4356; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4357; GFX10-WGP: ; %bb.0: ; %entry 4358; GFX10-WGP-NEXT: s_clause 0x1 4359; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4360; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4361; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4362; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4363; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4364; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4365; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4366; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4367; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4368; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4369; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4370; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4371; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4372; GFX10-WGP-NEXT: buffer_gl0_inv 4373; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4374; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4375; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4376; GFX10-WGP-NEXT: s_endpgm 4377; 4378; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4379; GFX10-CU: ; %bb.0: ; %entry 4380; GFX10-CU-NEXT: s_clause 0x1 4381; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4382; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4383; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4384; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4385; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4386; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4387; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4388; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4389; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4390; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4391; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4392; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4393; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4394; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4395; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4396; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4397; GFX10-CU-NEXT: s_endpgm 4398; 4399; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4400; SKIP-CACHE-INV: ; %bb.0: ; %entry 4401; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4402; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4403; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4404; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4405; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4406; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4407; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4408; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4409; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4410; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4411; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4412; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4413; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4414; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4415; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4416; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4417; SKIP-CACHE-INV-NEXT: s_endpgm 4418; 4419; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4420; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4421; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4422; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4423; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4424; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4425; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4426; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4427; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4428; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4429; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4430; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4431; 4432; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4433; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4434; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4435; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4436; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4437; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4438; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4439; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4440; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4441; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4442; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4443; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4444; GFX90A-TGSPLIT-NEXT: s_endpgm 4445; 4446; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4447; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4448; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4449; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4450; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4451; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4452; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4453; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4454; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4455; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4456; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4457; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4458; 4459; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg: 4460; GFX940-TGSPLIT: ; %bb.0: ; %entry 4461; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4462; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4463; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4464; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4465; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4466; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4467; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4468; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4469; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4470; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4471; GFX940-TGSPLIT-NEXT: s_endpgm 4472 i32* %out, i32 %in, i32 %old) { 4473entry: 4474 %gep = getelementptr i32, i32* %out, i32 4 4475 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire 4476 %val0 = extractvalue { i32, i1 } %val, 0 4477 store i32 %val0, i32* %out, align 4 4478 ret void 4479} 4480 4481define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg( 4482; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4483; GFX7: ; %bb.0: ; %entry 4484; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4485; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4486; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4487; GFX7-NEXT: s_add_u32 s4, s0, 16 4488; GFX7-NEXT: s_addc_u32 s5, s1, 0 4489; GFX7-NEXT: v_mov_b32_e32 v0, s4 4490; GFX7-NEXT: v_mov_b32_e32 v2, s2 4491; GFX7-NEXT: v_mov_b32_e32 v1, s5 4492; GFX7-NEXT: v_mov_b32_e32 v3, s3 4493; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4494; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4495; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4496; GFX7-NEXT: v_mov_b32_e32 v0, s0 4497; GFX7-NEXT: v_mov_b32_e32 v1, s1 4498; GFX7-NEXT: s_waitcnt vmcnt(0) 4499; GFX7-NEXT: flat_store_dword v[0:1], v2 4500; GFX7-NEXT: s_endpgm 4501; 4502; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4503; GFX10-WGP: ; %bb.0: ; %entry 4504; GFX10-WGP-NEXT: s_clause 0x1 4505; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4506; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4507; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4508; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4509; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4510; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4511; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4512; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4513; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4514; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4515; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4516; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4517; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4518; GFX10-WGP-NEXT: buffer_gl0_inv 4519; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4520; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4521; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4522; GFX10-WGP-NEXT: s_endpgm 4523; 4524; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4525; GFX10-CU: ; %bb.0: ; %entry 4526; GFX10-CU-NEXT: s_clause 0x1 4527; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4528; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4529; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4530; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4531; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4532; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4533; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4534; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4535; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4536; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4537; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4538; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4539; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4540; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4541; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4542; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4543; GFX10-CU-NEXT: s_endpgm 4544; 4545; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4546; SKIP-CACHE-INV: ; %bb.0: ; %entry 4547; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4548; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4549; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4550; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4551; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4552; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4553; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4554; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4555; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4556; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4557; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4558; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4559; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4560; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4561; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4562; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4563; SKIP-CACHE-INV-NEXT: s_endpgm 4564; 4565; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4566; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4567; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4568; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4569; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4570; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4571; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4572; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4573; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4574; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4575; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4576; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4577; 4578; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4579; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4580; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4581; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4582; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4583; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4584; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4585; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4586; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4587; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4588; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4589; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4590; GFX90A-TGSPLIT-NEXT: s_endpgm 4591; 4592; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4593; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4594; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4595; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4596; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4597; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4598; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4599; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4600; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4601; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4602; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4603; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4604; 4605; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg: 4606; GFX940-TGSPLIT: ; %bb.0: ; %entry 4607; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4608; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4609; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4610; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4611; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4612; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4613; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4614; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4615; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4616; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4617; GFX940-TGSPLIT-NEXT: s_endpgm 4618 i32* %out, i32 %in, i32 %old) { 4619entry: 4620 %gep = getelementptr i32, i32* %out, i32 4 4621 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst 4622 %val0 = extractvalue { i32, i1 } %val, 0 4623 store i32 %val0, i32* %out, align 4 4624 ret void 4625} 4626 4627define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg( 4628; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4629; GFX7: ; %bb.0: ; %entry 4630; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4631; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4632; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4633; GFX7-NEXT: s_add_u32 s4, s0, 16 4634; GFX7-NEXT: s_addc_u32 s5, s1, 0 4635; GFX7-NEXT: v_mov_b32_e32 v0, s4 4636; GFX7-NEXT: v_mov_b32_e32 v2, s2 4637; GFX7-NEXT: v_mov_b32_e32 v1, s5 4638; GFX7-NEXT: v_mov_b32_e32 v3, s3 4639; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4640; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4641; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4642; GFX7-NEXT: v_mov_b32_e32 v0, s0 4643; GFX7-NEXT: v_mov_b32_e32 v1, s1 4644; GFX7-NEXT: s_waitcnt vmcnt(0) 4645; GFX7-NEXT: flat_store_dword v[0:1], v2 4646; GFX7-NEXT: s_endpgm 4647; 4648; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4649; GFX10-WGP: ; %bb.0: ; %entry 4650; GFX10-WGP-NEXT: s_clause 0x1 4651; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4652; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4653; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4654; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4655; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4656; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4657; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4658; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4659; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4660; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4661; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4662; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4663; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4664; GFX10-WGP-NEXT: buffer_gl0_inv 4665; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4666; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4667; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4668; GFX10-WGP-NEXT: s_endpgm 4669; 4670; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4671; GFX10-CU: ; %bb.0: ; %entry 4672; GFX10-CU-NEXT: s_clause 0x1 4673; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4674; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4675; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4676; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4677; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4678; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4679; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4680; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4681; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4682; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4683; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4684; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4685; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4686; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4687; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4688; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4689; GFX10-CU-NEXT: s_endpgm 4690; 4691; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4692; SKIP-CACHE-INV: ; %bb.0: ; %entry 4693; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4694; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4695; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4696; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4697; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4698; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4699; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4700; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4701; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4702; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4703; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4704; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4705; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4706; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4707; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4708; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4709; SKIP-CACHE-INV-NEXT: s_endpgm 4710; 4711; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4712; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4713; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4714; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4715; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4716; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4717; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4718; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4719; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4720; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4721; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4722; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4723; 4724; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4725; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4726; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4727; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4728; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4729; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4730; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4731; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4732; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4733; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4734; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4735; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4736; GFX90A-TGSPLIT-NEXT: s_endpgm 4737; 4738; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4739; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4740; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4741; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4742; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4743; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4744; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4745; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4746; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4747; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4748; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4749; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4750; 4751; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg: 4752; GFX940-TGSPLIT: ; %bb.0: ; %entry 4753; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4754; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4755; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4756; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4757; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4758; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4759; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4760; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4761; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4762; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4763; GFX940-TGSPLIT-NEXT: s_endpgm 4764 i32* %out, i32 %in, i32 %old) { 4765entry: 4766 %gep = getelementptr i32, i32* %out, i32 4 4767 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst 4768 %val0 = extractvalue { i32, i1 } %val, 0 4769 store i32 %val0, i32* %out, align 4 4770 ret void 4771} 4772 4773define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg( 4774; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4775; GFX7: ; %bb.0: ; %entry 4776; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4777; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4778; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4779; GFX7-NEXT: s_add_u32 s4, s0, 16 4780; GFX7-NEXT: s_addc_u32 s5, s1, 0 4781; GFX7-NEXT: v_mov_b32_e32 v0, s4 4782; GFX7-NEXT: v_mov_b32_e32 v2, s2 4783; GFX7-NEXT: v_mov_b32_e32 v1, s5 4784; GFX7-NEXT: v_mov_b32_e32 v3, s3 4785; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4786; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4787; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4788; GFX7-NEXT: v_mov_b32_e32 v0, s0 4789; GFX7-NEXT: v_mov_b32_e32 v1, s1 4790; GFX7-NEXT: s_waitcnt vmcnt(0) 4791; GFX7-NEXT: flat_store_dword v[0:1], v2 4792; GFX7-NEXT: s_endpgm 4793; 4794; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4795; GFX10-WGP: ; %bb.0: ; %entry 4796; GFX10-WGP-NEXT: s_clause 0x1 4797; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4798; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4799; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4800; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4801; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4802; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4803; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4804; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4805; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4806; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4807; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4808; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4809; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4810; GFX10-WGP-NEXT: buffer_gl0_inv 4811; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4812; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4813; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4814; GFX10-WGP-NEXT: s_endpgm 4815; 4816; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4817; GFX10-CU: ; %bb.0: ; %entry 4818; GFX10-CU-NEXT: s_clause 0x1 4819; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4820; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4821; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4822; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4823; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4824; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4825; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4826; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4827; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4828; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4829; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4830; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4831; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4832; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4833; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4834; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4835; GFX10-CU-NEXT: s_endpgm 4836; 4837; SKIP-CACHE-INV-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4838; SKIP-CACHE-INV: ; %bb.0: ; %entry 4839; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4840; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4841; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4842; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4843; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4844; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4845; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4846; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4847; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4848; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4849; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4850; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4851; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4852; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4853; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 4854; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 4855; SKIP-CACHE-INV-NEXT: s_endpgm 4856; 4857; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4858; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 4859; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4860; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4861; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4862; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4863; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4864; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4865; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4866; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4867; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4868; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 4869; 4870; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4871; GFX90A-TGSPLIT: ; %bb.0: ; %entry 4872; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4873; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4874; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4875; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 4876; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 4877; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4878; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 4879; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4880; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 4881; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4882; GFX90A-TGSPLIT-NEXT: s_endpgm 4883; 4884; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4885; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 4886; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4887; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4888; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4889; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4890; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4891; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4892; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4893; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4894; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 4895; GFX940-NOTTGSPLIT-NEXT: s_endpgm 4896; 4897; GFX940-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg: 4898; GFX940-TGSPLIT: ; %bb.0: ; %entry 4899; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4900; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 4901; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 4902; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 4903; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 4904; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4905; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 4906; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 4907; GFX940-TGSPLIT-NEXT: buffer_inv sc0 4908; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 4909; GFX940-TGSPLIT-NEXT: s_endpgm 4910 i32* %out, i32 %in, i32 %old) { 4911entry: 4912 %gep = getelementptr i32, i32* %out, i32 4 4913 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst 4914 %val0 = extractvalue { i32, i1 } %val, 0 4915 store i32 %val0, i32* %out, align 4 4916 ret void 4917} 4918 4919define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg( 4920; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 4921; GFX7: ; %bb.0: ; %entry 4922; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4923; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 4924; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4925; GFX7-NEXT: s_add_u32 s4, s0, 16 4926; GFX7-NEXT: s_addc_u32 s5, s1, 0 4927; GFX7-NEXT: v_mov_b32_e32 v0, s4 4928; GFX7-NEXT: v_mov_b32_e32 v2, s2 4929; GFX7-NEXT: v_mov_b32_e32 v1, s5 4930; GFX7-NEXT: v_mov_b32_e32 v3, s3 4931; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4932; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4933; GFX7-NEXT: s_waitcnt lgkmcnt(0) 4934; GFX7-NEXT: v_mov_b32_e32 v0, s0 4935; GFX7-NEXT: v_mov_b32_e32 v1, s1 4936; GFX7-NEXT: s_waitcnt vmcnt(0) 4937; GFX7-NEXT: flat_store_dword v[0:1], v2 4938; GFX7-NEXT: s_endpgm 4939; 4940; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 4941; GFX10-WGP: ; %bb.0: ; %entry 4942; GFX10-WGP-NEXT: s_clause 0x1 4943; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4944; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4945; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 4946; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 4947; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 4948; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 4949; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 4950; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 4951; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 4952; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4953; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 4954; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4955; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 4956; GFX10-WGP-NEXT: buffer_gl0_inv 4957; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 4958; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 4959; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 4960; GFX10-WGP-NEXT: s_endpgm 4961; 4962; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 4963; GFX10-CU: ; %bb.0: ; %entry 4964; GFX10-CU-NEXT: s_clause 0x1 4965; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 4966; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 4967; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4968; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 4969; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 4970; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 4971; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 4972; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 4973; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 4974; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4975; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4976; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 4977; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 4978; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 4979; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 4980; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 4981; GFX10-CU-NEXT: s_endpgm 4982; 4983; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 4984; SKIP-CACHE-INV: ; %bb.0: ; %entry 4985; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 4986; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 4987; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4988; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 4989; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 4990; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 4991; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 4992; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 4993; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 4994; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4995; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 4996; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 4997; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 4998; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 4999; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 5000; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5001; SKIP-CACHE-INV-NEXT: s_endpgm 5002; 5003; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 5004; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5005; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5006; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5007; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5008; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5009; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 5010; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5011; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 5012; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5013; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5014; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5015; 5016; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 5017; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5018; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5019; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5020; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5021; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5022; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 5023; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5024; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 5025; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5026; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 5027; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5028; GFX90A-TGSPLIT-NEXT: s_endpgm 5029; 5030; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 5031; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5032; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 5033; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 5034; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5035; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5036; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 5037; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5038; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 5039; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5040; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5041; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5042; 5043; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg: 5044; GFX940-TGSPLIT: ; %bb.0: ; %entry 5045; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 5046; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 5047; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5048; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5049; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 5050; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5051; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 5052; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5053; GFX940-TGSPLIT-NEXT: buffer_inv sc0 5054; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5055; GFX940-TGSPLIT-NEXT: s_endpgm 5056 i32* %out, i32 %in, i32 %old) { 5057entry: 5058 %gep = getelementptr i32, i32* %out, i32 4 5059 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst 5060 %val0 = extractvalue { i32, i1 } %val, 0 5061 store i32 %val0, i32* %out, align 4 5062 ret void 5063} 5064 5065define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg( 5066; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5067; GFX7: ; %bb.0: ; %entry 5068; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5069; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 5070; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5071; GFX7-NEXT: s_add_u32 s4, s0, 16 5072; GFX7-NEXT: s_addc_u32 s5, s1, 0 5073; GFX7-NEXT: v_mov_b32_e32 v0, s4 5074; GFX7-NEXT: v_mov_b32_e32 v2, s2 5075; GFX7-NEXT: v_mov_b32_e32 v1, s5 5076; GFX7-NEXT: v_mov_b32_e32 v3, s3 5077; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5078; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 5079; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5080; GFX7-NEXT: v_mov_b32_e32 v0, s0 5081; GFX7-NEXT: v_mov_b32_e32 v1, s1 5082; GFX7-NEXT: s_waitcnt vmcnt(0) 5083; GFX7-NEXT: flat_store_dword v[0:1], v2 5084; GFX7-NEXT: s_endpgm 5085; 5086; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5087; GFX10-WGP: ; %bb.0: ; %entry 5088; GFX10-WGP-NEXT: s_clause 0x1 5089; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5090; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5091; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5092; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 5093; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 5094; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 5095; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 5096; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 5097; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 5098; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5099; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 5100; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 5101; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5102; GFX10-WGP-NEXT: buffer_gl0_inv 5103; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5104; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5105; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5106; GFX10-WGP-NEXT: s_endpgm 5107; 5108; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5109; GFX10-CU: ; %bb.0: ; %entry 5110; GFX10-CU-NEXT: s_clause 0x1 5111; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5112; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5113; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5114; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 5115; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 5116; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 5117; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 5118; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 5119; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 5120; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5121; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 5122; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5123; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5124; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5125; GFX10-CU-NEXT: s_waitcnt vmcnt(0) 5126; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5127; GFX10-CU-NEXT: s_endpgm 5128; 5129; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5130; SKIP-CACHE-INV: ; %bb.0: ; %entry 5131; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 5132; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 5133; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5134; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 5135; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 5136; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 5137; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 5138; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 5139; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 5140; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5141; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 5142; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5143; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5144; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5145; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) 5146; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5147; SKIP-CACHE-INV-NEXT: s_endpgm 5148; 5149; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5150; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5151; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5152; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5153; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5154; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5155; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 5156; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5157; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 5158; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5159; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5160; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5161; 5162; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5163; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5164; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 5165; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 5166; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5167; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5168; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 5169; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5170; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 5171; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5172; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 5173; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5174; GFX90A-TGSPLIT-NEXT: s_endpgm 5175; 5176; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5177; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5178; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 5179; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 5180; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5181; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5182; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 5183; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5184; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 5185; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5186; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5187; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5188; 5189; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg: 5190; GFX940-TGSPLIT: ; %bb.0: ; %entry 5191; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 5192; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 5193; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5194; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5195; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 5196; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5197; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 5198; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5199; GFX940-TGSPLIT-NEXT: buffer_inv sc0 5200; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5201; GFX940-TGSPLIT-NEXT: s_endpgm 5202 i32* %out, i32 %in, i32 %old) { 5203entry: 5204 %gep = getelementptr i32, i32* %out, i32 4 5205 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst 5206 %val0 = extractvalue { i32, i1 } %val, 0 5207 store i32 %val0, i32* %out, align 4 5208 ret void 5209} 5210 5211define amdgpu_kernel void @flat_workgroup_one_as_unordered_load( 5212; GFX7-LABEL: flat_workgroup_one_as_unordered_load: 5213; GFX7: ; %bb.0: ; %entry 5214; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5215; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5216; GFX7-NEXT: v_mov_b32_e32 v0, s0 5217; GFX7-NEXT: v_mov_b32_e32 v1, s1 5218; GFX7-NEXT: flat_load_dword v2, v[0:1] 5219; GFX7-NEXT: v_mov_b32_e32 v0, s2 5220; GFX7-NEXT: v_mov_b32_e32 v1, s3 5221; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5222; GFX7-NEXT: flat_store_dword v[0:1], v2 5223; GFX7-NEXT: s_endpgm 5224; 5225; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load: 5226; GFX10-WGP: ; %bb.0: ; %entry 5227; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5228; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5229; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5230; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5231; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] 5232; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 5233; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 5234; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5235; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5236; GFX10-WGP-NEXT: s_endpgm 5237; 5238; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load: 5239; GFX10-CU: ; %bb.0: ; %entry 5240; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5241; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5242; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5243; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5244; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 5245; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 5246; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 5247; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5248; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5249; GFX10-CU-NEXT: s_endpgm 5250; 5251; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load: 5252; SKIP-CACHE-INV: ; %bb.0: ; %entry 5253; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5254; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5255; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 5256; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 5257; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 5258; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5259; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5260; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5261; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5262; SKIP-CACHE-INV-NEXT: s_endpgm 5263; 5264; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load: 5265; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5266; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5267; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5268; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5269; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5270; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5271; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5272; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5273; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5274; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5275; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5276; 5277; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load: 5278; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5279; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5280; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5281; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5282; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5283; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5284; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5285; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5286; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5287; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5288; GFX90A-TGSPLIT-NEXT: s_endpgm 5289; 5290; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load: 5291; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5292; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5293; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5294; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5295; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5296; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5297; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5298; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5299; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5300; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5301; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5302; 5303; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load: 5304; GFX940-TGSPLIT: ; %bb.0: ; %entry 5305; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5306; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5307; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5308; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5309; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5310; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5311; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5312; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5313; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5314; GFX940-TGSPLIT-NEXT: s_endpgm 5315 i32* %in, i32* %out) { 5316entry: 5317 %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4 5318 store i32 %val, i32* %out 5319 ret void 5320} 5321 5322define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load( 5323; GFX7-LABEL: flat_workgroup_one_as_monotonic_load: 5324; GFX7: ; %bb.0: ; %entry 5325; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5326; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5327; GFX7-NEXT: v_mov_b32_e32 v0, s0 5328; GFX7-NEXT: v_mov_b32_e32 v1, s1 5329; GFX7-NEXT: flat_load_dword v2, v[0:1] 5330; GFX7-NEXT: v_mov_b32_e32 v0, s2 5331; GFX7-NEXT: v_mov_b32_e32 v1, s3 5332; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5333; GFX7-NEXT: flat_store_dword v[0:1], v2 5334; GFX7-NEXT: s_endpgm 5335; 5336; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load: 5337; GFX10-WGP: ; %bb.0: ; %entry 5338; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5339; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5340; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5341; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5342; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 5343; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 5344; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 5345; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5346; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5347; GFX10-WGP-NEXT: s_endpgm 5348; 5349; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load: 5350; GFX10-CU: ; %bb.0: ; %entry 5351; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5352; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5353; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5354; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5355; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 5356; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 5357; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 5358; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5359; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5360; GFX10-CU-NEXT: s_endpgm 5361; 5362; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load: 5363; SKIP-CACHE-INV: ; %bb.0: ; %entry 5364; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5365; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5366; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 5367; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 5368; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 5369; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5370; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5371; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5372; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5373; SKIP-CACHE-INV-NEXT: s_endpgm 5374; 5375; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load: 5376; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5377; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5378; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5379; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5380; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5381; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5382; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5383; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5384; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5385; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5386; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5387; 5388; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load: 5389; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5390; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5391; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5392; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5393; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5394; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 5395; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5396; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5397; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5398; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5399; GFX90A-TGSPLIT-NEXT: s_endpgm 5400; 5401; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load: 5402; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5403; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5404; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5405; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5406; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5407; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5408; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5409; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5410; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5411; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5412; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5413; 5414; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load: 5415; GFX940-TGSPLIT: ; %bb.0: ; %entry 5416; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5417; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5418; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5419; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5420; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5421; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5422; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5423; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5424; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5425; GFX940-TGSPLIT-NEXT: s_endpgm 5426 i32* %in, i32* %out) { 5427entry: 5428 %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4 5429 store i32 %val, i32* %out 5430 ret void 5431} 5432 5433define amdgpu_kernel void @flat_workgroup_one_as_acquire_load( 5434; GFX7-LABEL: flat_workgroup_one_as_acquire_load: 5435; GFX7: ; %bb.0: ; %entry 5436; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5437; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5438; GFX7-NEXT: v_mov_b32_e32 v0, s0 5439; GFX7-NEXT: v_mov_b32_e32 v1, s1 5440; GFX7-NEXT: flat_load_dword v2, v[0:1] 5441; GFX7-NEXT: v_mov_b32_e32 v0, s2 5442; GFX7-NEXT: v_mov_b32_e32 v1, s3 5443; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5444; GFX7-NEXT: flat_store_dword v[0:1], v2 5445; GFX7-NEXT: s_endpgm 5446; 5447; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load: 5448; GFX10-WGP: ; %bb.0: ; %entry 5449; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5450; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5451; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5452; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5453; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 5454; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 5455; GFX10-WGP-NEXT: buffer_gl0_inv 5456; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 5457; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 5458; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5459; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5460; GFX10-WGP-NEXT: s_endpgm 5461; 5462; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load: 5463; GFX10-CU: ; %bb.0: ; %entry 5464; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5465; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5466; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5467; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5468; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 5469; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 5470; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 5471; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5472; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5473; GFX10-CU-NEXT: s_endpgm 5474; 5475; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load: 5476; SKIP-CACHE-INV: ; %bb.0: ; %entry 5477; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5478; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5479; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 5480; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 5481; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 5482; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5483; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5484; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5485; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5486; SKIP-CACHE-INV-NEXT: s_endpgm 5487; 5488; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load: 5489; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5490; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5491; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5492; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5493; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5494; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5495; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5496; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5497; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5498; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5499; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5500; 5501; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load: 5502; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5503; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5504; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5505; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5506; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5507; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 5508; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5509; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 5510; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5511; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5512; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5513; GFX90A-TGSPLIT-NEXT: s_endpgm 5514; 5515; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load: 5516; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5517; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5518; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5519; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5520; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5521; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5522; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5523; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5524; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5525; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5526; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5527; 5528; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load: 5529; GFX940-TGSPLIT: ; %bb.0: ; %entry 5530; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5531; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5532; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5533; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5534; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5535; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5536; GFX940-TGSPLIT-NEXT: buffer_inv sc0 5537; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5538; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5539; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5540; GFX940-TGSPLIT-NEXT: s_endpgm 5541 i32* %in, i32* %out) { 5542entry: 5543 %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4 5544 store i32 %val, i32* %out 5545 ret void 5546} 5547 5548define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load( 5549; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load: 5550; GFX7: ; %bb.0: ; %entry 5551; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5552; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5553; GFX7-NEXT: v_mov_b32_e32 v0, s0 5554; GFX7-NEXT: v_mov_b32_e32 v1, s1 5555; GFX7-NEXT: flat_load_dword v2, v[0:1] 5556; GFX7-NEXT: v_mov_b32_e32 v0, s2 5557; GFX7-NEXT: v_mov_b32_e32 v1, s3 5558; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5559; GFX7-NEXT: flat_store_dword v[0:1], v2 5560; GFX7-NEXT: s_endpgm 5561; 5562; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load: 5563; GFX10-WGP: ; %bb.0: ; %entry 5564; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5565; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5566; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5567; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5568; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 5569; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 5570; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc 5571; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 5572; GFX10-WGP-NEXT: buffer_gl0_inv 5573; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 5574; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 5575; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5576; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5577; GFX10-WGP-NEXT: s_endpgm 5578; 5579; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load: 5580; GFX10-CU: ; %bb.0: ; %entry 5581; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5582; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5583; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5584; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5585; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] 5586; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 5587; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 5588; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5589; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5590; GFX10-CU-NEXT: s_endpgm 5591; 5592; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load: 5593; SKIP-CACHE-INV: ; %bb.0: ; %entry 5594; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5595; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5596; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0 5597; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1 5598; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] 5599; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5600; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5601; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5602; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5603; SKIP-CACHE-INV-NEXT: s_endpgm 5604; 5605; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load: 5606; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5607; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5608; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5609; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5610; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5611; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] 5612; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5613; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5614; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5615; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5616; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5617; 5618; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load: 5619; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5620; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 5621; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5622; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5623; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5624; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5625; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc 5626; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5627; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 5628; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5629; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5630; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5631; GFX90A-TGSPLIT-NEXT: s_endpgm 5632; 5633; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load: 5634; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5635; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5636; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5637; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5638; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5639; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5640; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5641; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5642; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 5643; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5644; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5645; 5646; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load: 5647; GFX940-TGSPLIT: ; %bb.0: ; %entry 5648; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 5649; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5650; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0 5651; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1 5652; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5653; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 5654; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5655; GFX940-TGSPLIT-NEXT: buffer_inv sc0 5656; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2 5657; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3 5658; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5659; GFX940-TGSPLIT-NEXT: s_endpgm 5660 i32* %in, i32* %out) { 5661entry: 5662 %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4 5663 store i32 %val, i32* %out 5664 ret void 5665} 5666 5667define amdgpu_kernel void @flat_workgroup_one_as_unordered_store( 5668; GFX7-LABEL: flat_workgroup_one_as_unordered_store: 5669; GFX7: ; %bb.0: ; %entry 5670; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 5671; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 5672; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5673; GFX7-NEXT: v_mov_b32_e32 v0, s0 5674; GFX7-NEXT: v_mov_b32_e32 v1, s1 5675; GFX7-NEXT: v_mov_b32_e32 v2, s2 5676; GFX7-NEXT: flat_store_dword v[0:1], v2 5677; GFX7-NEXT: s_endpgm 5678; 5679; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store: 5680; GFX10-WGP: ; %bb.0: ; %entry 5681; GFX10-WGP-NEXT: s_clause 0x1 5682; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5683; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 5684; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5685; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5686; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5687; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 5688; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5689; GFX10-WGP-NEXT: s_endpgm 5690; 5691; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store: 5692; GFX10-CU: ; %bb.0: ; %entry 5693; GFX10-CU-NEXT: s_clause 0x1 5694; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5695; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 5696; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5697; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5698; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5699; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 5700; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5701; GFX10-CU-NEXT: s_endpgm 5702; 5703; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store: 5704; SKIP-CACHE-INV: ; %bb.0: ; %entry 5705; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 5706; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 5707; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5708; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5709; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5710; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 5711; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5712; SKIP-CACHE-INV-NEXT: s_endpgm 5713; 5714; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store: 5715; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5716; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5717; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5718; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5719; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5720; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5721; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5722; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5723; 5724; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store: 5725; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5726; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5727; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5728; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5729; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5730; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5731; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5732; GFX90A-TGSPLIT-NEXT: s_endpgm 5733; 5734; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store: 5735; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5736; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5737; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5738; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5739; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5740; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5741; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5742; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5743; 5744; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store: 5745; GFX940-TGSPLIT: ; %bb.0: ; %entry 5746; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5747; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5748; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5749; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5750; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5751; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5752; GFX940-TGSPLIT-NEXT: s_endpgm 5753 i32 %in, i32* %out) { 5754entry: 5755 store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4 5756 ret void 5757} 5758 5759define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store( 5760; GFX7-LABEL: flat_workgroup_one_as_monotonic_store: 5761; GFX7: ; %bb.0: ; %entry 5762; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 5763; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 5764; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5765; GFX7-NEXT: v_mov_b32_e32 v0, s0 5766; GFX7-NEXT: v_mov_b32_e32 v1, s1 5767; GFX7-NEXT: v_mov_b32_e32 v2, s2 5768; GFX7-NEXT: flat_store_dword v[0:1], v2 5769; GFX7-NEXT: s_endpgm 5770; 5771; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store: 5772; GFX10-WGP: ; %bb.0: ; %entry 5773; GFX10-WGP-NEXT: s_clause 0x1 5774; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5775; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 5776; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5777; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5778; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5779; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 5780; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5781; GFX10-WGP-NEXT: s_endpgm 5782; 5783; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store: 5784; GFX10-CU: ; %bb.0: ; %entry 5785; GFX10-CU-NEXT: s_clause 0x1 5786; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5787; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 5788; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5789; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5790; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5791; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 5792; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5793; GFX10-CU-NEXT: s_endpgm 5794; 5795; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store: 5796; SKIP-CACHE-INV: ; %bb.0: ; %entry 5797; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 5798; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 5799; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5800; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5801; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5802; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 5803; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5804; SKIP-CACHE-INV-NEXT: s_endpgm 5805; 5806; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store: 5807; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5808; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5809; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5810; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5811; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5812; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5813; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5814; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5815; 5816; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store: 5817; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5818; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5819; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5820; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5821; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5822; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5823; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5824; GFX90A-TGSPLIT-NEXT: s_endpgm 5825; 5826; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store: 5827; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5828; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5829; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5830; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5831; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5832; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5833; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 5834; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5835; 5836; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store: 5837; GFX940-TGSPLIT: ; %bb.0: ; %entry 5838; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5839; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5840; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5841; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5842; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5843; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 5844; GFX940-TGSPLIT-NEXT: s_endpgm 5845 i32 %in, i32* %out) { 5846entry: 5847 store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4 5848 ret void 5849} 5850 5851define amdgpu_kernel void @flat_workgroup_one_as_release_store( 5852; GFX7-LABEL: flat_workgroup_one_as_release_store: 5853; GFX7: ; %bb.0: ; %entry 5854; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 5855; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 5856; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5857; GFX7-NEXT: v_mov_b32_e32 v0, s0 5858; GFX7-NEXT: v_mov_b32_e32 v1, s1 5859; GFX7-NEXT: v_mov_b32_e32 v2, s2 5860; GFX7-NEXT: flat_store_dword v[0:1], v2 5861; GFX7-NEXT: s_endpgm 5862; 5863; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store: 5864; GFX10-WGP: ; %bb.0: ; %entry 5865; GFX10-WGP-NEXT: s_clause 0x1 5866; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5867; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 5868; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5869; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5870; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5871; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 5872; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 5873; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 5874; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5875; GFX10-WGP-NEXT: s_endpgm 5876; 5877; GFX10-CU-LABEL: flat_workgroup_one_as_release_store: 5878; GFX10-CU: ; %bb.0: ; %entry 5879; GFX10-CU-NEXT: s_clause 0x1 5880; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5881; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 5882; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5883; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5884; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5885; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 5886; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5887; GFX10-CU-NEXT: s_endpgm 5888; 5889; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store: 5890; SKIP-CACHE-INV: ; %bb.0: ; %entry 5891; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 5892; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 5893; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5894; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5895; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5896; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 5897; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5898; SKIP-CACHE-INV-NEXT: s_endpgm 5899; 5900; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store: 5901; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5902; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5903; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5904; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5905; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5906; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5907; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 5908; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 5909; 5910; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store: 5911; GFX90A-TGSPLIT: ; %bb.0: ; %entry 5912; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5913; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 5914; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5915; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 5916; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 5917; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5918; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 5919; GFX90A-TGSPLIT-NEXT: s_endpgm 5920; 5921; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store: 5922; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 5923; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5924; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5925; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5926; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5927; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5928; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 5929; GFX940-NOTTGSPLIT-NEXT: s_endpgm 5930; 5931; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_store: 5932; GFX940-TGSPLIT: ; %bb.0: ; %entry 5933; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 5934; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 5935; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 5936; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 5937; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 5938; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 5939; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 5940; GFX940-TGSPLIT-NEXT: s_endpgm 5941 i32 %in, i32* %out) { 5942entry: 5943 store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4 5944 ret void 5945} 5946 5947define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store( 5948; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store: 5949; GFX7: ; %bb.0: ; %entry 5950; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 5951; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0 5952; GFX7-NEXT: s_waitcnt lgkmcnt(0) 5953; GFX7-NEXT: v_mov_b32_e32 v0, s0 5954; GFX7-NEXT: v_mov_b32_e32 v1, s1 5955; GFX7-NEXT: v_mov_b32_e32 v2, s2 5956; GFX7-NEXT: flat_store_dword v[0:1], v2 5957; GFX7-NEXT: s_endpgm 5958; 5959; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store: 5960; GFX10-WGP: ; %bb.0: ; %entry 5961; GFX10-WGP-NEXT: s_clause 0x1 5962; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5963; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0 5964; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 5965; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 5966; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 5967; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 5968; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 5969; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 5970; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 5971; GFX10-WGP-NEXT: s_endpgm 5972; 5973; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store: 5974; GFX10-CU: ; %bb.0: ; %entry 5975; GFX10-CU-NEXT: s_clause 0x1 5976; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5977; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0 5978; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 5979; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 5980; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 5981; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 5982; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 5983; GFX10-CU-NEXT: s_endpgm 5984; 5985; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store: 5986; SKIP-CACHE-INV: ; %bb.0: ; %entry 5987; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 5988; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0 5989; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 5990; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 5991; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 5992; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 5993; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 5994; SKIP-CACHE-INV-NEXT: s_endpgm 5995; 5996; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store: 5997; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 5998; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 5999; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 6000; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6001; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6002; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6003; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6004; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6005; 6006; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store: 6007; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6008; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 6009; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0 6010; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6011; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6012; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6013; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6014; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6015; GFX90A-TGSPLIT-NEXT: s_endpgm 6016; 6017; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store: 6018; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6019; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 6020; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 6021; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6022; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6023; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6024; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 6025; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6026; 6027; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store: 6028; GFX940-TGSPLIT: ; %bb.0: ; %entry 6029; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 6030; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0 6031; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6032; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6033; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6034; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6035; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 6036; GFX940-TGSPLIT-NEXT: s_endpgm 6037 i32 %in, i32* %out) { 6038entry: 6039 store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4 6040 ret void 6041} 6042 6043define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw( 6044; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6045; GFX7: ; %bb.0: ; %entry 6046; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6047; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6048; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6049; GFX7-NEXT: v_mov_b32_e32 v0, s0 6050; GFX7-NEXT: v_mov_b32_e32 v1, s1 6051; GFX7-NEXT: v_mov_b32_e32 v2, s2 6052; GFX7-NEXT: flat_atomic_swap v[0:1], v2 6053; GFX7-NEXT: s_endpgm 6054; 6055; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6056; GFX10-WGP: ; %bb.0: ; %entry 6057; GFX10-WGP-NEXT: s_clause 0x1 6058; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6059; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6060; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6061; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6062; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6063; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6064; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 6065; GFX10-WGP-NEXT: s_endpgm 6066; 6067; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6068; GFX10-CU: ; %bb.0: ; %entry 6069; GFX10-CU-NEXT: s_clause 0x1 6070; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6071; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6072; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6073; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6074; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6075; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6076; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 6077; GFX10-CU-NEXT: s_endpgm 6078; 6079; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6080; SKIP-CACHE-INV: ; %bb.0: ; %entry 6081; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6082; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6083; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6084; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6085; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6086; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6087; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 6088; SKIP-CACHE-INV-NEXT: s_endpgm 6089; 6090; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6091; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6092; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6093; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6094; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6095; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6096; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6097; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6098; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6099; 6100; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6101; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6102; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6103; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6104; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6105; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6106; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6107; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6108; GFX90A-TGSPLIT-NEXT: s_endpgm 6109; 6110; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6111; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6112; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6113; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6114; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6115; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6116; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6117; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6118; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6119; 6120; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw: 6121; GFX940-TGSPLIT: ; %bb.0: ; %entry 6122; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6123; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6124; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6125; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6126; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6127; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6128; GFX940-TGSPLIT-NEXT: s_endpgm 6129 i32* %out, i32 %in) { 6130entry: 6131 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic 6132 ret void 6133} 6134 6135define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw( 6136; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6137; GFX7: ; %bb.0: ; %entry 6138; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6139; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6140; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6141; GFX7-NEXT: v_mov_b32_e32 v0, s0 6142; GFX7-NEXT: v_mov_b32_e32 v1, s1 6143; GFX7-NEXT: v_mov_b32_e32 v2, s2 6144; GFX7-NEXT: flat_atomic_swap v[0:1], v2 6145; GFX7-NEXT: s_endpgm 6146; 6147; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6148; GFX10-WGP: ; %bb.0: ; %entry 6149; GFX10-WGP-NEXT: s_clause 0x1 6150; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6151; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6152; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6153; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6154; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6155; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6156; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 6157; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6158; GFX10-WGP-NEXT: buffer_gl0_inv 6159; GFX10-WGP-NEXT: s_endpgm 6160; 6161; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6162; GFX10-CU: ; %bb.0: ; %entry 6163; GFX10-CU-NEXT: s_clause 0x1 6164; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6165; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6166; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6167; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6168; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6169; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6170; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 6171; GFX10-CU-NEXT: s_endpgm 6172; 6173; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6174; SKIP-CACHE-INV: ; %bb.0: ; %entry 6175; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6176; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6177; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6178; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6179; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6180; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6181; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 6182; SKIP-CACHE-INV-NEXT: s_endpgm 6183; 6184; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6185; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6186; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6187; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6188; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6189; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6190; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6191; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6192; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6193; 6194; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6195; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6196; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6197; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6198; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6199; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6200; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6201; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6202; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6203; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6204; GFX90A-TGSPLIT-NEXT: s_endpgm 6205; 6206; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6207; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6208; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6209; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6210; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6211; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6212; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6213; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6214; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6215; 6216; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw: 6217; GFX940-TGSPLIT: ; %bb.0: ; %entry 6218; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6219; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6220; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6221; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6222; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6223; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6224; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6225; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6226; GFX940-TGSPLIT-NEXT: s_endpgm 6227 i32* %out, i32 %in) { 6228entry: 6229 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire 6230 ret void 6231} 6232 6233define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw( 6234; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw: 6235; GFX7: ; %bb.0: ; %entry 6236; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6237; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6238; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6239; GFX7-NEXT: v_mov_b32_e32 v0, s0 6240; GFX7-NEXT: v_mov_b32_e32 v1, s1 6241; GFX7-NEXT: v_mov_b32_e32 v2, s2 6242; GFX7-NEXT: flat_atomic_swap v[0:1], v2 6243; GFX7-NEXT: s_endpgm 6244; 6245; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw: 6246; GFX10-WGP: ; %bb.0: ; %entry 6247; GFX10-WGP-NEXT: s_clause 0x1 6248; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6249; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6250; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6251; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6252; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6253; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6254; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6255; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6256; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 6257; GFX10-WGP-NEXT: s_endpgm 6258; 6259; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw: 6260; GFX10-CU: ; %bb.0: ; %entry 6261; GFX10-CU-NEXT: s_clause 0x1 6262; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6263; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6264; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6265; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6266; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6267; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6268; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 6269; GFX10-CU-NEXT: s_endpgm 6270; 6271; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw: 6272; SKIP-CACHE-INV: ; %bb.0: ; %entry 6273; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6274; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6275; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6276; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6277; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6278; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6279; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 6280; SKIP-CACHE-INV-NEXT: s_endpgm 6281; 6282; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw: 6283; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6284; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6285; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6286; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6287; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6288; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6289; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6290; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6291; 6292; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw: 6293; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6294; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6295; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6296; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6297; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6298; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6299; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6300; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6301; GFX90A-TGSPLIT-NEXT: s_endpgm 6302; 6303; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw: 6304; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6305; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6306; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6307; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6308; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6309; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6310; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6311; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6312; 6313; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw: 6314; GFX940-TGSPLIT: ; %bb.0: ; %entry 6315; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6316; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6317; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6318; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6319; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6320; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6321; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6322; GFX940-TGSPLIT-NEXT: s_endpgm 6323 i32* %out, i32 %in) { 6324entry: 6325 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release 6326 ret void 6327} 6328 6329define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw( 6330; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6331; GFX7: ; %bb.0: ; %entry 6332; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6333; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6334; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6335; GFX7-NEXT: v_mov_b32_e32 v0, s0 6336; GFX7-NEXT: v_mov_b32_e32 v1, s1 6337; GFX7-NEXT: v_mov_b32_e32 v2, s2 6338; GFX7-NEXT: flat_atomic_swap v[0:1], v2 6339; GFX7-NEXT: s_endpgm 6340; 6341; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6342; GFX10-WGP: ; %bb.0: ; %entry 6343; GFX10-WGP-NEXT: s_clause 0x1 6344; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6345; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6346; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6347; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6348; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6349; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6350; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6351; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6352; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 6353; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6354; GFX10-WGP-NEXT: buffer_gl0_inv 6355; GFX10-WGP-NEXT: s_endpgm 6356; 6357; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6358; GFX10-CU: ; %bb.0: ; %entry 6359; GFX10-CU-NEXT: s_clause 0x1 6360; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6361; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6362; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6363; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6364; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6365; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6366; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 6367; GFX10-CU-NEXT: s_endpgm 6368; 6369; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6370; SKIP-CACHE-INV: ; %bb.0: ; %entry 6371; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6372; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6373; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6374; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6375; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6376; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6377; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 6378; SKIP-CACHE-INV-NEXT: s_endpgm 6379; 6380; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6381; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6382; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6383; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6384; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6385; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6386; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6387; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6388; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6389; 6390; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6391; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6392; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6393; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6394; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6395; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6396; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6397; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6398; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6399; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6400; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6401; GFX90A-TGSPLIT-NEXT: s_endpgm 6402; 6403; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6404; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6405; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6406; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6407; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6408; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6409; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6410; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6411; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6412; 6413; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw: 6414; GFX940-TGSPLIT: ; %bb.0: ; %entry 6415; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6416; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6417; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6418; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6419; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6420; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6421; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6422; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6423; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6424; GFX940-TGSPLIT-NEXT: s_endpgm 6425 i32* %out, i32 %in) { 6426entry: 6427 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel 6428 ret void 6429} 6430 6431define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw( 6432; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6433; GFX7: ; %bb.0: ; %entry 6434; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6435; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6436; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6437; GFX7-NEXT: v_mov_b32_e32 v0, s0 6438; GFX7-NEXT: v_mov_b32_e32 v1, s1 6439; GFX7-NEXT: v_mov_b32_e32 v2, s2 6440; GFX7-NEXT: flat_atomic_swap v[0:1], v2 6441; GFX7-NEXT: s_endpgm 6442; 6443; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6444; GFX10-WGP: ; %bb.0: ; %entry 6445; GFX10-WGP-NEXT: s_clause 0x1 6446; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6447; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6448; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6449; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6450; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6451; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6452; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6453; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6454; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 6455; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6456; GFX10-WGP-NEXT: buffer_gl0_inv 6457; GFX10-WGP-NEXT: s_endpgm 6458; 6459; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6460; GFX10-CU: ; %bb.0: ; %entry 6461; GFX10-CU-NEXT: s_clause 0x1 6462; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6463; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6464; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6465; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6466; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6467; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6468; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 6469; GFX10-CU-NEXT: s_endpgm 6470; 6471; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6472; SKIP-CACHE-INV: ; %bb.0: ; %entry 6473; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6474; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6475; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6476; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6477; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6478; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6479; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2 6480; SKIP-CACHE-INV-NEXT: s_endpgm 6481; 6482; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6483; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6484; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6485; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6486; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6487; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6488; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6489; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6490; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6491; 6492; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6493; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6494; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6495; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6496; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6497; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6498; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6499; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6500; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6501; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6502; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6503; GFX90A-TGSPLIT-NEXT: s_endpgm 6504; 6505; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6506; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6507; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6508; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6509; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6510; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6511; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6512; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6513; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6514; 6515; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw: 6516; GFX940-TGSPLIT: ; %bb.0: ; %entry 6517; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6518; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6519; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6520; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6521; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6522; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6523; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2 6524; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6525; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6526; GFX940-TGSPLIT-NEXT: s_endpgm 6527 i32* %out, i32 %in) { 6528entry: 6529 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst 6530 ret void 6531} 6532 6533define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw( 6534; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6535; GFX7: ; %bb.0: ; %entry 6536; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6537; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6538; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6539; GFX7-NEXT: v_mov_b32_e32 v0, s0 6540; GFX7-NEXT: v_mov_b32_e32 v1, s1 6541; GFX7-NEXT: v_mov_b32_e32 v2, s2 6542; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6543; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6544; GFX7-NEXT: flat_store_dword v[0:1], v2 6545; GFX7-NEXT: s_endpgm 6546; 6547; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6548; GFX10-WGP: ; %bb.0: ; %entry 6549; GFX10-WGP-NEXT: s_clause 0x1 6550; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6551; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6552; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6553; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6554; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6555; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6556; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6557; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6558; GFX10-WGP-NEXT: buffer_gl0_inv 6559; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6560; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 6561; GFX10-WGP-NEXT: s_endpgm 6562; 6563; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6564; GFX10-CU: ; %bb.0: ; %entry 6565; GFX10-CU-NEXT: s_clause 0x1 6566; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6567; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6568; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6569; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6570; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6571; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6572; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6573; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6574; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 6575; GFX10-CU-NEXT: s_endpgm 6576; 6577; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6578; SKIP-CACHE-INV: ; %bb.0: ; %entry 6579; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6580; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6581; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6582; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6583; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6584; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6585; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6586; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6587; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 6588; SKIP-CACHE-INV-NEXT: s_endpgm 6589; 6590; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6591; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6592; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6593; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6594; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6595; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6596; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6597; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6598; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6599; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6600; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6601; 6602; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6603; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6604; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6605; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6606; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6607; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6608; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6609; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6610; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6611; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6612; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6613; GFX90A-TGSPLIT-NEXT: s_endpgm 6614; 6615; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6616; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6617; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6618; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6619; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6620; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6621; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6622; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6623; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6624; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6625; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6626; 6627; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw: 6628; GFX940-TGSPLIT: ; %bb.0: ; %entry 6629; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6630; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6631; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6632; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6633; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6634; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6635; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6636; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6637; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6638; GFX940-TGSPLIT-NEXT: s_endpgm 6639 i32* %out, i32 %in) { 6640entry: 6641 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire 6642 store i32 %val, i32* %out, align 4 6643 ret void 6644} 6645 6646define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw( 6647; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6648; GFX7: ; %bb.0: ; %entry 6649; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6650; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6651; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6652; GFX7-NEXT: v_mov_b32_e32 v0, s0 6653; GFX7-NEXT: v_mov_b32_e32 v1, s1 6654; GFX7-NEXT: v_mov_b32_e32 v2, s2 6655; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6656; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6657; GFX7-NEXT: flat_store_dword v[0:1], v2 6658; GFX7-NEXT: s_endpgm 6659; 6660; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6661; GFX10-WGP: ; %bb.0: ; %entry 6662; GFX10-WGP-NEXT: s_clause 0x1 6663; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6664; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6665; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6666; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6667; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6668; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6669; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6670; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6671; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6672; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6673; GFX10-WGP-NEXT: buffer_gl0_inv 6674; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6675; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 6676; GFX10-WGP-NEXT: s_endpgm 6677; 6678; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6679; GFX10-CU: ; %bb.0: ; %entry 6680; GFX10-CU-NEXT: s_clause 0x1 6681; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6682; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6683; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6684; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6685; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6686; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6687; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6688; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6689; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 6690; GFX10-CU-NEXT: s_endpgm 6691; 6692; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6693; SKIP-CACHE-INV: ; %bb.0: ; %entry 6694; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6695; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6696; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6697; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6698; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6699; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6700; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6701; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6702; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 6703; SKIP-CACHE-INV-NEXT: s_endpgm 6704; 6705; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6706; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6707; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6708; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6709; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6710; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6711; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6712; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6713; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6714; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6715; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6716; 6717; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6718; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6719; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6720; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6721; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6722; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6723; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6724; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6725; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6726; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6727; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6728; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6729; GFX90A-TGSPLIT-NEXT: s_endpgm 6730; 6731; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6732; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6733; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6734; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6735; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6736; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6737; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6738; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6739; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6740; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6741; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6742; 6743; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw: 6744; GFX940-TGSPLIT: ; %bb.0: ; %entry 6745; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6746; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6747; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6748; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6749; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6750; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6751; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6752; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6753; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6754; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6755; GFX940-TGSPLIT-NEXT: s_endpgm 6756 i32* %out, i32 %in) { 6757entry: 6758 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel 6759 store i32 %val, i32* %out, align 4 6760 ret void 6761} 6762 6763define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw( 6764; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6765; GFX7: ; %bb.0: ; %entry 6766; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6767; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2 6768; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6769; GFX7-NEXT: v_mov_b32_e32 v0, s0 6770; GFX7-NEXT: v_mov_b32_e32 v1, s1 6771; GFX7-NEXT: v_mov_b32_e32 v2, s2 6772; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6773; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6774; GFX7-NEXT: flat_store_dword v[0:1], v2 6775; GFX7-NEXT: s_endpgm 6776; 6777; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6778; GFX10-WGP: ; %bb.0: ; %entry 6779; GFX10-WGP-NEXT: s_clause 0x1 6780; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6781; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8 6782; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6783; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6784; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6785; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6786; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6787; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 6788; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6789; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 6790; GFX10-WGP-NEXT: buffer_gl0_inv 6791; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6792; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 6793; GFX10-WGP-NEXT: s_endpgm 6794; 6795; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6796; GFX10-CU: ; %bb.0: ; %entry 6797; GFX10-CU-NEXT: s_clause 0x1 6798; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6799; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8 6800; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6801; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6802; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6803; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6804; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6805; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6806; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 6807; GFX10-CU-NEXT: s_endpgm 6808; 6809; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6810; SKIP-CACHE-INV: ; %bb.0: ; %entry 6811; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6812; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2 6813; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6814; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6815; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6816; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6817; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6818; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6819; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 6820; SKIP-CACHE-INV-NEXT: s_endpgm 6821; 6822; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6823; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6824; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6825; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6826; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6827; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6828; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6829; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6830; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6831; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6832; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6833; 6834; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6835; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6836; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6837; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8 6838; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6839; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6840; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2 6841; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6842; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc 6843; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6844; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 6845; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6846; GFX90A-TGSPLIT-NEXT: s_endpgm 6847; 6848; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6849; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6850; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6851; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6852; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6853; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6854; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6855; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6856; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 6857; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 6858; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6859; 6860; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw: 6861; GFX940-TGSPLIT: ; %bb.0: ; %entry 6862; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6863; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8 6864; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6865; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6866; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4 6867; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6868; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0 6869; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 6870; GFX940-TGSPLIT-NEXT: buffer_inv sc0 6871; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 6872; GFX940-TGSPLIT-NEXT: s_endpgm 6873 i32* %out, i32 %in) { 6874entry: 6875 %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst 6876 store i32 %val, i32* %out, align 4 6877 ret void 6878} 6879 6880define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg( 6881; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6882; GFX7: ; %bb.0: ; %entry 6883; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6884; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 6885; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6886; GFX7-NEXT: s_add_u32 s0, s0, 16 6887; GFX7-NEXT: s_addc_u32 s1, s1, 0 6888; GFX7-NEXT: v_mov_b32_e32 v0, s0 6889; GFX7-NEXT: v_mov_b32_e32 v2, s2 6890; GFX7-NEXT: v_mov_b32_e32 v1, s1 6891; GFX7-NEXT: v_mov_b32_e32 v3, s3 6892; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 6893; GFX7-NEXT: s_endpgm 6894; 6895; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6896; GFX10-WGP: ; %bb.0: ; %entry 6897; GFX10-WGP-NEXT: s_clause 0x1 6898; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6899; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 6900; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 6901; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 6902; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 6903; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 6904; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 6905; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 6906; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 6907; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 6908; GFX10-WGP-NEXT: s_endpgm 6909; 6910; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6911; GFX10-CU: ; %bb.0: ; %entry 6912; GFX10-CU-NEXT: s_clause 0x1 6913; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6914; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 6915; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 6916; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 6917; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 6918; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 6919; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 6920; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 6921; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 6922; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 6923; GFX10-CU-NEXT: s_endpgm 6924; 6925; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6926; SKIP-CACHE-INV: ; %bb.0: ; %entry 6927; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6928; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 6929; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 6930; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 6931; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 6932; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 6933; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 6934; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 6935; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 6936; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 6937; SKIP-CACHE-INV-NEXT: s_endpgm 6938; 6939; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6940; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 6941; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6942; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 6943; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6944; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6945; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 6946; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 6947; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 6948; 6949; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6950; GFX90A-TGSPLIT: ; %bb.0: ; %entry 6951; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6952; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 6953; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6954; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 6955; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 6956; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 6957; GFX90A-TGSPLIT-NEXT: s_endpgm 6958; 6959; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6960; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 6961; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6962; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 6963; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6964; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6965; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 6966; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 6967; GFX940-NOTTGSPLIT-NEXT: s_endpgm 6968; 6969; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg: 6970; GFX940-TGSPLIT: ; %bb.0: ; %entry 6971; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 6972; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 6973; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 6974; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 6975; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 6976; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 6977; GFX940-TGSPLIT-NEXT: s_endpgm 6978 i32* %out, i32 %in, i32 %old) { 6979entry: 6980 %gep = getelementptr i32, i32* %out, i32 4 6981 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic 6982 ret void 6983} 6984 6985define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg( 6986; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 6987; GFX7: ; %bb.0: ; %entry 6988; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 6989; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 6990; GFX7-NEXT: s_waitcnt lgkmcnt(0) 6991; GFX7-NEXT: s_add_u32 s0, s0, 16 6992; GFX7-NEXT: s_addc_u32 s1, s1, 0 6993; GFX7-NEXT: v_mov_b32_e32 v0, s0 6994; GFX7-NEXT: v_mov_b32_e32 v2, s2 6995; GFX7-NEXT: v_mov_b32_e32 v1, s1 6996; GFX7-NEXT: v_mov_b32_e32 v3, s3 6997; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 6998; GFX7-NEXT: s_endpgm 6999; 7000; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7001; GFX10-WGP: ; %bb.0: ; %entry 7002; GFX10-WGP-NEXT: s_clause 0x1 7003; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7004; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7005; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7006; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7007; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7008; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7009; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7010; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7011; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7012; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7013; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7014; GFX10-WGP-NEXT: buffer_gl0_inv 7015; GFX10-WGP-NEXT: s_endpgm 7016; 7017; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7018; GFX10-CU: ; %bb.0: ; %entry 7019; GFX10-CU-NEXT: s_clause 0x1 7020; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7021; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7022; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7023; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7024; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7025; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7026; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7027; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7028; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7029; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7030; GFX10-CU-NEXT: s_endpgm 7031; 7032; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7033; SKIP-CACHE-INV: ; %bb.0: ; %entry 7034; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7035; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7036; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7037; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7038; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7039; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7040; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7041; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7042; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7043; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7044; SKIP-CACHE-INV-NEXT: s_endpgm 7045; 7046; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7047; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7048; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7049; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7050; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7051; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7052; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7053; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7054; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7055; 7056; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7057; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7058; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7059; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7060; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7061; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7062; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7063; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7064; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7065; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7066; GFX90A-TGSPLIT-NEXT: s_endpgm 7067; 7068; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7069; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7070; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7071; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7072; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7073; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7074; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7075; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7076; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7077; 7078; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg: 7079; GFX940-TGSPLIT: ; %bb.0: ; %entry 7080; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7081; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7082; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7083; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7084; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7085; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7086; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7087; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7088; GFX940-TGSPLIT-NEXT: s_endpgm 7089 i32* %out, i32 %in, i32 %old) { 7090entry: 7091 %gep = getelementptr i32, i32* %out, i32 4 7092 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic 7093 ret void 7094} 7095 7096define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg( 7097; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7098; GFX7: ; %bb.0: ; %entry 7099; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7100; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7101; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7102; GFX7-NEXT: s_add_u32 s0, s0, 16 7103; GFX7-NEXT: s_addc_u32 s1, s1, 0 7104; GFX7-NEXT: v_mov_b32_e32 v0, s0 7105; GFX7-NEXT: v_mov_b32_e32 v2, s2 7106; GFX7-NEXT: v_mov_b32_e32 v1, s1 7107; GFX7-NEXT: v_mov_b32_e32 v3, s3 7108; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7109; GFX7-NEXT: s_endpgm 7110; 7111; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7112; GFX10-WGP: ; %bb.0: ; %entry 7113; GFX10-WGP-NEXT: s_clause 0x1 7114; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7115; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7116; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7117; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7118; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7119; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7120; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7121; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7122; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7123; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7124; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7125; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7126; GFX10-WGP-NEXT: s_endpgm 7127; 7128; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7129; GFX10-CU: ; %bb.0: ; %entry 7130; GFX10-CU-NEXT: s_clause 0x1 7131; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7132; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7133; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7134; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7135; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7136; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7137; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7138; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7139; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7140; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7141; GFX10-CU-NEXT: s_endpgm 7142; 7143; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7144; SKIP-CACHE-INV: ; %bb.0: ; %entry 7145; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7146; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7147; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7148; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7149; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7150; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7151; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7152; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7153; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7154; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7155; SKIP-CACHE-INV-NEXT: s_endpgm 7156; 7157; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7158; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7159; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7160; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7161; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7162; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7163; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7164; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7165; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7166; 7167; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7168; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7169; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7170; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7171; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7172; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7173; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7174; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7175; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7176; GFX90A-TGSPLIT-NEXT: s_endpgm 7177; 7178; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7179; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7180; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7181; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7182; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7183; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7184; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7185; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7186; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7187; 7188; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg: 7189; GFX940-TGSPLIT: ; %bb.0: ; %entry 7190; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7191; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7192; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7193; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7194; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7195; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7196; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7197; GFX940-TGSPLIT-NEXT: s_endpgm 7198 i32* %out, i32 %in, i32 %old) { 7199entry: 7200 %gep = getelementptr i32, i32* %out, i32 4 7201 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic 7202 ret void 7203} 7204 7205define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg( 7206; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7207; GFX7: ; %bb.0: ; %entry 7208; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7209; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7210; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7211; GFX7-NEXT: s_add_u32 s0, s0, 16 7212; GFX7-NEXT: s_addc_u32 s1, s1, 0 7213; GFX7-NEXT: v_mov_b32_e32 v0, s0 7214; GFX7-NEXT: v_mov_b32_e32 v2, s2 7215; GFX7-NEXT: v_mov_b32_e32 v1, s1 7216; GFX7-NEXT: v_mov_b32_e32 v3, s3 7217; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7218; GFX7-NEXT: s_endpgm 7219; 7220; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7221; GFX10-WGP: ; %bb.0: ; %entry 7222; GFX10-WGP-NEXT: s_clause 0x1 7223; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7224; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7225; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7226; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7227; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7228; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7229; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7230; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7231; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7232; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7233; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7234; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7235; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7236; GFX10-WGP-NEXT: buffer_gl0_inv 7237; GFX10-WGP-NEXT: s_endpgm 7238; 7239; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7240; GFX10-CU: ; %bb.0: ; %entry 7241; GFX10-CU-NEXT: s_clause 0x1 7242; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7243; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7244; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7245; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7246; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7247; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7248; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7249; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7250; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7251; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7252; GFX10-CU-NEXT: s_endpgm 7253; 7254; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7255; SKIP-CACHE-INV: ; %bb.0: ; %entry 7256; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7257; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7258; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7259; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7260; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7261; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7262; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7263; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7264; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7265; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7266; SKIP-CACHE-INV-NEXT: s_endpgm 7267; 7268; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7269; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7270; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7271; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7272; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7273; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7274; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7275; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7276; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7277; 7278; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7279; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7280; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7281; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7282; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7283; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7284; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7285; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7286; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7287; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7288; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7289; GFX90A-TGSPLIT-NEXT: s_endpgm 7290; 7291; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7292; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7293; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7294; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7295; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7296; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7297; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7298; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7299; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7300; 7301; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg: 7302; GFX940-TGSPLIT: ; %bb.0: ; %entry 7303; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7304; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7305; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7306; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7307; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7308; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7309; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7310; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7311; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7312; GFX940-TGSPLIT-NEXT: s_endpgm 7313 i32* %out, i32 %in, i32 %old) { 7314entry: 7315 %gep = getelementptr i32, i32* %out, i32 4 7316 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic 7317 ret void 7318} 7319 7320define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg( 7321; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7322; GFX7: ; %bb.0: ; %entry 7323; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7324; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7325; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7326; GFX7-NEXT: s_add_u32 s0, s0, 16 7327; GFX7-NEXT: s_addc_u32 s1, s1, 0 7328; GFX7-NEXT: v_mov_b32_e32 v0, s0 7329; GFX7-NEXT: v_mov_b32_e32 v2, s2 7330; GFX7-NEXT: v_mov_b32_e32 v1, s1 7331; GFX7-NEXT: v_mov_b32_e32 v3, s3 7332; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7333; GFX7-NEXT: s_endpgm 7334; 7335; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7336; GFX10-WGP: ; %bb.0: ; %entry 7337; GFX10-WGP-NEXT: s_clause 0x1 7338; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7339; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7340; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7341; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7342; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7343; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7344; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7345; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7346; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7347; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7348; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7349; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7350; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7351; GFX10-WGP-NEXT: buffer_gl0_inv 7352; GFX10-WGP-NEXT: s_endpgm 7353; 7354; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7355; GFX10-CU: ; %bb.0: ; %entry 7356; GFX10-CU-NEXT: s_clause 0x1 7357; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7358; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7359; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7360; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7361; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7362; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7363; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7364; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7365; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7366; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7367; GFX10-CU-NEXT: s_endpgm 7368; 7369; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7370; SKIP-CACHE-INV: ; %bb.0: ; %entry 7371; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7372; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7373; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7374; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7375; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7376; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7377; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7378; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7379; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7380; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7381; SKIP-CACHE-INV-NEXT: s_endpgm 7382; 7383; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7384; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7385; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7386; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7387; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7388; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7389; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7390; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7391; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7392; 7393; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7394; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7395; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7396; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7397; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7398; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7399; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7400; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7401; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7402; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7403; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7404; GFX90A-TGSPLIT-NEXT: s_endpgm 7405; 7406; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7407; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7408; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7409; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7410; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7411; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7412; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7413; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7414; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7415; 7416; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg: 7417; GFX940-TGSPLIT: ; %bb.0: ; %entry 7418; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7419; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7420; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7421; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7422; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7423; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7424; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7425; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7426; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7427; GFX940-TGSPLIT-NEXT: s_endpgm 7428 i32* %out, i32 %in, i32 %old) { 7429entry: 7430 %gep = getelementptr i32, i32* %out, i32 4 7431 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic 7432 ret void 7433} 7434 7435define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg( 7436; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7437; GFX7: ; %bb.0: ; %entry 7438; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7439; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7440; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7441; GFX7-NEXT: s_add_u32 s0, s0, 16 7442; GFX7-NEXT: s_addc_u32 s1, s1, 0 7443; GFX7-NEXT: v_mov_b32_e32 v0, s0 7444; GFX7-NEXT: v_mov_b32_e32 v2, s2 7445; GFX7-NEXT: v_mov_b32_e32 v1, s1 7446; GFX7-NEXT: v_mov_b32_e32 v3, s3 7447; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7448; GFX7-NEXT: s_endpgm 7449; 7450; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7451; GFX10-WGP: ; %bb.0: ; %entry 7452; GFX10-WGP-NEXT: s_clause 0x1 7453; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7454; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7455; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7456; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7457; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7458; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7459; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7460; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7461; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7462; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7463; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7464; GFX10-WGP-NEXT: buffer_gl0_inv 7465; GFX10-WGP-NEXT: s_endpgm 7466; 7467; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7468; GFX10-CU: ; %bb.0: ; %entry 7469; GFX10-CU-NEXT: s_clause 0x1 7470; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7471; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7472; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7473; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7474; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7475; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7476; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7477; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7478; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7479; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7480; GFX10-CU-NEXT: s_endpgm 7481; 7482; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7483; SKIP-CACHE-INV: ; %bb.0: ; %entry 7484; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7485; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7486; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7487; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7488; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7489; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7490; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7491; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7492; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7493; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7494; SKIP-CACHE-INV-NEXT: s_endpgm 7495; 7496; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7497; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7498; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7499; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7500; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7501; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7502; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7503; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7504; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7505; 7506; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7507; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7508; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7509; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7510; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7511; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7512; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7513; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7514; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7515; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7516; GFX90A-TGSPLIT-NEXT: s_endpgm 7517; 7518; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7519; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7520; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7521; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7522; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7523; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7524; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7525; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7526; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7527; 7528; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg: 7529; GFX940-TGSPLIT: ; %bb.0: ; %entry 7530; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7531; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7532; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7533; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7534; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7535; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7536; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7537; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7538; GFX940-TGSPLIT-NEXT: s_endpgm 7539 i32* %out, i32 %in, i32 %old) { 7540entry: 7541 %gep = getelementptr i32, i32* %out, i32 4 7542 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire 7543 ret void 7544} 7545 7546define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg( 7547; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7548; GFX7: ; %bb.0: ; %entry 7549; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7550; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7551; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7552; GFX7-NEXT: s_add_u32 s0, s0, 16 7553; GFX7-NEXT: s_addc_u32 s1, s1, 0 7554; GFX7-NEXT: v_mov_b32_e32 v0, s0 7555; GFX7-NEXT: v_mov_b32_e32 v2, s2 7556; GFX7-NEXT: v_mov_b32_e32 v1, s1 7557; GFX7-NEXT: v_mov_b32_e32 v3, s3 7558; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7559; GFX7-NEXT: s_endpgm 7560; 7561; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7562; GFX10-WGP: ; %bb.0: ; %entry 7563; GFX10-WGP-NEXT: s_clause 0x1 7564; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7565; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7566; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7567; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7568; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7569; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7570; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7571; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7572; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7573; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7574; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7575; GFX10-WGP-NEXT: buffer_gl0_inv 7576; GFX10-WGP-NEXT: s_endpgm 7577; 7578; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7579; GFX10-CU: ; %bb.0: ; %entry 7580; GFX10-CU-NEXT: s_clause 0x1 7581; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7582; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7583; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7584; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7585; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7586; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7587; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7588; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7589; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7590; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7591; GFX10-CU-NEXT: s_endpgm 7592; 7593; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7594; SKIP-CACHE-INV: ; %bb.0: ; %entry 7595; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7596; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7597; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7598; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7599; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7600; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7601; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7602; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7603; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7604; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7605; SKIP-CACHE-INV-NEXT: s_endpgm 7606; 7607; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7608; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7609; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7610; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7611; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7612; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7613; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7614; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7615; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7616; 7617; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7618; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7619; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7620; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7621; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7622; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7623; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7624; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7625; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7626; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7627; GFX90A-TGSPLIT-NEXT: s_endpgm 7628; 7629; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7630; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7631; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7632; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7633; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7634; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7635; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7636; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7637; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7638; 7639; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg: 7640; GFX940-TGSPLIT: ; %bb.0: ; %entry 7641; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7642; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7643; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7644; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7645; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7646; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7647; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7648; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7649; GFX940-TGSPLIT-NEXT: s_endpgm 7650 i32* %out, i32 %in, i32 %old) { 7651entry: 7652 %gep = getelementptr i32, i32* %out, i32 4 7653 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire 7654 ret void 7655} 7656 7657define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg( 7658; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7659; GFX7: ; %bb.0: ; %entry 7660; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7661; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7662; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7663; GFX7-NEXT: s_add_u32 s0, s0, 16 7664; GFX7-NEXT: s_addc_u32 s1, s1, 0 7665; GFX7-NEXT: v_mov_b32_e32 v0, s0 7666; GFX7-NEXT: v_mov_b32_e32 v2, s2 7667; GFX7-NEXT: v_mov_b32_e32 v1, s1 7668; GFX7-NEXT: v_mov_b32_e32 v3, s3 7669; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7670; GFX7-NEXT: s_endpgm 7671; 7672; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7673; GFX10-WGP: ; %bb.0: ; %entry 7674; GFX10-WGP-NEXT: s_clause 0x1 7675; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7676; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7677; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7678; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7679; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7680; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7681; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7682; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7683; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7684; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7685; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7686; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7687; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7688; GFX10-WGP-NEXT: buffer_gl0_inv 7689; GFX10-WGP-NEXT: s_endpgm 7690; 7691; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7692; GFX10-CU: ; %bb.0: ; %entry 7693; GFX10-CU-NEXT: s_clause 0x1 7694; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7695; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7696; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7697; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7698; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7699; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7700; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7701; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7702; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7703; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7704; GFX10-CU-NEXT: s_endpgm 7705; 7706; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7707; SKIP-CACHE-INV: ; %bb.0: ; %entry 7708; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7709; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7710; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7711; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7712; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7713; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7714; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7715; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7716; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7717; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7718; SKIP-CACHE-INV-NEXT: s_endpgm 7719; 7720; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7721; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7722; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7723; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7724; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7725; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7726; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7727; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7728; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7729; 7730; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7731; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7732; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7733; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7734; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7735; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7736; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7737; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7738; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7739; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7740; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7741; GFX90A-TGSPLIT-NEXT: s_endpgm 7742; 7743; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7744; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7745; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7746; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7747; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7748; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7749; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7750; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7751; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7752; 7753; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg: 7754; GFX940-TGSPLIT: ; %bb.0: ; %entry 7755; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7756; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7757; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7758; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7759; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7760; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7761; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7762; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7763; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7764; GFX940-TGSPLIT-NEXT: s_endpgm 7765 i32* %out, i32 %in, i32 %old) { 7766entry: 7767 %gep = getelementptr i32, i32* %out, i32 4 7768 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire 7769 ret void 7770} 7771 7772define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg( 7773; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7774; GFX7: ; %bb.0: ; %entry 7775; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7776; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7777; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7778; GFX7-NEXT: s_add_u32 s0, s0, 16 7779; GFX7-NEXT: s_addc_u32 s1, s1, 0 7780; GFX7-NEXT: v_mov_b32_e32 v0, s0 7781; GFX7-NEXT: v_mov_b32_e32 v2, s2 7782; GFX7-NEXT: v_mov_b32_e32 v1, s1 7783; GFX7-NEXT: v_mov_b32_e32 v3, s3 7784; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7785; GFX7-NEXT: s_endpgm 7786; 7787; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7788; GFX10-WGP: ; %bb.0: ; %entry 7789; GFX10-WGP-NEXT: s_clause 0x1 7790; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7791; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7792; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7793; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7794; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7795; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7796; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7797; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7798; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7799; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7800; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7801; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7802; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7803; GFX10-WGP-NEXT: buffer_gl0_inv 7804; GFX10-WGP-NEXT: s_endpgm 7805; 7806; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7807; GFX10-CU: ; %bb.0: ; %entry 7808; GFX10-CU-NEXT: s_clause 0x1 7809; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7810; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7811; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7812; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7813; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7814; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7815; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7816; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7817; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7818; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7819; GFX10-CU-NEXT: s_endpgm 7820; 7821; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7822; SKIP-CACHE-INV: ; %bb.0: ; %entry 7823; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7824; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7825; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7826; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7827; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7828; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7829; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7830; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7831; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7832; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7833; SKIP-CACHE-INV-NEXT: s_endpgm 7834; 7835; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7836; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7837; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7838; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7839; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7840; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7841; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7842; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7843; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7844; 7845; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7846; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7847; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7848; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7849; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7850; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7851; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7852; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7853; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7854; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7855; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7856; GFX90A-TGSPLIT-NEXT: s_endpgm 7857; 7858; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7859; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7860; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7861; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7862; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7863; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7864; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7865; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7866; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7867; 7868; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg: 7869; GFX940-TGSPLIT: ; %bb.0: ; %entry 7870; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7871; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7872; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7873; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7874; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7875; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7876; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7877; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7878; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7879; GFX940-TGSPLIT-NEXT: s_endpgm 7880 i32* %out, i32 %in, i32 %old) { 7881entry: 7882 %gep = getelementptr i32, i32* %out, i32 4 7883 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire 7884 ret void 7885} 7886 7887define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg( 7888; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7889; GFX7: ; %bb.0: ; %entry 7890; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7891; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 7892; GFX7-NEXT: s_waitcnt lgkmcnt(0) 7893; GFX7-NEXT: s_add_u32 s0, s0, 16 7894; GFX7-NEXT: s_addc_u32 s1, s1, 0 7895; GFX7-NEXT: v_mov_b32_e32 v0, s0 7896; GFX7-NEXT: v_mov_b32_e32 v2, s2 7897; GFX7-NEXT: v_mov_b32_e32 v1, s1 7898; GFX7-NEXT: v_mov_b32_e32 v3, s3 7899; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7900; GFX7-NEXT: s_endpgm 7901; 7902; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7903; GFX10-WGP: ; %bb.0: ; %entry 7904; GFX10-WGP-NEXT: s_clause 0x1 7905; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7906; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7907; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 7908; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 7909; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 7910; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 7911; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 7912; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 7913; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 7914; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 7915; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7916; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7917; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 7918; GFX10-WGP-NEXT: buffer_gl0_inv 7919; GFX10-WGP-NEXT: s_endpgm 7920; 7921; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7922; GFX10-CU: ; %bb.0: ; %entry 7923; GFX10-CU-NEXT: s_clause 0x1 7924; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7925; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7926; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 7927; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 7928; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 7929; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 7930; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 7931; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 7932; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 7933; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7934; GFX10-CU-NEXT: s_endpgm 7935; 7936; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7937; SKIP-CACHE-INV: ; %bb.0: ; %entry 7938; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7939; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 7940; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 7941; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 7942; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 7943; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 7944; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 7945; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 7946; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 7947; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 7948; SKIP-CACHE-INV-NEXT: s_endpgm 7949; 7950; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7951; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 7952; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7953; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7954; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7955; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7956; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7957; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7958; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 7959; 7960; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7961; GFX90A-TGSPLIT: ; %bb.0: ; %entry 7962; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 7963; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 7964; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7965; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 7966; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 7967; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7968; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7969; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7970; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 7971; GFX90A-TGSPLIT-NEXT: s_endpgm 7972; 7973; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7974; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 7975; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7976; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7977; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7978; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7979; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7980; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7981; GFX940-NOTTGSPLIT-NEXT: s_endpgm 7982; 7983; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg: 7984; GFX940-TGSPLIT: ; %bb.0: ; %entry 7985; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 7986; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 7987; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 7988; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 7989; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 7990; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7991; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 7992; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 7993; GFX940-TGSPLIT-NEXT: buffer_inv sc0 7994; GFX940-TGSPLIT-NEXT: s_endpgm 7995 i32* %out, i32 %in, i32 %old) { 7996entry: 7997 %gep = getelementptr i32, i32* %out, i32 4 7998 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire 7999 ret void 8000} 8001 8002define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg( 8003; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8004; GFX7: ; %bb.0: ; %entry 8005; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8006; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8007; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8008; GFX7-NEXT: s_add_u32 s0, s0, 16 8009; GFX7-NEXT: s_addc_u32 s1, s1, 0 8010; GFX7-NEXT: v_mov_b32_e32 v0, s0 8011; GFX7-NEXT: v_mov_b32_e32 v2, s2 8012; GFX7-NEXT: v_mov_b32_e32 v1, s1 8013; GFX7-NEXT: v_mov_b32_e32 v3, s3 8014; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8015; GFX7-NEXT: s_endpgm 8016; 8017; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8018; GFX10-WGP: ; %bb.0: ; %entry 8019; GFX10-WGP-NEXT: s_clause 0x1 8020; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8021; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8022; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8023; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 8024; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 8025; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8026; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8027; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8028; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8029; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8030; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8031; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8032; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8033; GFX10-WGP-NEXT: buffer_gl0_inv 8034; GFX10-WGP-NEXT: s_endpgm 8035; 8036; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8037; GFX10-CU: ; %bb.0: ; %entry 8038; GFX10-CU-NEXT: s_clause 0x1 8039; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8040; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8041; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8042; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 8043; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 8044; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8045; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8046; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8047; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8048; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8049; GFX10-CU-NEXT: s_endpgm 8050; 8051; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8052; SKIP-CACHE-INV: ; %bb.0: ; %entry 8053; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8054; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8055; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8056; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 8057; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 8058; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8059; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8060; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8061; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8062; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8063; SKIP-CACHE-INV-NEXT: s_endpgm 8064; 8065; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8066; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8067; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8068; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8069; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8070; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8071; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8072; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8073; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8074; 8075; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8076; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8077; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8078; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8079; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8080; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8081; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8082; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8083; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8084; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8085; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8086; GFX90A-TGSPLIT-NEXT: s_endpgm 8087; 8088; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8089; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8090; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8091; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8092; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8093; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8094; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8095; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8096; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8097; 8098; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg: 8099; GFX940-TGSPLIT: ; %bb.0: ; %entry 8100; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8101; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8102; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8103; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8104; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8105; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8106; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8107; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8108; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8109; GFX940-TGSPLIT-NEXT: s_endpgm 8110 i32* %out, i32 %in, i32 %old) { 8111entry: 8112 %gep = getelementptr i32, i32* %out, i32 4 8113 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst 8114 ret void 8115} 8116 8117define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg( 8118; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8119; GFX7: ; %bb.0: ; %entry 8120; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8121; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8122; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8123; GFX7-NEXT: s_add_u32 s0, s0, 16 8124; GFX7-NEXT: s_addc_u32 s1, s1, 0 8125; GFX7-NEXT: v_mov_b32_e32 v0, s0 8126; GFX7-NEXT: v_mov_b32_e32 v2, s2 8127; GFX7-NEXT: v_mov_b32_e32 v1, s1 8128; GFX7-NEXT: v_mov_b32_e32 v3, s3 8129; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8130; GFX7-NEXT: s_endpgm 8131; 8132; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8133; GFX10-WGP: ; %bb.0: ; %entry 8134; GFX10-WGP-NEXT: s_clause 0x1 8135; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8136; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8137; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8138; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 8139; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 8140; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8141; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8142; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8143; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8144; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8145; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8146; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8147; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8148; GFX10-WGP-NEXT: buffer_gl0_inv 8149; GFX10-WGP-NEXT: s_endpgm 8150; 8151; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8152; GFX10-CU: ; %bb.0: ; %entry 8153; GFX10-CU-NEXT: s_clause 0x1 8154; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8155; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8156; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8157; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 8158; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 8159; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8160; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8161; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8162; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8163; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8164; GFX10-CU-NEXT: s_endpgm 8165; 8166; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8167; SKIP-CACHE-INV: ; %bb.0: ; %entry 8168; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8169; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8170; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8171; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 8172; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 8173; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8174; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8175; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8176; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8177; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8178; SKIP-CACHE-INV-NEXT: s_endpgm 8179; 8180; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8181; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8182; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8183; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8184; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8185; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8186; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8187; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8188; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8189; 8190; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8191; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8192; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8193; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8194; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8195; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8196; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8197; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8198; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8199; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8200; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8201; GFX90A-TGSPLIT-NEXT: s_endpgm 8202; 8203; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8204; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8205; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8206; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8207; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8208; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8209; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8210; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8211; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8212; 8213; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg: 8214; GFX940-TGSPLIT: ; %bb.0: ; %entry 8215; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8216; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8217; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8218; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8219; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8220; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8221; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8222; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8223; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8224; GFX940-TGSPLIT-NEXT: s_endpgm 8225 i32* %out, i32 %in, i32 %old) { 8226entry: 8227 %gep = getelementptr i32, i32* %out, i32 4 8228 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst 8229 ret void 8230} 8231 8232define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg( 8233; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8234; GFX7: ; %bb.0: ; %entry 8235; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8236; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8237; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8238; GFX7-NEXT: s_add_u32 s0, s0, 16 8239; GFX7-NEXT: s_addc_u32 s1, s1, 0 8240; GFX7-NEXT: v_mov_b32_e32 v0, s0 8241; GFX7-NEXT: v_mov_b32_e32 v2, s2 8242; GFX7-NEXT: v_mov_b32_e32 v1, s1 8243; GFX7-NEXT: v_mov_b32_e32 v3, s3 8244; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8245; GFX7-NEXT: s_endpgm 8246; 8247; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8248; GFX10-WGP: ; %bb.0: ; %entry 8249; GFX10-WGP-NEXT: s_clause 0x1 8250; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8251; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8252; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8253; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 8254; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 8255; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8256; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8257; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8258; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8259; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8260; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8261; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8262; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8263; GFX10-WGP-NEXT: buffer_gl0_inv 8264; GFX10-WGP-NEXT: s_endpgm 8265; 8266; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8267; GFX10-CU: ; %bb.0: ; %entry 8268; GFX10-CU-NEXT: s_clause 0x1 8269; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8270; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8271; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8272; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 8273; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 8274; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8275; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8276; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8277; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8278; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8279; GFX10-CU-NEXT: s_endpgm 8280; 8281; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8282; SKIP-CACHE-INV: ; %bb.0: ; %entry 8283; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8284; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8285; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8286; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 8287; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 8288; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8289; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8290; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8291; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8292; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8293; SKIP-CACHE-INV-NEXT: s_endpgm 8294; 8295; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8296; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8297; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8298; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8299; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8300; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8301; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8302; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8303; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8304; 8305; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8306; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8307; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8308; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8309; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8310; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8311; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8312; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8313; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8314; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8315; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8316; GFX90A-TGSPLIT-NEXT: s_endpgm 8317; 8318; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8319; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8320; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8321; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8322; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8323; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8324; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8325; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8326; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8327; 8328; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg: 8329; GFX940-TGSPLIT: ; %bb.0: ; %entry 8330; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8331; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8332; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8333; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8334; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8335; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8336; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8337; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8338; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8339; GFX940-TGSPLIT-NEXT: s_endpgm 8340 i32* %out, i32 %in, i32 %old) { 8341entry: 8342 %gep = getelementptr i32, i32* %out, i32 4 8343 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst 8344 ret void 8345} 8346 8347define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg( 8348; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8349; GFX7: ; %bb.0: ; %entry 8350; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8351; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8352; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8353; GFX7-NEXT: s_add_u32 s0, s0, 16 8354; GFX7-NEXT: s_addc_u32 s1, s1, 0 8355; GFX7-NEXT: v_mov_b32_e32 v0, s0 8356; GFX7-NEXT: v_mov_b32_e32 v2, s2 8357; GFX7-NEXT: v_mov_b32_e32 v1, s1 8358; GFX7-NEXT: v_mov_b32_e32 v3, s3 8359; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8360; GFX7-NEXT: s_endpgm 8361; 8362; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8363; GFX10-WGP: ; %bb.0: ; %entry 8364; GFX10-WGP-NEXT: s_clause 0x1 8365; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8366; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8367; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8368; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 8369; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 8370; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8371; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8372; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8373; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8374; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8375; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8376; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8377; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8378; GFX10-WGP-NEXT: buffer_gl0_inv 8379; GFX10-WGP-NEXT: s_endpgm 8380; 8381; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8382; GFX10-CU: ; %bb.0: ; %entry 8383; GFX10-CU-NEXT: s_clause 0x1 8384; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8385; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8386; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8387; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 8388; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 8389; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8390; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8391; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8392; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8393; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8394; GFX10-CU-NEXT: s_endpgm 8395; 8396; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8397; SKIP-CACHE-INV: ; %bb.0: ; %entry 8398; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8399; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8400; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8401; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 8402; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 8403; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8404; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8405; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8406; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8407; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8408; SKIP-CACHE-INV-NEXT: s_endpgm 8409; 8410; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8411; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8412; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8413; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8414; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8415; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8416; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8417; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8418; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8419; 8420; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8421; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8422; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8423; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8424; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8425; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8426; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8427; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8428; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8429; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8430; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8431; GFX90A-TGSPLIT-NEXT: s_endpgm 8432; 8433; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8434; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8435; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8436; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8437; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8438; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8439; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8440; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8441; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8442; 8443; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg: 8444; GFX940-TGSPLIT: ; %bb.0: ; %entry 8445; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8446; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8447; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8448; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8449; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8450; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8451; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8452; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8453; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8454; GFX940-TGSPLIT-NEXT: s_endpgm 8455 i32* %out, i32 %in, i32 %old) { 8456entry: 8457 %gep = getelementptr i32, i32* %out, i32 4 8458 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst 8459 ret void 8460} 8461 8462define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg( 8463; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8464; GFX7: ; %bb.0: ; %entry 8465; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8466; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8467; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8468; GFX7-NEXT: s_add_u32 s0, s0, 16 8469; GFX7-NEXT: s_addc_u32 s1, s1, 0 8470; GFX7-NEXT: v_mov_b32_e32 v0, s0 8471; GFX7-NEXT: v_mov_b32_e32 v2, s2 8472; GFX7-NEXT: v_mov_b32_e32 v1, s1 8473; GFX7-NEXT: v_mov_b32_e32 v3, s3 8474; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8475; GFX7-NEXT: s_endpgm 8476; 8477; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8478; GFX10-WGP: ; %bb.0: ; %entry 8479; GFX10-WGP-NEXT: s_clause 0x1 8480; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8481; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8482; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8483; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16 8484; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0 8485; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8486; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8487; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8488; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8489; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8490; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8491; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8492; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8493; GFX10-WGP-NEXT: buffer_gl0_inv 8494; GFX10-WGP-NEXT: s_endpgm 8495; 8496; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8497; GFX10-CU: ; %bb.0: ; %entry 8498; GFX10-CU-NEXT: s_clause 0x1 8499; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8500; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8501; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8502; GFX10-CU-NEXT: s_add_u32 s0, s0, 16 8503; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0 8504; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8505; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8506; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8507; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8508; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8509; GFX10-CU-NEXT: s_endpgm 8510; 8511; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8512; SKIP-CACHE-INV: ; %bb.0: ; %entry 8513; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8514; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8515; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8516; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, 16 8517; SKIP-CACHE-INV-NEXT: s_addc_u32 s3, s3, 0 8518; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8519; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8520; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8521; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8522; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] 8523; SKIP-CACHE-INV-NEXT: s_endpgm 8524; 8525; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8526; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8527; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8528; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8529; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8530; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8531; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8532; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8533; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8534; 8535; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8536; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8537; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8538; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8539; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8540; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8541; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8542; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8543; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8544; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8545; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8546; GFX90A-TGSPLIT-NEXT: s_endpgm 8547; 8548; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8549; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8550; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8551; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8552; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8553; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8554; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8555; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8556; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8557; 8558; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg: 8559; GFX940-TGSPLIT: ; %bb.0: ; %entry 8560; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8561; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8562; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8563; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8564; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8565; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8566; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16 8567; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8568; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8569; GFX940-TGSPLIT-NEXT: s_endpgm 8570 i32* %out, i32 %in, i32 %old) { 8571entry: 8572 %gep = getelementptr i32, i32* %out, i32 4 8573 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst 8574 ret void 8575} 8576 8577define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg( 8578; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8579; GFX7: ; %bb.0: ; %entry 8580; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8581; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8582; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8583; GFX7-NEXT: s_add_u32 s4, s0, 16 8584; GFX7-NEXT: s_addc_u32 s5, s1, 0 8585; GFX7-NEXT: v_mov_b32_e32 v0, s4 8586; GFX7-NEXT: v_mov_b32_e32 v2, s2 8587; GFX7-NEXT: v_mov_b32_e32 v1, s5 8588; GFX7-NEXT: v_mov_b32_e32 v3, s3 8589; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8590; GFX7-NEXT: v_mov_b32_e32 v0, s0 8591; GFX7-NEXT: v_mov_b32_e32 v1, s1 8592; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8593; GFX7-NEXT: flat_store_dword v[0:1], v2 8594; GFX7-NEXT: s_endpgm 8595; 8596; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8597; GFX10-WGP: ; %bb.0: ; %entry 8598; GFX10-WGP-NEXT: s_clause 0x1 8599; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8600; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8601; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8602; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 8603; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 8604; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 8605; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8606; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 8607; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8608; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8609; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8610; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8611; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8612; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 8613; GFX10-WGP-NEXT: s_endpgm 8614; 8615; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8616; GFX10-CU: ; %bb.0: ; %entry 8617; GFX10-CU-NEXT: s_clause 0x1 8618; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8619; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8620; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8621; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 8622; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 8623; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 8624; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8625; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 8626; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8627; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8628; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8629; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8630; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8631; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 8632; GFX10-CU-NEXT: s_endpgm 8633; 8634; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8635; SKIP-CACHE-INV: ; %bb.0: ; %entry 8636; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8637; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8638; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8639; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 8640; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 8641; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 8642; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8643; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 8644; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8645; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8646; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8647; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8648; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8649; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 8650; SKIP-CACHE-INV-NEXT: s_endpgm 8651; 8652; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8653; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8654; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8655; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8656; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8657; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8658; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8659; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8660; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8661; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8662; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8663; 8664; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8665; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8666; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8667; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8668; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8669; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8670; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8671; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8672; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8673; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8674; GFX90A-TGSPLIT-NEXT: s_endpgm 8675; 8676; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8677; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8678; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8679; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8680; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8681; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8682; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8683; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8684; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8685; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8686; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8687; 8688; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg: 8689; GFX940-TGSPLIT: ; %bb.0: ; %entry 8690; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8691; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8692; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8693; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8694; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8695; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8696; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8697; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8698; GFX940-TGSPLIT-NEXT: s_endpgm 8699 i32* %out, i32 %in, i32 %old) { 8700entry: 8701 %gep = getelementptr i32, i32* %out, i32 4 8702 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic 8703 %val0 = extractvalue { i32, i1 } %val, 0 8704 store i32 %val0, i32* %out, align 4 8705 ret void 8706} 8707 8708define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg( 8709; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8710; GFX7: ; %bb.0: ; %entry 8711; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8712; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8713; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8714; GFX7-NEXT: s_add_u32 s4, s0, 16 8715; GFX7-NEXT: s_addc_u32 s5, s1, 0 8716; GFX7-NEXT: v_mov_b32_e32 v0, s4 8717; GFX7-NEXT: v_mov_b32_e32 v2, s2 8718; GFX7-NEXT: v_mov_b32_e32 v1, s5 8719; GFX7-NEXT: v_mov_b32_e32 v3, s3 8720; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8721; GFX7-NEXT: v_mov_b32_e32 v0, s0 8722; GFX7-NEXT: v_mov_b32_e32 v1, s1 8723; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8724; GFX7-NEXT: flat_store_dword v[0:1], v2 8725; GFX7-NEXT: s_endpgm 8726; 8727; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8728; GFX10-WGP: ; %bb.0: ; %entry 8729; GFX10-WGP-NEXT: s_clause 0x1 8730; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8731; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8732; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8733; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 8734; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 8735; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 8736; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8737; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 8738; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8739; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8740; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8741; GFX10-WGP-NEXT: buffer_gl0_inv 8742; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8743; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8744; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8745; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 8746; GFX10-WGP-NEXT: s_endpgm 8747; 8748; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8749; GFX10-CU: ; %bb.0: ; %entry 8750; GFX10-CU-NEXT: s_clause 0x1 8751; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8752; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8753; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8754; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 8755; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 8756; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 8757; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8758; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 8759; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8760; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8761; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8762; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8763; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8764; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 8765; GFX10-CU-NEXT: s_endpgm 8766; 8767; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8768; SKIP-CACHE-INV: ; %bb.0: ; %entry 8769; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8770; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8771; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8772; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 8773; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 8774; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 8775; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8776; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 8777; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8778; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8779; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8780; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8781; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8782; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 8783; SKIP-CACHE-INV-NEXT: s_endpgm 8784; 8785; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8786; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8787; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8788; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8789; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8790; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8791; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8792; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8793; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8794; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8795; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8796; 8797; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8798; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8799; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8800; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8801; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8802; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8803; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8804; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8805; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8806; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 8807; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8808; GFX90A-TGSPLIT-NEXT: s_endpgm 8809; 8810; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8811; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8812; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8813; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8814; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8815; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8816; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8817; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8818; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8819; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8820; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8821; 8822; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg: 8823; GFX940-TGSPLIT: ; %bb.0: ; %entry 8824; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8825; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8826; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8827; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8828; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8829; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8830; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8831; GFX940-TGSPLIT-NEXT: buffer_inv sc0 8832; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8833; GFX940-TGSPLIT-NEXT: s_endpgm 8834 i32* %out, i32 %in, i32 %old) { 8835entry: 8836 %gep = getelementptr i32, i32* %out, i32 4 8837 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic 8838 %val0 = extractvalue { i32, i1 } %val, 0 8839 store i32 %val0, i32* %out, align 4 8840 ret void 8841} 8842 8843define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg( 8844; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8845; GFX7: ; %bb.0: ; %entry 8846; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8847; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8848; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8849; GFX7-NEXT: s_add_u32 s4, s0, 16 8850; GFX7-NEXT: s_addc_u32 s5, s1, 0 8851; GFX7-NEXT: v_mov_b32_e32 v0, s4 8852; GFX7-NEXT: v_mov_b32_e32 v2, s2 8853; GFX7-NEXT: v_mov_b32_e32 v1, s5 8854; GFX7-NEXT: v_mov_b32_e32 v3, s3 8855; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8856; GFX7-NEXT: v_mov_b32_e32 v0, s0 8857; GFX7-NEXT: v_mov_b32_e32 v1, s1 8858; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8859; GFX7-NEXT: flat_store_dword v[0:1], v2 8860; GFX7-NEXT: s_endpgm 8861; 8862; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8863; GFX10-WGP: ; %bb.0: ; %entry 8864; GFX10-WGP-NEXT: s_clause 0x1 8865; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8866; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8867; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 8868; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 8869; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 8870; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 8871; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 8872; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 8873; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 8874; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 8875; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 8876; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8877; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 8878; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 8879; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8880; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 8881; GFX10-WGP-NEXT: s_endpgm 8882; 8883; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8884; GFX10-CU: ; %bb.0: ; %entry 8885; GFX10-CU-NEXT: s_clause 0x1 8886; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8887; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8888; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 8889; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 8890; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 8891; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 8892; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 8893; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 8894; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 8895; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8896; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 8897; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 8898; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8899; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 8900; GFX10-CU-NEXT: s_endpgm 8901; 8902; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8903; SKIP-CACHE-INV: ; %bb.0: ; %entry 8904; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8905; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 8906; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 8907; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 8908; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 8909; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 8910; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 8911; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 8912; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 8913; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8914; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 8915; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 8916; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8917; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 8918; SKIP-CACHE-INV-NEXT: s_endpgm 8919; 8920; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8921; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 8922; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8923; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8924; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8925; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8926; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8927; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8928; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8929; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8930; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 8931; 8932; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8933; GFX90A-TGSPLIT: ; %bb.0: ; %entry 8934; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8935; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 8936; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8937; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 8938; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 8939; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8940; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 8941; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8942; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8943; GFX90A-TGSPLIT-NEXT: s_endpgm 8944; 8945; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8946; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 8947; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8948; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8949; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8950; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8951; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8952; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8953; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8954; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 8955; GFX940-NOTTGSPLIT-NEXT: s_endpgm 8956; 8957; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg: 8958; GFX940-TGSPLIT: ; %bb.0: ; %entry 8959; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 8960; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 8961; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 8962; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 8963; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 8964; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8965; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 8966; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 8967; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 8968; GFX940-TGSPLIT-NEXT: s_endpgm 8969 i32* %out, i32 %in, i32 %old) { 8970entry: 8971 %gep = getelementptr i32, i32* %out, i32 4 8972 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic 8973 %val0 = extractvalue { i32, i1 } %val, 0 8974 store i32 %val0, i32* %out, align 4 8975 ret void 8976} 8977 8978define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg( 8979; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 8980; GFX7: ; %bb.0: ; %entry 8981; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 8982; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 8983; GFX7-NEXT: s_waitcnt lgkmcnt(0) 8984; GFX7-NEXT: s_add_u32 s4, s0, 16 8985; GFX7-NEXT: s_addc_u32 s5, s1, 0 8986; GFX7-NEXT: v_mov_b32_e32 v0, s4 8987; GFX7-NEXT: v_mov_b32_e32 v2, s2 8988; GFX7-NEXT: v_mov_b32_e32 v1, s5 8989; GFX7-NEXT: v_mov_b32_e32 v3, s3 8990; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 8991; GFX7-NEXT: v_mov_b32_e32 v0, s0 8992; GFX7-NEXT: v_mov_b32_e32 v1, s1 8993; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 8994; GFX7-NEXT: flat_store_dword v[0:1], v2 8995; GFX7-NEXT: s_endpgm 8996; 8997; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 8998; GFX10-WGP: ; %bb.0: ; %entry 8999; GFX10-WGP-NEXT: s_clause 0x1 9000; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9001; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9002; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9003; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9004; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9005; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9006; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9007; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9008; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9009; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9010; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9011; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9012; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9013; GFX10-WGP-NEXT: buffer_gl0_inv 9014; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9015; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9016; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9017; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9018; GFX10-WGP-NEXT: s_endpgm 9019; 9020; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9021; GFX10-CU: ; %bb.0: ; %entry 9022; GFX10-CU-NEXT: s_clause 0x1 9023; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9024; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9025; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9026; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9027; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9028; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9029; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9030; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9031; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9032; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9033; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9034; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9035; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9036; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9037; GFX10-CU-NEXT: s_endpgm 9038; 9039; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9040; SKIP-CACHE-INV: ; %bb.0: ; %entry 9041; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9042; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9043; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9044; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9045; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9046; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9047; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9048; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9049; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9050; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9051; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9052; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9053; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9054; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9055; SKIP-CACHE-INV-NEXT: s_endpgm 9056; 9057; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9058; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9059; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9060; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9061; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9062; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9063; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9064; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9065; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9066; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9067; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9068; 9069; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9070; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9071; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9072; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9073; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9074; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9075; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9076; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9077; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9078; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9079; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9080; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9081; GFX90A-TGSPLIT-NEXT: s_endpgm 9082; 9083; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9084; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9085; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9086; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9087; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9088; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9089; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9090; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9091; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9092; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9093; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9094; 9095; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg: 9096; GFX940-TGSPLIT: ; %bb.0: ; %entry 9097; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9098; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9099; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9100; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9101; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9102; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9103; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9104; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9105; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9106; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9107; GFX940-TGSPLIT-NEXT: s_endpgm 9108 i32* %out, i32 %in, i32 %old) { 9109entry: 9110 %gep = getelementptr i32, i32* %out, i32 4 9111 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic 9112 %val0 = extractvalue { i32, i1 } %val, 0 9113 store i32 %val0, i32* %out, align 4 9114 ret void 9115} 9116 9117define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg( 9118; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9119; GFX7: ; %bb.0: ; %entry 9120; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9121; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9122; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9123; GFX7-NEXT: s_add_u32 s4, s0, 16 9124; GFX7-NEXT: s_addc_u32 s5, s1, 0 9125; GFX7-NEXT: v_mov_b32_e32 v0, s4 9126; GFX7-NEXT: v_mov_b32_e32 v2, s2 9127; GFX7-NEXT: v_mov_b32_e32 v1, s5 9128; GFX7-NEXT: v_mov_b32_e32 v3, s3 9129; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9130; GFX7-NEXT: v_mov_b32_e32 v0, s0 9131; GFX7-NEXT: v_mov_b32_e32 v1, s1 9132; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9133; GFX7-NEXT: flat_store_dword v[0:1], v2 9134; GFX7-NEXT: s_endpgm 9135; 9136; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9137; GFX10-WGP: ; %bb.0: ; %entry 9138; GFX10-WGP-NEXT: s_clause 0x1 9139; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9140; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9141; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9142; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9143; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9144; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9145; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9146; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9147; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9148; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9149; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9150; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9151; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9152; GFX10-WGP-NEXT: buffer_gl0_inv 9153; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9154; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9155; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9156; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9157; GFX10-WGP-NEXT: s_endpgm 9158; 9159; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9160; GFX10-CU: ; %bb.0: ; %entry 9161; GFX10-CU-NEXT: s_clause 0x1 9162; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9163; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9164; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9165; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9166; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9167; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9168; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9169; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9170; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9171; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9172; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9173; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9174; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9175; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9176; GFX10-CU-NEXT: s_endpgm 9177; 9178; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9179; SKIP-CACHE-INV: ; %bb.0: ; %entry 9180; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9181; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9182; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9183; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9184; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9185; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9186; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9187; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9188; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9189; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9190; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9191; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9192; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9193; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9194; SKIP-CACHE-INV-NEXT: s_endpgm 9195; 9196; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9197; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9198; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9199; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9200; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9201; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9202; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9203; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9204; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9205; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9206; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9207; 9208; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9209; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9210; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9211; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9212; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9213; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9214; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9215; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9216; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9217; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9218; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9219; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9220; GFX90A-TGSPLIT-NEXT: s_endpgm 9221; 9222; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9223; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9224; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9225; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9226; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9227; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9228; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9229; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9230; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9231; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9232; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9233; 9234; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg: 9235; GFX940-TGSPLIT: ; %bb.0: ; %entry 9236; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9237; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9238; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9239; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9240; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9241; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9242; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9243; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9244; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9245; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9246; GFX940-TGSPLIT-NEXT: s_endpgm 9247 i32* %out, i32 %in, i32 %old) { 9248entry: 9249 %gep = getelementptr i32, i32* %out, i32 4 9250 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic 9251 %val0 = extractvalue { i32, i1 } %val, 0 9252 store i32 %val0, i32* %out, align 4 9253 ret void 9254} 9255 9256define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg( 9257; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9258; GFX7: ; %bb.0: ; %entry 9259; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9260; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9261; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9262; GFX7-NEXT: s_add_u32 s4, s0, 16 9263; GFX7-NEXT: s_addc_u32 s5, s1, 0 9264; GFX7-NEXT: v_mov_b32_e32 v0, s4 9265; GFX7-NEXT: v_mov_b32_e32 v2, s2 9266; GFX7-NEXT: v_mov_b32_e32 v1, s5 9267; GFX7-NEXT: v_mov_b32_e32 v3, s3 9268; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9269; GFX7-NEXT: v_mov_b32_e32 v0, s0 9270; GFX7-NEXT: v_mov_b32_e32 v1, s1 9271; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9272; GFX7-NEXT: flat_store_dword v[0:1], v2 9273; GFX7-NEXT: s_endpgm 9274; 9275; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9276; GFX10-WGP: ; %bb.0: ; %entry 9277; GFX10-WGP-NEXT: s_clause 0x1 9278; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9279; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9280; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9281; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9282; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9283; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9284; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9285; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9286; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9287; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9288; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9289; GFX10-WGP-NEXT: buffer_gl0_inv 9290; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9291; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9292; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9293; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9294; GFX10-WGP-NEXT: s_endpgm 9295; 9296; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9297; GFX10-CU: ; %bb.0: ; %entry 9298; GFX10-CU-NEXT: s_clause 0x1 9299; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9300; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9301; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9302; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9303; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9304; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9305; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9306; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9307; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9308; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9309; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9310; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9311; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9312; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9313; GFX10-CU-NEXT: s_endpgm 9314; 9315; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9316; SKIP-CACHE-INV: ; %bb.0: ; %entry 9317; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9318; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9319; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9320; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9321; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9322; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9323; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9324; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9325; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9326; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9327; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9328; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9329; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9330; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9331; SKIP-CACHE-INV-NEXT: s_endpgm 9332; 9333; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9334; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9335; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9336; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9337; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9338; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9339; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9340; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9341; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9342; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9343; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9344; 9345; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9346; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9347; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9348; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9349; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9350; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9351; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9352; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9353; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9354; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9355; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9356; GFX90A-TGSPLIT-NEXT: s_endpgm 9357; 9358; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9359; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9360; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9361; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9362; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9363; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9364; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9365; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9366; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9367; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9368; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9369; 9370; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg: 9371; GFX940-TGSPLIT: ; %bb.0: ; %entry 9372; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9373; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9374; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9375; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9376; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9377; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9378; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9379; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9380; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9381; GFX940-TGSPLIT-NEXT: s_endpgm 9382 i32* %out, i32 %in, i32 %old) { 9383entry: 9384 %gep = getelementptr i32, i32* %out, i32 4 9385 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire 9386 %val0 = extractvalue { i32, i1 } %val, 0 9387 store i32 %val0, i32* %out, align 4 9388 ret void 9389} 9390 9391define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg( 9392; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9393; GFX7: ; %bb.0: ; %entry 9394; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9395; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9396; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9397; GFX7-NEXT: s_add_u32 s4, s0, 16 9398; GFX7-NEXT: s_addc_u32 s5, s1, 0 9399; GFX7-NEXT: v_mov_b32_e32 v0, s4 9400; GFX7-NEXT: v_mov_b32_e32 v2, s2 9401; GFX7-NEXT: v_mov_b32_e32 v1, s5 9402; GFX7-NEXT: v_mov_b32_e32 v3, s3 9403; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9404; GFX7-NEXT: v_mov_b32_e32 v0, s0 9405; GFX7-NEXT: v_mov_b32_e32 v1, s1 9406; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9407; GFX7-NEXT: flat_store_dword v[0:1], v2 9408; GFX7-NEXT: s_endpgm 9409; 9410; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9411; GFX10-WGP: ; %bb.0: ; %entry 9412; GFX10-WGP-NEXT: s_clause 0x1 9413; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9414; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9415; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9416; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9417; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9418; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9419; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9420; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9421; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9422; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9423; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9424; GFX10-WGP-NEXT: buffer_gl0_inv 9425; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9426; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9427; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9428; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9429; GFX10-WGP-NEXT: s_endpgm 9430; 9431; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9432; GFX10-CU: ; %bb.0: ; %entry 9433; GFX10-CU-NEXT: s_clause 0x1 9434; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9435; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9436; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9437; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9438; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9439; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9440; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9441; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9442; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9443; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9444; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9445; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9446; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9447; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9448; GFX10-CU-NEXT: s_endpgm 9449; 9450; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9451; SKIP-CACHE-INV: ; %bb.0: ; %entry 9452; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9453; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9454; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9455; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9456; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9457; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9458; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9459; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9460; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9461; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9462; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9463; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9464; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9465; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9466; SKIP-CACHE-INV-NEXT: s_endpgm 9467; 9468; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9469; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9470; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9471; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9472; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9473; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9474; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9475; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9476; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9477; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9478; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9479; 9480; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9481; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9482; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9483; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9484; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9485; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9486; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9487; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9488; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9489; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9490; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9491; GFX90A-TGSPLIT-NEXT: s_endpgm 9492; 9493; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9494; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9495; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9496; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9497; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9498; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9499; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9500; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9501; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9502; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9503; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9504; 9505; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg: 9506; GFX940-TGSPLIT: ; %bb.0: ; %entry 9507; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9508; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9509; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9510; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9511; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9512; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9513; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9514; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9515; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9516; GFX940-TGSPLIT-NEXT: s_endpgm 9517 i32* %out, i32 %in, i32 %old) { 9518entry: 9519 %gep = getelementptr i32, i32* %out, i32 4 9520 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire 9521 %val0 = extractvalue { i32, i1 } %val, 0 9522 store i32 %val0, i32* %out, align 4 9523 ret void 9524} 9525 9526define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg( 9527; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9528; GFX7: ; %bb.0: ; %entry 9529; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9530; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9531; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9532; GFX7-NEXT: s_add_u32 s4, s0, 16 9533; GFX7-NEXT: s_addc_u32 s5, s1, 0 9534; GFX7-NEXT: v_mov_b32_e32 v0, s4 9535; GFX7-NEXT: v_mov_b32_e32 v2, s2 9536; GFX7-NEXT: v_mov_b32_e32 v1, s5 9537; GFX7-NEXT: v_mov_b32_e32 v3, s3 9538; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9539; GFX7-NEXT: v_mov_b32_e32 v0, s0 9540; GFX7-NEXT: v_mov_b32_e32 v1, s1 9541; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9542; GFX7-NEXT: flat_store_dword v[0:1], v2 9543; GFX7-NEXT: s_endpgm 9544; 9545; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9546; GFX10-WGP: ; %bb.0: ; %entry 9547; GFX10-WGP-NEXT: s_clause 0x1 9548; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9549; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9550; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9551; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9552; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9553; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9554; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9555; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9556; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9557; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9558; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9559; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9560; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9561; GFX10-WGP-NEXT: buffer_gl0_inv 9562; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9563; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9564; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9565; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9566; GFX10-WGP-NEXT: s_endpgm 9567; 9568; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9569; GFX10-CU: ; %bb.0: ; %entry 9570; GFX10-CU-NEXT: s_clause 0x1 9571; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9572; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9573; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9574; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9575; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9576; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9577; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9578; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9579; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9580; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9581; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9582; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9583; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9584; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9585; GFX10-CU-NEXT: s_endpgm 9586; 9587; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9588; SKIP-CACHE-INV: ; %bb.0: ; %entry 9589; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9590; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9591; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9592; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9593; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9594; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9595; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9596; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9597; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9598; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9599; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9600; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9601; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9602; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9603; SKIP-CACHE-INV-NEXT: s_endpgm 9604; 9605; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9606; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9607; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9608; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9609; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9610; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9611; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9612; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9613; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9614; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9615; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9616; 9617; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9618; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9619; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9620; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9621; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9622; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9623; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9624; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9625; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9626; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9627; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9628; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9629; GFX90A-TGSPLIT-NEXT: s_endpgm 9630; 9631; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9632; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9633; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9634; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9635; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9636; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9637; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9638; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9639; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9640; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9641; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9642; 9643; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg: 9644; GFX940-TGSPLIT: ; %bb.0: ; %entry 9645; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9646; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9647; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9648; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9649; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9650; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9651; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9652; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9653; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9654; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9655; GFX940-TGSPLIT-NEXT: s_endpgm 9656 i32* %out, i32 %in, i32 %old) { 9657entry: 9658 %gep = getelementptr i32, i32* %out, i32 4 9659 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire 9660 %val0 = extractvalue { i32, i1 } %val, 0 9661 store i32 %val0, i32* %out, align 4 9662 ret void 9663} 9664 9665define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg( 9666; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9667; GFX7: ; %bb.0: ; %entry 9668; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9669; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9670; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9671; GFX7-NEXT: s_add_u32 s4, s0, 16 9672; GFX7-NEXT: s_addc_u32 s5, s1, 0 9673; GFX7-NEXT: v_mov_b32_e32 v0, s4 9674; GFX7-NEXT: v_mov_b32_e32 v2, s2 9675; GFX7-NEXT: v_mov_b32_e32 v1, s5 9676; GFX7-NEXT: v_mov_b32_e32 v3, s3 9677; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9678; GFX7-NEXT: v_mov_b32_e32 v0, s0 9679; GFX7-NEXT: v_mov_b32_e32 v1, s1 9680; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9681; GFX7-NEXT: flat_store_dword v[0:1], v2 9682; GFX7-NEXT: s_endpgm 9683; 9684; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9685; GFX10-WGP: ; %bb.0: ; %entry 9686; GFX10-WGP-NEXT: s_clause 0x1 9687; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9688; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9689; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9690; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9691; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9692; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9693; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9694; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9695; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9696; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9697; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9698; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9699; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9700; GFX10-WGP-NEXT: buffer_gl0_inv 9701; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9702; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9703; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9704; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9705; GFX10-WGP-NEXT: s_endpgm 9706; 9707; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9708; GFX10-CU: ; %bb.0: ; %entry 9709; GFX10-CU-NEXT: s_clause 0x1 9710; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9711; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9712; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9713; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9714; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9715; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9716; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9717; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9718; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9719; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9720; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9721; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9722; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9723; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9724; GFX10-CU-NEXT: s_endpgm 9725; 9726; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9727; SKIP-CACHE-INV: ; %bb.0: ; %entry 9728; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9729; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9730; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9731; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9732; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9733; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9734; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9735; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9736; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9737; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9738; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9739; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9740; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9741; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9742; SKIP-CACHE-INV-NEXT: s_endpgm 9743; 9744; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9745; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9746; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9747; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9748; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9749; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9750; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9751; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9752; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9753; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9754; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9755; 9756; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9757; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9758; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9759; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9760; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9761; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9762; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9763; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9764; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9765; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9766; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9767; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9768; GFX90A-TGSPLIT-NEXT: s_endpgm 9769; 9770; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9771; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9772; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9773; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9774; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9775; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9776; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9777; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9778; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9779; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9780; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9781; 9782; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg: 9783; GFX940-TGSPLIT: ; %bb.0: ; %entry 9784; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9785; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9786; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9787; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9788; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9789; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9790; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9791; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9792; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9793; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9794; GFX940-TGSPLIT-NEXT: s_endpgm 9795 i32* %out, i32 %in, i32 %old) { 9796entry: 9797 %gep = getelementptr i32, i32* %out, i32 4 9798 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire 9799 %val0 = extractvalue { i32, i1 } %val, 0 9800 store i32 %val0, i32* %out, align 4 9801 ret void 9802} 9803 9804define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg( 9805; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9806; GFX7: ; %bb.0: ; %entry 9807; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9808; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9809; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9810; GFX7-NEXT: s_add_u32 s4, s0, 16 9811; GFX7-NEXT: s_addc_u32 s5, s1, 0 9812; GFX7-NEXT: v_mov_b32_e32 v0, s4 9813; GFX7-NEXT: v_mov_b32_e32 v2, s2 9814; GFX7-NEXT: v_mov_b32_e32 v1, s5 9815; GFX7-NEXT: v_mov_b32_e32 v3, s3 9816; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9817; GFX7-NEXT: v_mov_b32_e32 v0, s0 9818; GFX7-NEXT: v_mov_b32_e32 v1, s1 9819; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9820; GFX7-NEXT: flat_store_dword v[0:1], v2 9821; GFX7-NEXT: s_endpgm 9822; 9823; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9824; GFX10-WGP: ; %bb.0: ; %entry 9825; GFX10-WGP-NEXT: s_clause 0x1 9826; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9827; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9828; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9829; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9830; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9831; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9832; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9833; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9834; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9835; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9836; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9837; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9838; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9839; GFX10-WGP-NEXT: buffer_gl0_inv 9840; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9841; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9842; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9843; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9844; GFX10-WGP-NEXT: s_endpgm 9845; 9846; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9847; GFX10-CU: ; %bb.0: ; %entry 9848; GFX10-CU-NEXT: s_clause 0x1 9849; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9850; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9851; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9852; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9853; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9854; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9855; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9856; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9857; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9858; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9859; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9860; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 9861; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9862; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 9863; GFX10-CU-NEXT: s_endpgm 9864; 9865; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9866; SKIP-CACHE-INV: ; %bb.0: ; %entry 9867; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9868; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 9869; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 9870; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 9871; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 9872; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 9873; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 9874; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 9875; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 9876; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9877; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 9878; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 9879; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9880; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 9881; SKIP-CACHE-INV-NEXT: s_endpgm 9882; 9883; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9884; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 9885; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9886; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9887; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9888; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9889; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9890; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9891; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9892; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9893; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 9894; 9895; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9896; GFX90A-TGSPLIT: ; %bb.0: ; %entry 9897; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9898; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9899; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9900; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 9901; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 9902; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9903; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 9904; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9905; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 9906; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9907; GFX90A-TGSPLIT-NEXT: s_endpgm 9908; 9909; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9910; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 9911; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9912; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9913; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9914; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9915; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9916; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9917; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9918; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 9919; GFX940-NOTTGSPLIT-NEXT: s_endpgm 9920; 9921; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg: 9922; GFX940-TGSPLIT: ; %bb.0: ; %entry 9923; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 9924; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 9925; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 9926; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 9927; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 9928; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9929; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 9930; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 9931; GFX940-TGSPLIT-NEXT: buffer_inv sc0 9932; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 9933; GFX940-TGSPLIT-NEXT: s_endpgm 9934 i32* %out, i32 %in, i32 %old) { 9935entry: 9936 %gep = getelementptr i32, i32* %out, i32 4 9937 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire 9938 %val0 = extractvalue { i32, i1 } %val, 0 9939 store i32 %val0, i32* %out, align 4 9940 ret void 9941} 9942 9943define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg( 9944; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 9945; GFX7: ; %bb.0: ; %entry 9946; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9947; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 9948; GFX7-NEXT: s_waitcnt lgkmcnt(0) 9949; GFX7-NEXT: s_add_u32 s4, s0, 16 9950; GFX7-NEXT: s_addc_u32 s5, s1, 0 9951; GFX7-NEXT: v_mov_b32_e32 v0, s4 9952; GFX7-NEXT: v_mov_b32_e32 v2, s2 9953; GFX7-NEXT: v_mov_b32_e32 v1, s5 9954; GFX7-NEXT: v_mov_b32_e32 v3, s3 9955; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9956; GFX7-NEXT: v_mov_b32_e32 v0, s0 9957; GFX7-NEXT: v_mov_b32_e32 v1, s1 9958; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 9959; GFX7-NEXT: flat_store_dword v[0:1], v2 9960; GFX7-NEXT: s_endpgm 9961; 9962; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 9963; GFX10-WGP: ; %bb.0: ; %entry 9964; GFX10-WGP-NEXT: s_clause 0x1 9965; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9966; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9967; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9968; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 9969; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 9970; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 9971; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 9972; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 9973; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 9974; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9975; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 9976; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9977; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 9978; GFX10-WGP-NEXT: buffer_gl0_inv 9979; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 9980; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 9981; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 9982; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 9983; GFX10-WGP-NEXT: s_endpgm 9984; 9985; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 9986; GFX10-CU: ; %bb.0: ; %entry 9987; GFX10-CU-NEXT: s_clause 0x1 9988; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 9989; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 9990; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 9991; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 9992; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 9993; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 9994; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 9995; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 9996; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 9997; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 9998; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 9999; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 10000; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10001; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 10002; GFX10-CU-NEXT: s_endpgm 10003; 10004; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 10005; SKIP-CACHE-INV: ; %bb.0: ; %entry 10006; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10007; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 10008; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 10009; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 10010; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 10011; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 10012; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 10013; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 10014; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 10015; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10016; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 10017; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 10018; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10019; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 10020; SKIP-CACHE-INV-NEXT: s_endpgm 10021; 10022; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 10023; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 10024; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10025; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10026; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10027; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10028; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10029; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10030; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10031; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10032; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 10033; 10034; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 10035; GFX90A-TGSPLIT: ; %bb.0: ; %entry 10036; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10037; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10038; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10039; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10040; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10041; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10042; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10043; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10044; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 10045; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10046; GFX90A-TGSPLIT-NEXT: s_endpgm 10047; 10048; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 10049; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 10050; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10051; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10052; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10053; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10054; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10055; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10056; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10057; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10058; GFX940-NOTTGSPLIT-NEXT: s_endpgm 10059; 10060; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg: 10061; GFX940-TGSPLIT: ; %bb.0: ; %entry 10062; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10063; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10064; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10065; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10066; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10067; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10068; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10069; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10070; GFX940-TGSPLIT-NEXT: buffer_inv sc0 10071; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10072; GFX940-TGSPLIT-NEXT: s_endpgm 10073 i32* %out, i32 %in, i32 %old) { 10074entry: 10075 %gep = getelementptr i32, i32* %out, i32 4 10076 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst 10077 %val0 = extractvalue { i32, i1 } %val, 0 10078 store i32 %val0, i32* %out, align 4 10079 ret void 10080} 10081 10082define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg( 10083; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10084; GFX7: ; %bb.0: ; %entry 10085; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10086; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 10087; GFX7-NEXT: s_waitcnt lgkmcnt(0) 10088; GFX7-NEXT: s_add_u32 s4, s0, 16 10089; GFX7-NEXT: s_addc_u32 s5, s1, 0 10090; GFX7-NEXT: v_mov_b32_e32 v0, s4 10091; GFX7-NEXT: v_mov_b32_e32 v2, s2 10092; GFX7-NEXT: v_mov_b32_e32 v1, s5 10093; GFX7-NEXT: v_mov_b32_e32 v3, s3 10094; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10095; GFX7-NEXT: v_mov_b32_e32 v0, s0 10096; GFX7-NEXT: v_mov_b32_e32 v1, s1 10097; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10098; GFX7-NEXT: flat_store_dword v[0:1], v2 10099; GFX7-NEXT: s_endpgm 10100; 10101; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10102; GFX10-WGP: ; %bb.0: ; %entry 10103; GFX10-WGP-NEXT: s_clause 0x1 10104; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10105; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10106; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10107; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 10108; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 10109; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 10110; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 10111; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 10112; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 10113; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10114; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 10115; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10116; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10117; GFX10-WGP-NEXT: buffer_gl0_inv 10118; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 10119; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 10120; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10121; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 10122; GFX10-WGP-NEXT: s_endpgm 10123; 10124; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10125; GFX10-CU: ; %bb.0: ; %entry 10126; GFX10-CU-NEXT: s_clause 0x1 10127; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10128; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10129; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 10130; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 10131; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 10132; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 10133; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 10134; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 10135; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 10136; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10137; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 10138; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 10139; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10140; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 10141; GFX10-CU-NEXT: s_endpgm 10142; 10143; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10144; SKIP-CACHE-INV: ; %bb.0: ; %entry 10145; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10146; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 10147; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 10148; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 10149; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 10150; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 10151; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 10152; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 10153; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 10154; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10155; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 10156; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 10157; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10158; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 10159; SKIP-CACHE-INV-NEXT: s_endpgm 10160; 10161; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10162; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 10163; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10164; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10165; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10166; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10167; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10168; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10169; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10170; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10171; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 10172; 10173; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10174; GFX90A-TGSPLIT: ; %bb.0: ; %entry 10175; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10176; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10177; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10178; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10179; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10180; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10181; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10182; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10183; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 10184; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10185; GFX90A-TGSPLIT-NEXT: s_endpgm 10186; 10187; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10188; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 10189; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10190; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10191; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10192; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10193; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10194; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10195; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10196; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10197; GFX940-NOTTGSPLIT-NEXT: s_endpgm 10198; 10199; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg: 10200; GFX940-TGSPLIT: ; %bb.0: ; %entry 10201; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10202; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10203; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10204; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10205; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10206; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10207; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10208; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10209; GFX940-TGSPLIT-NEXT: buffer_inv sc0 10210; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10211; GFX940-TGSPLIT-NEXT: s_endpgm 10212 i32* %out, i32 %in, i32 %old) { 10213entry: 10214 %gep = getelementptr i32, i32* %out, i32 4 10215 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst 10216 %val0 = extractvalue { i32, i1 } %val, 0 10217 store i32 %val0, i32* %out, align 4 10218 ret void 10219} 10220 10221define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg( 10222; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10223; GFX7: ; %bb.0: ; %entry 10224; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10225; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 10226; GFX7-NEXT: s_waitcnt lgkmcnt(0) 10227; GFX7-NEXT: s_add_u32 s4, s0, 16 10228; GFX7-NEXT: s_addc_u32 s5, s1, 0 10229; GFX7-NEXT: v_mov_b32_e32 v0, s4 10230; GFX7-NEXT: v_mov_b32_e32 v2, s2 10231; GFX7-NEXT: v_mov_b32_e32 v1, s5 10232; GFX7-NEXT: v_mov_b32_e32 v3, s3 10233; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10234; GFX7-NEXT: v_mov_b32_e32 v0, s0 10235; GFX7-NEXT: v_mov_b32_e32 v1, s1 10236; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10237; GFX7-NEXT: flat_store_dword v[0:1], v2 10238; GFX7-NEXT: s_endpgm 10239; 10240; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10241; GFX10-WGP: ; %bb.0: ; %entry 10242; GFX10-WGP-NEXT: s_clause 0x1 10243; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10244; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10245; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10246; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 10247; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 10248; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 10249; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 10250; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 10251; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 10252; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10253; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 10254; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10255; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10256; GFX10-WGP-NEXT: buffer_gl0_inv 10257; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 10258; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 10259; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10260; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 10261; GFX10-WGP-NEXT: s_endpgm 10262; 10263; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10264; GFX10-CU: ; %bb.0: ; %entry 10265; GFX10-CU-NEXT: s_clause 0x1 10266; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10267; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10268; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 10269; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 10270; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 10271; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 10272; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 10273; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 10274; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 10275; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10276; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 10277; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 10278; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10279; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 10280; GFX10-CU-NEXT: s_endpgm 10281; 10282; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10283; SKIP-CACHE-INV: ; %bb.0: ; %entry 10284; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10285; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 10286; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 10287; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 10288; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 10289; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 10290; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 10291; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 10292; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 10293; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10294; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 10295; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 10296; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10297; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 10298; SKIP-CACHE-INV-NEXT: s_endpgm 10299; 10300; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10301; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 10302; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10303; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10304; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10305; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10306; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10307; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10308; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10309; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10310; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 10311; 10312; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10313; GFX90A-TGSPLIT: ; %bb.0: ; %entry 10314; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10315; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10316; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10317; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10318; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10319; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10320; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10321; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10322; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 10323; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10324; GFX90A-TGSPLIT-NEXT: s_endpgm 10325; 10326; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10327; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 10328; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10329; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10330; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10331; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10332; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10333; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10334; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10335; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10336; GFX940-NOTTGSPLIT-NEXT: s_endpgm 10337; 10338; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg: 10339; GFX940-TGSPLIT: ; %bb.0: ; %entry 10340; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10341; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10342; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10343; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10344; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10345; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10346; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10347; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10348; GFX940-TGSPLIT-NEXT: buffer_inv sc0 10349; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10350; GFX940-TGSPLIT-NEXT: s_endpgm 10351 i32* %out, i32 %in, i32 %old) { 10352entry: 10353 %gep = getelementptr i32, i32* %out, i32 4 10354 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst 10355 %val0 = extractvalue { i32, i1 } %val, 0 10356 store i32 %val0, i32* %out, align 4 10357 ret void 10358} 10359 10360define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg( 10361; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10362; GFX7: ; %bb.0: ; %entry 10363; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10364; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 10365; GFX7-NEXT: s_waitcnt lgkmcnt(0) 10366; GFX7-NEXT: s_add_u32 s4, s0, 16 10367; GFX7-NEXT: s_addc_u32 s5, s1, 0 10368; GFX7-NEXT: v_mov_b32_e32 v0, s4 10369; GFX7-NEXT: v_mov_b32_e32 v2, s2 10370; GFX7-NEXT: v_mov_b32_e32 v1, s5 10371; GFX7-NEXT: v_mov_b32_e32 v3, s3 10372; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10373; GFX7-NEXT: v_mov_b32_e32 v0, s0 10374; GFX7-NEXT: v_mov_b32_e32 v1, s1 10375; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10376; GFX7-NEXT: flat_store_dword v[0:1], v2 10377; GFX7-NEXT: s_endpgm 10378; 10379; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10380; GFX10-WGP: ; %bb.0: ; %entry 10381; GFX10-WGP-NEXT: s_clause 0x1 10382; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10383; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10384; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10385; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 10386; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 10387; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 10388; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 10389; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 10390; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 10391; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10392; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 10393; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10394; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10395; GFX10-WGP-NEXT: buffer_gl0_inv 10396; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 10397; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 10398; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10399; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 10400; GFX10-WGP-NEXT: s_endpgm 10401; 10402; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10403; GFX10-CU: ; %bb.0: ; %entry 10404; GFX10-CU-NEXT: s_clause 0x1 10405; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10406; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10407; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 10408; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 10409; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 10410; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 10411; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 10412; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 10413; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 10414; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10415; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 10416; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 10417; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10418; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 10419; GFX10-CU-NEXT: s_endpgm 10420; 10421; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10422; SKIP-CACHE-INV: ; %bb.0: ; %entry 10423; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10424; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 10425; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 10426; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 10427; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 10428; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 10429; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 10430; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 10431; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 10432; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10433; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 10434; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 10435; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10436; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 10437; SKIP-CACHE-INV-NEXT: s_endpgm 10438; 10439; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10440; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 10441; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10442; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10443; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10444; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10445; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10446; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10447; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10448; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10449; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 10450; 10451; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10452; GFX90A-TGSPLIT: ; %bb.0: ; %entry 10453; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10454; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10455; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10456; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10457; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10458; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10459; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10460; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10461; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 10462; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10463; GFX90A-TGSPLIT-NEXT: s_endpgm 10464; 10465; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10466; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 10467; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10468; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10469; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10470; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10471; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10472; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10473; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10474; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10475; GFX940-NOTTGSPLIT-NEXT: s_endpgm 10476; 10477; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg: 10478; GFX940-TGSPLIT: ; %bb.0: ; %entry 10479; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10480; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10481; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10482; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10483; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10484; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10485; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10486; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10487; GFX940-TGSPLIT-NEXT: buffer_inv sc0 10488; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10489; GFX940-TGSPLIT-NEXT: s_endpgm 10490 i32* %out, i32 %in, i32 %old) { 10491entry: 10492 %gep = getelementptr i32, i32* %out, i32 4 10493 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst 10494 %val0 = extractvalue { i32, i1 } %val, 0 10495 store i32 %val0, i32* %out, align 4 10496 ret void 10497} 10498 10499define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg( 10500; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10501; GFX7: ; %bb.0: ; %entry 10502; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10503; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2 10504; GFX7-NEXT: s_waitcnt lgkmcnt(0) 10505; GFX7-NEXT: s_add_u32 s4, s0, 16 10506; GFX7-NEXT: s_addc_u32 s5, s1, 0 10507; GFX7-NEXT: v_mov_b32_e32 v0, s4 10508; GFX7-NEXT: v_mov_b32_e32 v2, s2 10509; GFX7-NEXT: v_mov_b32_e32 v1, s5 10510; GFX7-NEXT: v_mov_b32_e32 v3, s3 10511; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10512; GFX7-NEXT: v_mov_b32_e32 v0, s0 10513; GFX7-NEXT: v_mov_b32_e32 v1, s1 10514; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10515; GFX7-NEXT: flat_store_dword v[0:1], v2 10516; GFX7-NEXT: s_endpgm 10517; 10518; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10519; GFX10-WGP: ; %bb.0: ; %entry 10520; GFX10-WGP-NEXT: s_clause 0x1 10521; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10522; GFX10-WGP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10523; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10524; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16 10525; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0 10526; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4 10527; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 10528; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5 10529; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 10530; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10531; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 10532; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10533; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) 10534; GFX10-WGP-NEXT: buffer_gl0_inv 10535; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 10536; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 10537; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) 10538; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 10539; GFX10-WGP-NEXT: s_endpgm 10540; 10541; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10542; GFX10-CU: ; %bb.0: ; %entry 10543; GFX10-CU-NEXT: s_clause 0x1 10544; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10545; GFX10-CU-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10546; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) 10547; GFX10-CU-NEXT: s_add_u32 s4, s0, 16 10548; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0 10549; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4 10550; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 10551; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5 10552; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 10553; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10554; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 10555; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 10556; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10557; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 10558; GFX10-CU-NEXT: s_endpgm 10559; 10560; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10561; SKIP-CACHE-INV: ; %bb.0: ; %entry 10562; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10563; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2 10564; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0) 10565; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s2, 16 10566; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s3, 0 10567; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4 10568; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s1 10569; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5 10570; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0 10571; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc 10572; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2 10573; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3 10574; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10575; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2 10576; SKIP-CACHE-INV-NEXT: s_endpgm 10577; 10578; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10579; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry 10580; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10581; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10582; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10583; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10584; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10585; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10586; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10587; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10588; GFX90A-NOTTGSPLIT-NEXT: s_endpgm 10589; 10590; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10591; GFX90A-TGSPLIT: ; %bb.0: ; %entry 10592; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 10593; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8 10594; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10595; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 10596; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] 10597; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10598; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc 10599; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10600; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol 10601; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10602; GFX90A-TGSPLIT-NEXT: s_endpgm 10603; 10604; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10605; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry 10606; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10607; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10608; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10609; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10610; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10611; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10612; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 10613; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 10614; GFX940-NOTTGSPLIT-NEXT: s_endpgm 10615; 10616; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg: 10617; GFX940-TGSPLIT: ; %bb.0: ; %entry 10618; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 10619; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 10620; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0) 10621; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3] 10622; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[4:5] 10623; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10624; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 10625; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) 10626; GFX940-TGSPLIT-NEXT: buffer_inv sc0 10627; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 10628; GFX940-TGSPLIT-NEXT: s_endpgm 10629 i32* %out, i32 %in, i32 %old) { 10630entry: 10631 %gep = getelementptr i32, i32* %out, i32 4 10632 %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst 10633 %val0 = extractvalue { i32, i1 } %val, 0 10634 store i32 %val0, i32* %out, align 4 10635 ret void 10636} 10637 10638