Lines Matching refs:CU

4 …sa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
11 …sa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
40 ; GFX10-CU-LABEL: flat_system_unordered_load:
41 ; GFX10-CU:       ; %bb.0: ; %entry
42 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51 ; GFX10-CU-NEXT:    s_endpgm
130 ; GFX11-CU-LABEL: flat_system_unordered_load:
131 ; GFX11-CU:       ; %bb.0: ; %entry
132 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
133 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
135 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
136 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
137 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
138 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
139 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
140 ; GFX11-CU-NEXT:    s_endpgm
175 ; GFX10-CU-LABEL: flat_system_monotonic_load:
176 ; GFX10-CU:       ; %bb.0: ; %entry
177 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
179 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
180 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
181 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
182 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
183 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
184 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
186 ; GFX10-CU-NEXT:    s_endpgm
265 ; GFX11-CU-LABEL: flat_system_monotonic_load:
266 ; GFX11-CU:       ; %bb.0: ; %entry
267 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
268 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
270 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
271 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
272 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
273 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
274 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
275 ; GFX11-CU-NEXT:    s_endpgm
313 ; GFX10-CU-LABEL: flat_system_acquire_load:
314 ; GFX10-CU:       ; %bb.0: ; %entry
315 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
316 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
317 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
318 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
319 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
320 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
321 ; GFX10-CU-NEXT:    buffer_gl0_inv
322 ; GFX10-CU-NEXT:    buffer_gl1_inv
323 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
324 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
325 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
326 ; GFX10-CU-NEXT:    s_endpgm
413 ; GFX11-CU-LABEL: flat_system_acquire_load:
414 ; GFX11-CU:       ; %bb.0: ; %entry
415 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
416 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
417 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
418 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
419 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
420 ; GFX11-CU-NEXT:    buffer_gl0_inv
421 ; GFX11-CU-NEXT:    buffer_gl1_inv
422 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
423 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
424 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
425 ; GFX11-CU-NEXT:    s_endpgm
466 ; GFX10-CU-LABEL: flat_system_seq_cst_load:
467 ; GFX10-CU:       ; %bb.0: ; %entry
468 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
469 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
470 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
471 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
472 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
473 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
474 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
475 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
476 ; GFX10-CU-NEXT:    buffer_gl0_inv
477 ; GFX10-CU-NEXT:    buffer_gl1_inv
478 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
479 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
480 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
481 ; GFX10-CU-NEXT:    s_endpgm
575 ; GFX11-CU-LABEL: flat_system_seq_cst_load:
576 ; GFX11-CU:       ; %bb.0: ; %entry
577 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
578 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
579 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
580 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
581 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
582 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
583 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
584 ; GFX11-CU-NEXT:    buffer_gl0_inv
585 ; GFX11-CU-NEXT:    buffer_gl1_inv
586 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
587 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
588 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
589 ; GFX11-CU-NEXT:    s_endpgm
621 ; GFX10-CU-LABEL: flat_system_unordered_store:
622 ; GFX10-CU:       ; %bb.0: ; %entry
623 ; GFX10-CU-NEXT:    s_clause 0x1
624 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
625 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
626 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
627 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
628 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
629 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
630 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
631 ; GFX10-CU-NEXT:    s_endpgm
696 ; GFX11-CU-LABEL: flat_system_unordered_store:
697 ; GFX11-CU:       ; %bb.0: ; %entry
698 ; GFX11-CU-NEXT:    s_clause 0x1
699 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
700 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
701 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
702 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
703 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
704 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
705 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
706 ; GFX11-CU-NEXT:    s_endpgm
737 ; GFX10-CU-LABEL: flat_system_monotonic_store:
738 ; GFX10-CU:       ; %bb.0: ; %entry
739 ; GFX10-CU-NEXT:    s_clause 0x1
740 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
741 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
742 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
743 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
744 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
745 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
746 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
747 ; GFX10-CU-NEXT:    s_endpgm
812 ; GFX11-CU-LABEL: flat_system_monotonic_store:
813 ; GFX11-CU:       ; %bb.0: ; %entry
814 ; GFX11-CU-NEXT:    s_clause 0x1
815 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
816 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
817 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
818 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
819 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
820 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
821 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
822 ; GFX11-CU-NEXT:    s_endpgm
856 ; GFX10-CU-LABEL: flat_system_release_store:
857 ; GFX10-CU:       ; %bb.0: ; %entry
858 ; GFX10-CU-NEXT:    s_clause 0x1
859 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
860 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
861 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
862 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
863 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
864 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
865 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
866 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
867 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
868 ; GFX10-CU-NEXT:    s_endpgm
944 ; GFX11-CU-LABEL: flat_system_release_store:
945 ; GFX11-CU:       ; %bb.0: ; %entry
946 ; GFX11-CU-NEXT:    s_clause 0x1
947 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
948 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
949 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
950 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
951 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
952 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
953 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
954 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
955 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
956 ; GFX11-CU-NEXT:    s_endpgm
990 ; GFX10-CU-LABEL: flat_system_seq_cst_store:
991 ; GFX10-CU:       ; %bb.0: ; %entry
992 ; GFX10-CU-NEXT:    s_clause 0x1
993 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
994 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
995 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
996 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
997 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
998 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
999 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1000 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1001 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1002 ; GFX10-CU-NEXT:    s_endpgm
1078 ; GFX11-CU-LABEL: flat_system_seq_cst_store:
1079 ; GFX11-CU:       ; %bb.0: ; %entry
1080 ; GFX11-CU-NEXT:    s_clause 0x1
1081 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1082 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1083 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1084 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1085 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1086 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1087 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1088 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1089 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1090 ; GFX11-CU-NEXT:    s_endpgm
1121 ; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
1122 ; GFX10-CU:       ; %bb.0: ; %entry
1123 ; GFX10-CU-NEXT:    s_clause 0x1
1124 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1125 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1126 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1127 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1128 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1129 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1130 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1131 ; GFX10-CU-NEXT:    s_endpgm
1196 ; GFX11-CU-LABEL: flat_system_monotonic_atomicrmw:
1197 ; GFX11-CU:       ; %bb.0: ; %entry
1198 ; GFX11-CU-NEXT:    s_clause 0x1
1199 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1200 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1201 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1202 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1203 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1204 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1205 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1206 ; GFX11-CU-NEXT:    s_endpgm
1243 ; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
1244 ; GFX10-CU:       ; %bb.0: ; %entry
1245 ; GFX10-CU-NEXT:    s_clause 0x1
1246 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1247 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1248 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1249 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1250 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1251 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1252 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1253 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1254 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1255 ; GFX10-CU-NEXT:    buffer_gl0_inv
1256 ; GFX10-CU-NEXT:    buffer_gl1_inv
1257 ; GFX10-CU-NEXT:    s_endpgm
1336 ; GFX11-CU-LABEL: flat_system_acquire_atomicrmw:
1337 ; GFX11-CU:       ; %bb.0: ; %entry
1338 ; GFX11-CU-NEXT:    s_clause 0x1
1339 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1340 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1341 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1342 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1343 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1344 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1345 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1346 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1347 ; GFX11-CU-NEXT:    buffer_gl0_inv
1348 ; GFX11-CU-NEXT:    buffer_gl1_inv
1349 ; GFX11-CU-NEXT:    s_endpgm
1383 ; GFX10-CU-LABEL: flat_system_release_atomicrmw:
1384 ; GFX10-CU:       ; %bb.0: ; %entry
1385 ; GFX10-CU-NEXT:    s_clause 0x1
1386 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1387 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1388 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1389 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1390 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1391 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1392 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1393 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1394 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1395 ; GFX10-CU-NEXT:    s_endpgm
1471 ; GFX11-CU-LABEL: flat_system_release_atomicrmw:
1472 ; GFX11-CU:       ; %bb.0: ; %entry
1473 ; GFX11-CU-NEXT:    s_clause 0x1
1474 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1475 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1476 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1477 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1478 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1479 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1480 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1481 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1482 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1483 ; GFX11-CU-NEXT:    s_endpgm
1523 ; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
1524 ; GFX10-CU:       ; %bb.0: ; %entry
1525 ; GFX10-CU-NEXT:    s_clause 0x1
1526 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1527 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1528 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1529 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1530 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1531 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1532 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1533 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1534 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1535 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1536 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1537 ; GFX10-CU-NEXT:    buffer_gl0_inv
1538 ; GFX10-CU-NEXT:    buffer_gl1_inv
1539 ; GFX10-CU-NEXT:    s_endpgm
1629 ; GFX11-CU-LABEL: flat_system_acq_rel_atomicrmw:
1630 ; GFX11-CU:       ; %bb.0: ; %entry
1631 ; GFX11-CU-NEXT:    s_clause 0x1
1632 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1633 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1634 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1635 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1636 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1637 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1638 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1639 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1640 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1641 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1642 ; GFX11-CU-NEXT:    buffer_gl0_inv
1643 ; GFX11-CU-NEXT:    buffer_gl1_inv
1644 ; GFX11-CU-NEXT:    s_endpgm
1684 ; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
1685 ; GFX10-CU:       ; %bb.0: ; %entry
1686 ; GFX10-CU-NEXT:    s_clause 0x1
1687 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1688 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1689 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1690 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1691 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1692 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1693 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1694 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1695 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1696 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1697 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1698 ; GFX10-CU-NEXT:    buffer_gl0_inv
1699 ; GFX10-CU-NEXT:    buffer_gl1_inv
1700 ; GFX10-CU-NEXT:    s_endpgm
1790 ; GFX11-CU-LABEL: flat_system_seq_cst_atomicrmw:
1791 ; GFX11-CU:       ; %bb.0: ; %entry
1792 ; GFX11-CU-NEXT:    s_clause 0x1
1793 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1794 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1795 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1796 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1797 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1798 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1799 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1800 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1801 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1802 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1803 ; GFX11-CU-NEXT:    buffer_gl0_inv
1804 ; GFX11-CU-NEXT:    buffer_gl1_inv
1805 ; GFX11-CU-NEXT:    s_endpgm
1843 ; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1844 ; GFX10-CU:       ; %bb.0: ; %entry
1845 ; GFX10-CU-NEXT:    s_clause 0x1
1846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1847 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1848 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1849 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1851 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1852 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1853 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1854 ; GFX10-CU-NEXT:    buffer_gl0_inv
1855 ; GFX10-CU-NEXT:    buffer_gl1_inv
1856 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1857 ; GFX10-CU-NEXT:    s_endpgm
1942 ; GFX11-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1943 ; GFX11-CU:       ; %bb.0: ; %entry
1944 ; GFX11-CU-NEXT:    s_clause 0x1
1945 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1946 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1947 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1948 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1949 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1950 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1951 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1952 ; GFX11-CU-NEXT:    buffer_gl0_inv
1953 ; GFX11-CU-NEXT:    buffer_gl1_inv
1954 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1955 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1956 ; GFX11-CU-NEXT:    s_endpgm
1998 ; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
1999 ; GFX10-CU:       ; %bb.0: ; %entry
2000 ; GFX10-CU-NEXT:    s_clause 0x1
2001 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2002 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2003 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2004 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2005 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2006 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2007 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2008 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2009 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2010 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2011 ; GFX10-CU-NEXT:    buffer_gl0_inv
2012 ; GFX10-CU-NEXT:    buffer_gl1_inv
2013 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2014 ; GFX10-CU-NEXT:    s_endpgm
2110 ; GFX11-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
2111 ; GFX11-CU:       ; %bb.0: ; %entry
2112 ; GFX11-CU-NEXT:    s_clause 0x1
2113 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2114 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2115 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2116 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2117 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2118 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2119 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2120 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2121 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2122 ; GFX11-CU-NEXT:    buffer_gl0_inv
2123 ; GFX11-CU-NEXT:    buffer_gl1_inv
2124 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2125 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2126 ; GFX11-CU-NEXT:    s_endpgm
2168 ; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2169 ; GFX10-CU:       ; %bb.0: ; %entry
2170 ; GFX10-CU-NEXT:    s_clause 0x1
2171 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2172 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2173 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2174 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2175 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2176 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2177 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2178 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2179 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2180 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2181 ; GFX10-CU-NEXT:    buffer_gl0_inv
2182 ; GFX10-CU-NEXT:    buffer_gl1_inv
2183 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2184 ; GFX10-CU-NEXT:    s_endpgm
2280 ; GFX11-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2281 ; GFX11-CU:       ; %bb.0: ; %entry
2282 ; GFX11-CU-NEXT:    s_clause 0x1
2283 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2284 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2285 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2286 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2287 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2288 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2289 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2290 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2291 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2292 ; GFX11-CU-NEXT:    buffer_gl0_inv
2293 ; GFX11-CU-NEXT:    buffer_gl1_inv
2294 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2295 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2296 ; GFX11-CU-NEXT:    s_endpgm
2334 ; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2335 ; GFX10-CU:       ; %bb.0: ; %entry
2336 ; GFX10-CU-NEXT:    s_clause 0x1
2337 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2338 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2339 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2340 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2341 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2342 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2343 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2344 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2345 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2346 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2347 ; GFX10-CU-NEXT:    s_endpgm
2415 ; GFX11-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2416 ; GFX11-CU:       ; %bb.0: ; %entry
2417 ; GFX11-CU-NEXT:    s_clause 0x1
2418 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2419 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2420 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2421 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2422 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2423 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2424 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2425 ; GFX11-CU-NEXT:    s_endpgm
2469 ; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2470 ; GFX10-CU:       ; %bb.0: ; %entry
2471 ; GFX10-CU-NEXT:    s_clause 0x1
2472 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2473 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2474 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2475 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2476 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2477 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2478 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2479 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2480 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2481 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2482 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2483 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2484 ; GFX10-CU-NEXT:    buffer_gl0_inv
2485 ; GFX10-CU-NEXT:    buffer_gl1_inv
2486 ; GFX10-CU-NEXT:    s_endpgm
2568 ; GFX11-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2569 ; GFX11-CU:       ; %bb.0: ; %entry
2570 ; GFX11-CU-NEXT:    s_clause 0x1
2571 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2572 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2573 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2574 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2575 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2576 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2577 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2578 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2579 ; GFX11-CU-NEXT:    buffer_gl0_inv
2580 ; GFX11-CU-NEXT:    buffer_gl1_inv
2581 ; GFX11-CU-NEXT:    s_endpgm
2622 ; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2623 ; GFX10-CU:       ; %bb.0: ; %entry
2624 ; GFX10-CU-NEXT:    s_clause 0x1
2625 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2626 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2627 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2628 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2629 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2630 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2631 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2632 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2633 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2634 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2635 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2636 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2637 ; GFX10-CU-NEXT:    s_endpgm
2716 ; GFX11-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2717 ; GFX11-CU:       ; %bb.0: ; %entry
2718 ; GFX11-CU-NEXT:    s_clause 0x1
2719 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2720 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2721 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2722 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2723 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2724 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2725 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2726 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2727 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2728 ; GFX11-CU-NEXT:    s_endpgm
2775 ; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2776 ; GFX10-CU:       ; %bb.0: ; %entry
2777 ; GFX10-CU-NEXT:    s_clause 0x1
2778 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2779 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2780 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2781 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2782 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2783 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2784 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2785 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2786 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2787 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2788 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2789 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2790 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2791 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2792 ; GFX10-CU-NEXT:    buffer_gl0_inv
2793 ; GFX10-CU-NEXT:    buffer_gl1_inv
2794 ; GFX10-CU-NEXT:    s_endpgm
2887 ; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2888 ; GFX11-CU:       ; %bb.0: ; %entry
2889 ; GFX11-CU-NEXT:    s_clause 0x1
2890 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2891 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2892 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2893 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2894 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2895 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2896 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2897 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2898 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2899 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2900 ; GFX11-CU-NEXT:    buffer_gl0_inv
2901 ; GFX11-CU-NEXT:    buffer_gl1_inv
2902 ; GFX11-CU-NEXT:    s_endpgm
2949 ; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2950 ; GFX10-CU:       ; %bb.0: ; %entry
2951 ; GFX10-CU-NEXT:    s_clause 0x1
2952 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2953 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2954 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2955 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2956 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2957 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2958 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2959 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2960 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2961 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2962 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2963 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2964 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2965 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2966 ; GFX10-CU-NEXT:    buffer_gl0_inv
2967 ; GFX10-CU-NEXT:    buffer_gl1_inv
2968 ; GFX10-CU-NEXT:    s_endpgm
3061 ; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3062 ; GFX11-CU:       ; %bb.0: ; %entry
3063 ; GFX11-CU-NEXT:    s_clause 0x1
3064 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3065 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3066 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3067 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3068 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3069 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3070 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3071 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3072 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3073 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3074 ; GFX11-CU-NEXT:    buffer_gl0_inv
3075 ; GFX11-CU-NEXT:    buffer_gl1_inv
3076 ; GFX11-CU-NEXT:    s_endpgm
3120 ; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3121 ; GFX10-CU:       ; %bb.0: ; %entry
3122 ; GFX10-CU-NEXT:    s_clause 0x1
3123 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3124 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3125 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3126 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3127 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3128 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3129 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3130 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3131 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3132 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3133 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3134 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3135 ; GFX10-CU-NEXT:    buffer_gl0_inv
3136 ; GFX10-CU-NEXT:    buffer_gl1_inv
3137 ; GFX10-CU-NEXT:    s_endpgm
3219 ; GFX11-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3220 ; GFX11-CU:       ; %bb.0: ; %entry
3221 ; GFX11-CU-NEXT:    s_clause 0x1
3222 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3223 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3224 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3225 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3226 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3227 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3228 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3229 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3230 ; GFX11-CU-NEXT:    buffer_gl0_inv
3231 ; GFX11-CU-NEXT:    buffer_gl1_inv
3232 ; GFX11-CU-NEXT:    s_endpgm
3276 ; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3277 ; GFX10-CU:       ; %bb.0: ; %entry
3278 ; GFX10-CU-NEXT:    s_clause 0x1
3279 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3280 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3281 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3282 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3283 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3284 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3285 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3286 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3287 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3288 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3289 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3290 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3291 ; GFX10-CU-NEXT:    buffer_gl0_inv
3292 ; GFX10-CU-NEXT:    buffer_gl1_inv
3293 ; GFX10-CU-NEXT:    s_endpgm
3375 ; GFX11-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3376 ; GFX11-CU:       ; %bb.0: ; %entry
3377 ; GFX11-CU-NEXT:    s_clause 0x1
3378 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3379 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3380 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3381 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3382 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3383 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3384 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3385 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3386 ; GFX11-CU-NEXT:    buffer_gl0_inv
3387 ; GFX11-CU-NEXT:    buffer_gl1_inv
3388 ; GFX11-CU-NEXT:    s_endpgm
3435 ; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
3436 ; GFX10-CU:       ; %bb.0: ; %entry
3437 ; GFX10-CU-NEXT:    s_clause 0x1
3438 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3439 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3440 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3441 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3442 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3443 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3444 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3445 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3446 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3447 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3448 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3449 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3450 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3451 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3452 ; GFX10-CU-NEXT:    buffer_gl0_inv
3453 ; GFX10-CU-NEXT:    buffer_gl1_inv
3454 ; GFX10-CU-NEXT:    s_endpgm
3547 ; GFX11-CU-LABEL: flat_system_release_acquire_cmpxchg:
3548 ; GFX11-CU:       ; %bb.0: ; %entry
3549 ; GFX11-CU-NEXT:    s_clause 0x1
3550 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3551 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3552 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3553 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3554 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3555 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3556 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3557 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3558 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3559 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3560 ; GFX11-CU-NEXT:    buffer_gl0_inv
3561 ; GFX11-CU-NEXT:    buffer_gl1_inv
3562 ; GFX11-CU-NEXT:    s_endpgm
3609 ; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3610 ; GFX10-CU:       ; %bb.0: ; %entry
3611 ; GFX10-CU-NEXT:    s_clause 0x1
3612 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3613 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3614 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3615 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3616 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3617 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3618 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3619 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3620 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3621 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3622 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3623 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3624 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3625 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3626 ; GFX10-CU-NEXT:    buffer_gl0_inv
3627 ; GFX10-CU-NEXT:    buffer_gl1_inv
3628 ; GFX10-CU-NEXT:    s_endpgm
3721 ; GFX11-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3722 ; GFX11-CU:       ; %bb.0: ; %entry
3723 ; GFX11-CU-NEXT:    s_clause 0x1
3724 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3725 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3726 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3727 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3728 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3729 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3730 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3731 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3732 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3733 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3734 ; GFX11-CU-NEXT:    buffer_gl0_inv
3735 ; GFX11-CU-NEXT:    buffer_gl1_inv
3736 ; GFX11-CU-NEXT:    s_endpgm
3783 ; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3784 ; GFX10-CU:       ; %bb.0: ; %entry
3785 ; GFX10-CU-NEXT:    s_clause 0x1
3786 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3787 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3788 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3789 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3790 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3791 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3792 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3793 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3794 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3795 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3796 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3797 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3798 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3799 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3800 ; GFX10-CU-NEXT:    buffer_gl0_inv
3801 ; GFX10-CU-NEXT:    buffer_gl1_inv
3802 ; GFX10-CU-NEXT:    s_endpgm
3895 ; GFX11-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3896 ; GFX11-CU:       ; %bb.0: ; %entry
3897 ; GFX11-CU-NEXT:    s_clause 0x1
3898 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3899 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3900 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3901 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3902 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3903 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3904 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3905 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3906 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3907 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3908 ; GFX11-CU-NEXT:    buffer_gl0_inv
3909 ; GFX11-CU-NEXT:    buffer_gl1_inv
3910 ; GFX11-CU-NEXT:    s_endpgm
3957 ; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3958 ; GFX10-CU:       ; %bb.0: ; %entry
3959 ; GFX10-CU-NEXT:    s_clause 0x1
3960 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3961 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3962 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3963 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3964 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3965 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3966 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3967 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3968 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3969 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3970 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3971 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3972 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3973 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3974 ; GFX10-CU-NEXT:    buffer_gl0_inv
3975 ; GFX10-CU-NEXT:    buffer_gl1_inv
3976 ; GFX10-CU-NEXT:    s_endpgm
4069 ; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4070 ; GFX11-CU:       ; %bb.0: ; %entry
4071 ; GFX11-CU-NEXT:    s_clause 0x1
4072 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4073 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4074 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4075 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4076 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4077 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4078 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4079 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4080 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4081 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4082 ; GFX11-CU-NEXT:    buffer_gl0_inv
4083 ; GFX11-CU-NEXT:    buffer_gl1_inv
4084 ; GFX11-CU-NEXT:    s_endpgm
4131 ; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4132 ; GFX10-CU:       ; %bb.0: ; %entry
4133 ; GFX10-CU-NEXT:    s_clause 0x1
4134 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4135 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4136 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4137 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4138 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4139 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4140 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4141 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4142 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4143 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4144 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4145 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4146 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4147 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4148 ; GFX10-CU-NEXT:    buffer_gl0_inv
4149 ; GFX10-CU-NEXT:    buffer_gl1_inv
4150 ; GFX10-CU-NEXT:    s_endpgm
4243 ; GFX11-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4244 ; GFX11-CU:       ; %bb.0: ; %entry
4245 ; GFX11-CU-NEXT:    s_clause 0x1
4246 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4247 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4248 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4249 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4250 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4251 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4252 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4253 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4254 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4255 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4256 ; GFX11-CU-NEXT:    buffer_gl0_inv
4257 ; GFX11-CU-NEXT:    buffer_gl1_inv
4258 ; GFX11-CU-NEXT:    s_endpgm
4305 ; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4306 ; GFX10-CU:       ; %bb.0: ; %entry
4307 ; GFX10-CU-NEXT:    s_clause 0x1
4308 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4309 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4310 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4311 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4312 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4313 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4314 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4315 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4316 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4317 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4318 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4319 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4320 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4321 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4322 ; GFX10-CU-NEXT:    buffer_gl0_inv
4323 ; GFX10-CU-NEXT:    buffer_gl1_inv
4324 ; GFX10-CU-NEXT:    s_endpgm
4417 ; GFX11-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4418 ; GFX11-CU:       ; %bb.0: ; %entry
4419 ; GFX11-CU-NEXT:    s_clause 0x1
4420 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4421 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4422 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4423 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4424 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4425 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4426 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4427 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4428 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4429 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4430 ; GFX11-CU-NEXT:    buffer_gl0_inv
4431 ; GFX11-CU-NEXT:    buffer_gl1_inv
4432 ; GFX11-CU-NEXT:    s_endpgm
4479 ; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4480 ; GFX10-CU:       ; %bb.0: ; %entry
4481 ; GFX10-CU-NEXT:    s_clause 0x1
4482 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4483 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4484 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4485 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4486 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4487 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4488 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4489 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4490 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4491 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4492 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4493 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4494 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4495 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4496 ; GFX10-CU-NEXT:    buffer_gl0_inv
4497 ; GFX10-CU-NEXT:    buffer_gl1_inv
4498 ; GFX10-CU-NEXT:    s_endpgm
4591 ; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4592 ; GFX11-CU:       ; %bb.0: ; %entry
4593 ; GFX11-CU-NEXT:    s_clause 0x1
4594 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4595 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4596 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4597 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4598 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4599 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4600 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4601 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4602 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4603 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4604 ; GFX11-CU-NEXT:    buffer_gl0_inv
4605 ; GFX11-CU-NEXT:    buffer_gl1_inv
4606 ; GFX11-CU-NEXT:    s_endpgm
4653 ; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4654 ; GFX10-CU:       ; %bb.0: ; %entry
4655 ; GFX10-CU-NEXT:    s_clause 0x1
4656 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4657 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4658 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4659 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4660 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4661 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4662 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4663 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4664 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4665 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4666 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4667 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4668 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4669 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4670 ; GFX10-CU-NEXT:    buffer_gl0_inv
4671 ; GFX10-CU-NEXT:    buffer_gl1_inv
4672 ; GFX10-CU-NEXT:    s_endpgm
4765 ; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4766 ; GFX11-CU:       ; %bb.0: ; %entry
4767 ; GFX11-CU-NEXT:    s_clause 0x1
4768 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4769 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4770 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4771 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4772 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4773 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4774 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4775 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4776 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4777 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4778 ; GFX11-CU-NEXT:    buffer_gl0_inv
4779 ; GFX11-CU-NEXT:    buffer_gl1_inv
4780 ; GFX11-CU-NEXT:    s_endpgm
4826 ; GFX10-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4827 ; GFX10-CU:       ; %bb.0: ; %entry
4828 ; GFX10-CU-NEXT:    s_clause 0x1
4829 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4830 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4831 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4832 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4833 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4834 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4835 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4836 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4837 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4838 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4839 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4840 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4841 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4842 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4843 ; GFX10-CU-NEXT:    s_endpgm
4925 ; GFX11-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4926 ; GFX11-CU:       ; %bb.0: ; %entry
4927 ; GFX11-CU-NEXT:    s_clause 0x1
4928 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4929 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4930 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4931 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4932 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4933 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4934 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4935 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4936 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4937 ; GFX11-CU-NEXT:    s_endpgm
4988 ; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4989 ; GFX10-CU:       ; %bb.0: ; %entry
4990 ; GFX10-CU-NEXT:    s_clause 0x1
4991 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4992 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4993 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4994 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4995 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4996 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4997 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4998 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4999 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5000 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5001 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5002 ; GFX10-CU-NEXT:    buffer_gl0_inv
5003 ; GFX10-CU-NEXT:    buffer_gl1_inv
5004 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5005 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5006 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5007 ; GFX10-CU-NEXT:    s_endpgm
5097 ; GFX11-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5098 ; GFX11-CU:       ; %bb.0: ; %entry
5099 ; GFX11-CU-NEXT:    s_clause 0x1
5100 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5101 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5102 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5103 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5104 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5105 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5106 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5107 ; GFX11-CU-NEXT:    buffer_gl0_inv
5108 ; GFX11-CU-NEXT:    buffer_gl1_inv
5109 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5110 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5111 ; GFX11-CU-NEXT:    s_endpgm
5162 ; GFX10-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5163 ; GFX10-CU:       ; %bb.0: ; %entry
5164 ; GFX10-CU-NEXT:    s_clause 0x1
5165 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5166 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5167 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5168 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5169 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5170 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5171 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5172 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5173 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5174 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5175 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5176 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5177 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5178 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5179 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5180 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5181 ; GFX10-CU-NEXT:    s_endpgm
5274 ; GFX11-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5275 ; GFX11-CU:       ; %bb.0: ; %entry
5276 ; GFX11-CU-NEXT:    s_clause 0x1
5277 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5278 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5279 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5280 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5281 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5282 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5283 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5284 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5285 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5286 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5287 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5288 ; GFX11-CU-NEXT:    s_endpgm
5342 ; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5343 ; GFX10-CU:       ; %bb.0: ; %entry
5344 ; GFX10-CU-NEXT:    s_clause 0x1
5345 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5346 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5347 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5348 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5349 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5350 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5351 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5352 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5353 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5354 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5355 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5356 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5357 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5358 ; GFX10-CU-NEXT:    buffer_gl0_inv
5359 ; GFX10-CU-NEXT:    buffer_gl1_inv
5360 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5361 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5362 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5363 ; GFX10-CU-NEXT:    s_endpgm
5464 ; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5465 ; GFX11-CU:       ; %bb.0: ; %entry
5466 ; GFX11-CU-NEXT:    s_clause 0x1
5467 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5468 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5469 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5470 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5471 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5472 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5473 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5474 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5475 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5476 ; GFX11-CU-NEXT:    buffer_gl0_inv
5477 ; GFX11-CU-NEXT:    buffer_gl1_inv
5478 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5479 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5480 ; GFX11-CU-NEXT:    s_endpgm
5534 ; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5535 ; GFX10-CU:       ; %bb.0: ; %entry
5536 ; GFX10-CU-NEXT:    s_clause 0x1
5537 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5538 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5539 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5540 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5541 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5542 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5543 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5544 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5545 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5546 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5547 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5548 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5549 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5550 ; GFX10-CU-NEXT:    buffer_gl0_inv
5551 ; GFX10-CU-NEXT:    buffer_gl1_inv
5552 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5553 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5554 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5555 ; GFX10-CU-NEXT:    s_endpgm
5656 ; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5657 ; GFX11-CU:       ; %bb.0: ; %entry
5658 ; GFX11-CU-NEXT:    s_clause 0x1
5659 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5660 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5661 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5662 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5663 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5664 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5665 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5666 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5667 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5668 ; GFX11-CU-NEXT:    buffer_gl0_inv
5669 ; GFX11-CU-NEXT:    buffer_gl1_inv
5670 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5671 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5672 ; GFX11-CU-NEXT:    s_endpgm
5723 ; GFX10-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5724 ; GFX10-CU:       ; %bb.0: ; %entry
5725 ; GFX10-CU-NEXT:    s_clause 0x1
5726 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5727 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5728 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5729 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5730 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5731 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5732 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5733 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5734 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5735 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5736 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5737 ; GFX10-CU-NEXT:    buffer_gl0_inv
5738 ; GFX10-CU-NEXT:    buffer_gl1_inv
5739 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5740 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5741 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5742 ; GFX10-CU-NEXT:    s_endpgm
5832 ; GFX11-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5833 ; GFX11-CU:       ; %bb.0: ; %entry
5834 ; GFX11-CU-NEXT:    s_clause 0x1
5835 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5836 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5837 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5838 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5839 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5840 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5841 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5842 ; GFX11-CU-NEXT:    buffer_gl0_inv
5843 ; GFX11-CU-NEXT:    buffer_gl1_inv
5844 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5845 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5846 ; GFX11-CU-NEXT:    s_endpgm
5897 ; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5898 ; GFX10-CU:       ; %bb.0: ; %entry
5899 ; GFX10-CU-NEXT:    s_clause 0x1
5900 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5901 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5902 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5903 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5904 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5905 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5906 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5907 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5908 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5909 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5910 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5911 ; GFX10-CU-NEXT:    buffer_gl0_inv
5912 ; GFX10-CU-NEXT:    buffer_gl1_inv
5913 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5914 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5915 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5916 ; GFX10-CU-NEXT:    s_endpgm
6006 ; GFX11-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6007 ; GFX11-CU:       ; %bb.0: ; %entry
6008 ; GFX11-CU-NEXT:    s_clause 0x1
6009 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6010 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6011 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6012 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6013 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6014 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6015 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6016 ; GFX11-CU-NEXT:    buffer_gl0_inv
6017 ; GFX11-CU-NEXT:    buffer_gl1_inv
6018 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6019 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6020 ; GFX11-CU-NEXT:    s_endpgm
6074 ; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6075 ; GFX10-CU:       ; %bb.0: ; %entry
6076 ; GFX10-CU-NEXT:    s_clause 0x1
6077 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6078 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6079 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6080 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6081 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6082 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6083 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6084 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6085 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6086 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6087 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6088 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6089 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6090 ; GFX10-CU-NEXT:    buffer_gl0_inv
6091 ; GFX10-CU-NEXT:    buffer_gl1_inv
6092 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6093 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6094 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6095 ; GFX10-CU-NEXT:    s_endpgm
6196 ; GFX11-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6197 ; GFX11-CU:       ; %bb.0: ; %entry
6198 ; GFX11-CU-NEXT:    s_clause 0x1
6199 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6200 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6201 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6202 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6203 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6204 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6205 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6206 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6207 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6208 ; GFX11-CU-NEXT:    buffer_gl0_inv
6209 ; GFX11-CU-NEXT:    buffer_gl1_inv
6210 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6211 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6212 ; GFX11-CU-NEXT:    s_endpgm
6266 ; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6267 ; GFX10-CU:       ; %bb.0: ; %entry
6268 ; GFX10-CU-NEXT:    s_clause 0x1
6269 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6270 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6271 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6272 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6273 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6274 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6275 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6276 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6277 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6278 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6279 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6280 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6281 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6282 ; GFX10-CU-NEXT:    buffer_gl0_inv
6283 ; GFX10-CU-NEXT:    buffer_gl1_inv
6284 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6285 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6286 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6287 ; GFX10-CU-NEXT:    s_endpgm
6388 ; GFX11-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6389 ; GFX11-CU:       ; %bb.0: ; %entry
6390 ; GFX11-CU-NEXT:    s_clause 0x1
6391 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6392 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6393 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6394 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6395 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6396 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6397 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6398 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6399 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6400 ; GFX11-CU-NEXT:    buffer_gl0_inv
6401 ; GFX11-CU-NEXT:    buffer_gl1_inv
6402 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6403 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6404 ; GFX11-CU-NEXT:    s_endpgm
6458 ; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6459 ; GFX10-CU:       ; %bb.0: ; %entry
6460 ; GFX10-CU-NEXT:    s_clause 0x1
6461 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6462 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6463 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6464 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6465 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6466 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6467 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6469 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6470 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6471 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6472 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6473 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6474 ; GFX10-CU-NEXT:    buffer_gl0_inv
6475 ; GFX10-CU-NEXT:    buffer_gl1_inv
6476 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6477 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6478 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6479 ; GFX10-CU-NEXT:    s_endpgm
6580 ; GFX11-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6581 ; GFX11-CU:       ; %bb.0: ; %entry
6582 ; GFX11-CU-NEXT:    s_clause 0x1
6583 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6584 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6585 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6586 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6587 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6588 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6589 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6590 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6591 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6592 ; GFX11-CU-NEXT:    buffer_gl0_inv
6593 ; GFX11-CU-NEXT:    buffer_gl1_inv
6594 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6595 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6596 ; GFX11-CU-NEXT:    s_endpgm
6650 ; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6651 ; GFX10-CU:       ; %bb.0: ; %entry
6652 ; GFX10-CU-NEXT:    s_clause 0x1
6653 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6654 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6655 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6656 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6657 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6658 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6659 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6660 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6661 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6662 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6663 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6664 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6665 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6666 ; GFX10-CU-NEXT:    buffer_gl0_inv
6667 ; GFX10-CU-NEXT:    buffer_gl1_inv
6668 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6669 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6670 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6671 ; GFX10-CU-NEXT:    s_endpgm
6772 ; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6773 ; GFX11-CU:       ; %bb.0: ; %entry
6774 ; GFX11-CU-NEXT:    s_clause 0x1
6775 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6776 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6777 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6778 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6779 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6780 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6781 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6782 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6783 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6784 ; GFX11-CU-NEXT:    buffer_gl0_inv
6785 ; GFX11-CU-NEXT:    buffer_gl1_inv
6786 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6787 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6788 ; GFX11-CU-NEXT:    s_endpgm
6842 ; GFX10-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6843 ; GFX10-CU:       ; %bb.0: ; %entry
6844 ; GFX10-CU-NEXT:    s_clause 0x1
6845 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6847 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6848 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6849 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6851 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6852 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6853 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6854 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6855 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6856 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6857 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6858 ; GFX10-CU-NEXT:    buffer_gl0_inv
6859 ; GFX10-CU-NEXT:    buffer_gl1_inv
6860 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6861 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6862 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6863 ; GFX10-CU-NEXT:    s_endpgm
6964 ; GFX11-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6965 ; GFX11-CU:       ; %bb.0: ; %entry
6966 ; GFX11-CU-NEXT:    s_clause 0x1
6967 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6968 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6969 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6970 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6971 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6972 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6973 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6974 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6975 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6976 ; GFX11-CU-NEXT:    buffer_gl0_inv
6977 ; GFX11-CU-NEXT:    buffer_gl1_inv
6978 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6979 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6980 ; GFX11-CU-NEXT:    s_endpgm
7034 ; GFX10-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7035 ; GFX10-CU:       ; %bb.0: ; %entry
7036 ; GFX10-CU-NEXT:    s_clause 0x1
7037 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7038 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7039 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7040 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7041 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7042 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7043 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7044 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7045 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7046 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7047 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7048 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7049 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7050 ; GFX10-CU-NEXT:    buffer_gl0_inv
7051 ; GFX10-CU-NEXT:    buffer_gl1_inv
7052 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7053 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7054 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7055 ; GFX10-CU-NEXT:    s_endpgm
7156 ; GFX11-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7157 ; GFX11-CU:       ; %bb.0: ; %entry
7158 ; GFX11-CU-NEXT:    s_clause 0x1
7159 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7160 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7161 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7162 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7163 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7164 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7165 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7166 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7167 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7168 ; GFX11-CU-NEXT:    buffer_gl0_inv
7169 ; GFX11-CU-NEXT:    buffer_gl1_inv
7170 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7171 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7172 ; GFX11-CU-NEXT:    s_endpgm
7226 ; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7227 ; GFX10-CU:       ; %bb.0: ; %entry
7228 ; GFX10-CU-NEXT:    s_clause 0x1
7229 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7230 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7231 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7232 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7233 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7234 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7235 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7236 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7237 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7238 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7239 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7240 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7241 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7242 ; GFX10-CU-NEXT:    buffer_gl0_inv
7243 ; GFX10-CU-NEXT:    buffer_gl1_inv
7244 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7245 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7246 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7247 ; GFX10-CU-NEXT:    s_endpgm
7348 ; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7349 ; GFX11-CU:       ; %bb.0: ; %entry
7350 ; GFX11-CU-NEXT:    s_clause 0x1
7351 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7352 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7353 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7354 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7355 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7356 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7357 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7358 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7359 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7360 ; GFX11-CU-NEXT:    buffer_gl0_inv
7361 ; GFX11-CU-NEXT:    buffer_gl1_inv
7362 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7363 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7364 ; GFX11-CU-NEXT:    s_endpgm
7418 ; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7419 ; GFX10-CU:       ; %bb.0: ; %entry
7420 ; GFX10-CU-NEXT:    s_clause 0x1
7421 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7422 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7423 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7424 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7425 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7426 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7427 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7428 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7429 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7430 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7431 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7432 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7433 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7434 ; GFX10-CU-NEXT:    buffer_gl0_inv
7435 ; GFX10-CU-NEXT:    buffer_gl1_inv
7436 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7437 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7438 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7439 ; GFX10-CU-NEXT:    s_endpgm
7540 ; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7541 ; GFX11-CU:       ; %bb.0: ; %entry
7542 ; GFX11-CU-NEXT:    s_clause 0x1
7543 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7544 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7545 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7546 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7547 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7548 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7549 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7550 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7551 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7552 ; GFX11-CU-NEXT:    buffer_gl0_inv
7553 ; GFX11-CU-NEXT:    buffer_gl1_inv
7554 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7555 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7556 ; GFX11-CU-NEXT:    s_endpgm
7593 ; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
7594 ; GFX10-CU:       ; %bb.0: ; %entry
7595 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7596 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7597 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7598 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7599 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
7600 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7601 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7602 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7603 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7604 ; GFX10-CU-NEXT:    s_endpgm
7683 ; GFX11-CU-LABEL: flat_system_one_as_unordered_load:
7684 ; GFX11-CU:       ; %bb.0: ; %entry
7685 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7686 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7687 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7688 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7689 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7690 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7691 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7692 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7693 ; GFX11-CU-NEXT:    s_endpgm
7728 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
7729 ; GFX10-CU:       ; %bb.0: ; %entry
7730 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7731 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7732 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7733 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7734 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7735 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7736 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7737 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7738 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7739 ; GFX10-CU-NEXT:    s_endpgm
7818 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_load:
7819 ; GFX11-CU:       ; %bb.0: ; %entry
7820 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7821 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7822 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7823 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7824 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7825 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7826 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7827 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7828 ; GFX11-CU-NEXT:    s_endpgm
7868 ; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
7869 ; GFX10-CU:       ; %bb.0: ; %entry
7870 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7871 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7872 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7873 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7874 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7875 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7876 ; GFX10-CU-NEXT:    buffer_gl0_inv
7877 ; GFX10-CU-NEXT:    buffer_gl1_inv
7878 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7879 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7880 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7881 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7882 ; GFX10-CU-NEXT:    s_endpgm
7973 ; GFX11-CU-LABEL: flat_system_one_as_acquire_load:
7974 ; GFX11-CU:       ; %bb.0: ; %entry
7975 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7976 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7977 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7978 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7979 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
7980 ; GFX11-CU-NEXT:    buffer_gl0_inv
7981 ; GFX11-CU-NEXT:    buffer_gl1_inv
7982 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7983 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7984 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7985 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7986 ; GFX11-CU-NEXT:    s_endpgm
8029 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
8030 ; GFX10-CU:       ; %bb.0: ; %entry
8031 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8032 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8033 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8034 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8035 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8036 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8037 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8038 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8039 ; GFX10-CU-NEXT:    buffer_gl0_inv
8040 ; GFX10-CU-NEXT:    buffer_gl1_inv
8041 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
8042 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
8043 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8044 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8045 ; GFX10-CU-NEXT:    s_endpgm
8143 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_load:
8144 ; GFX11-CU:       ; %bb.0: ; %entry
8145 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8146 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8147 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8148 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8149 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8150 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8151 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8152 ; GFX11-CU-NEXT:    buffer_gl0_inv
8153 ; GFX11-CU-NEXT:    buffer_gl1_inv
8154 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8155 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8156 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8157 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8158 ; GFX11-CU-NEXT:    s_endpgm
8190 ; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
8191 ; GFX10-CU:       ; %bb.0: ; %entry
8192 ; GFX10-CU-NEXT:    s_clause 0x1
8193 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8194 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8195 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8196 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8197 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8198 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8199 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8200 ; GFX10-CU-NEXT:    s_endpgm
8265 ; GFX11-CU-LABEL: flat_system_one_as_unordered_store:
8266 ; GFX11-CU:       ; %bb.0: ; %entry
8267 ; GFX11-CU-NEXT:    s_clause 0x1
8268 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8269 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8270 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8271 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8272 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8273 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8274 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8275 ; GFX11-CU-NEXT:    s_endpgm
8306 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
8307 ; GFX10-CU:       ; %bb.0: ; %entry
8308 ; GFX10-CU-NEXT:    s_clause 0x1
8309 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8310 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8311 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8312 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8313 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8314 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8315 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8316 ; GFX10-CU-NEXT:    s_endpgm
8381 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_store:
8382 ; GFX11-CU:       ; %bb.0: ; %entry
8383 ; GFX11-CU-NEXT:    s_clause 0x1
8384 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8385 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8386 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8387 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8388 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8389 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8390 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8391 ; GFX11-CU-NEXT:    s_endpgm
8425 ; GFX10-CU-LABEL: flat_system_one_as_release_store:
8426 ; GFX10-CU:       ; %bb.0: ; %entry
8427 ; GFX10-CU-NEXT:    s_clause 0x1
8428 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8429 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8430 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8431 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8432 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8433 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8434 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8435 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8436 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8437 ; GFX10-CU-NEXT:    s_endpgm
8513 ; GFX11-CU-LABEL: flat_system_one_as_release_store:
8514 ; GFX11-CU:       ; %bb.0: ; %entry
8515 ; GFX11-CU-NEXT:    s_clause 0x1
8516 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8517 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8518 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8519 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8520 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8521 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8522 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8523 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8524 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8525 ; GFX11-CU-NEXT:    s_endpgm
8559 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
8560 ; GFX10-CU:       ; %bb.0: ; %entry
8561 ; GFX10-CU-NEXT:    s_clause 0x1
8562 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8563 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8564 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8565 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8566 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8567 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8568 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8569 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8570 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8571 ; GFX10-CU-NEXT:    s_endpgm
8647 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_store:
8648 ; GFX11-CU:       ; %bb.0: ; %entry
8649 ; GFX11-CU-NEXT:    s_clause 0x1
8650 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8651 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8652 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8653 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8654 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8655 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8656 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8657 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8658 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8659 ; GFX11-CU-NEXT:    s_endpgm
8690 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8691 ; GFX10-CU:       ; %bb.0: ; %entry
8692 ; GFX10-CU-NEXT:    s_clause 0x1
8693 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8694 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8695 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8696 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8697 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8698 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8699 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8700 ; GFX10-CU-NEXT:    s_endpgm
8765 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8766 ; GFX11-CU:       ; %bb.0: ; %entry
8767 ; GFX11-CU-NEXT:    s_clause 0x1
8768 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8769 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8770 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8771 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8772 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8773 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8774 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8775 ; GFX11-CU-NEXT:    s_endpgm
8811 ; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
8812 ; GFX10-CU:       ; %bb.0: ; %entry
8813 ; GFX10-CU-NEXT:    s_clause 0x1
8814 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8815 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8816 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8817 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8818 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8819 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8820 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8821 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8822 ; GFX10-CU-NEXT:    buffer_gl0_inv
8823 ; GFX10-CU-NEXT:    buffer_gl1_inv
8824 ; GFX10-CU-NEXT:    s_endpgm
8902 ; GFX11-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
8903 ; GFX11-CU:       ; %bb.0: ; %entry
8904 ; GFX11-CU-NEXT:    s_clause 0x1
8905 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8906 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8907 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8908 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8909 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8910 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8911 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8912 ; GFX11-CU-NEXT:    buffer_gl0_inv
8913 ; GFX11-CU-NEXT:    buffer_gl1_inv
8914 ; GFX11-CU-NEXT:    s_endpgm
8948 ; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
8949 ; GFX10-CU:       ; %bb.0: ; %entry
8950 ; GFX10-CU-NEXT:    s_clause 0x1
8951 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8952 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8953 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8954 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8955 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8956 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8957 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8958 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8959 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8960 ; GFX10-CU-NEXT:    s_endpgm
9036 ; GFX11-CU-LABEL: flat_system_one_as_release_atomicrmw:
9037 ; GFX11-CU:       ; %bb.0: ; %entry
9038 ; GFX11-CU-NEXT:    s_clause 0x1
9039 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9040 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9041 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9042 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9043 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9044 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9045 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9046 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9047 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9048 ; GFX11-CU-NEXT:    s_endpgm
9087 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9088 ; GFX10-CU:       ; %bb.0: ; %entry
9089 ; GFX10-CU-NEXT:    s_clause 0x1
9090 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9091 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9092 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9093 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9094 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9095 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9096 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9097 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9098 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9099 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9100 ; GFX10-CU-NEXT:    buffer_gl0_inv
9101 ; GFX10-CU-NEXT:    buffer_gl1_inv
9102 ; GFX10-CU-NEXT:    s_endpgm
9191 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9192 ; GFX11-CU:       ; %bb.0: ; %entry
9193 ; GFX11-CU-NEXT:    s_clause 0x1
9194 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9195 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9196 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9197 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9198 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9199 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9200 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9201 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9202 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9203 ; GFX11-CU-NEXT:    buffer_gl0_inv
9204 ; GFX11-CU-NEXT:    buffer_gl1_inv
9205 ; GFX11-CU-NEXT:    s_endpgm
9244 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9245 ; GFX10-CU:       ; %bb.0: ; %entry
9246 ; GFX10-CU-NEXT:    s_clause 0x1
9247 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9248 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9249 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9250 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9251 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9252 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9253 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9254 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9255 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9256 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9257 ; GFX10-CU-NEXT:    buffer_gl0_inv
9258 ; GFX10-CU-NEXT:    buffer_gl1_inv
9259 ; GFX10-CU-NEXT:    s_endpgm
9348 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9349 ; GFX11-CU:       ; %bb.0: ; %entry
9350 ; GFX11-CU-NEXT:    s_clause 0x1
9351 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9352 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9353 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9354 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9355 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9356 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9357 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9358 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9359 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9360 ; GFX11-CU-NEXT:    buffer_gl0_inv
9361 ; GFX11-CU-NEXT:    buffer_gl1_inv
9362 ; GFX11-CU-NEXT:    s_endpgm
9402 ; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9403 ; GFX10-CU:       ; %bb.0: ; %entry
9404 ; GFX10-CU-NEXT:    s_clause 0x1
9405 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9406 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9407 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9408 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9409 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9410 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9411 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9412 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9413 ; GFX10-CU-NEXT:    buffer_gl0_inv
9414 ; GFX10-CU-NEXT:    buffer_gl1_inv
9415 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9416 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9417 ; GFX10-CU-NEXT:    s_endpgm
9505 ; GFX11-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9506 ; GFX11-CU:       ; %bb.0: ; %entry
9507 ; GFX11-CU-NEXT:    s_clause 0x1
9508 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9509 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9510 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9511 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9512 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9513 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9514 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9515 ; GFX11-CU-NEXT:    buffer_gl0_inv
9516 ; GFX11-CU-NEXT:    buffer_gl1_inv
9517 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9518 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9519 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9520 ; GFX11-CU-NEXT:    s_endpgm
9564 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9565 ; GFX10-CU:       ; %bb.0: ; %entry
9566 ; GFX10-CU-NEXT:    s_clause 0x1
9567 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9568 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9569 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9570 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9571 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9572 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9573 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9574 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9575 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9576 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9577 ; GFX10-CU-NEXT:    buffer_gl0_inv
9578 ; GFX10-CU-NEXT:    buffer_gl1_inv
9579 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9580 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9581 ; GFX10-CU-NEXT:    s_endpgm
9680 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9681 ; GFX11-CU:       ; %bb.0: ; %entry
9682 ; GFX11-CU-NEXT:    s_clause 0x1
9683 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9684 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9685 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9686 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9687 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9688 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9689 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9690 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9691 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9692 ; GFX11-CU-NEXT:    buffer_gl0_inv
9693 ; GFX11-CU-NEXT:    buffer_gl1_inv
9694 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9695 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9696 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9697 ; GFX11-CU-NEXT:    s_endpgm
9741 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9742 ; GFX10-CU:       ; %bb.0: ; %entry
9743 ; GFX10-CU-NEXT:    s_clause 0x1
9744 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9745 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9746 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9747 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9748 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9749 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9750 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9751 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9752 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9753 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9754 ; GFX10-CU-NEXT:    buffer_gl0_inv
9755 ; GFX10-CU-NEXT:    buffer_gl1_inv
9756 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9757 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9758 ; GFX10-CU-NEXT:    s_endpgm
9857 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9858 ; GFX11-CU:       ; %bb.0: ; %entry
9859 ; GFX11-CU-NEXT:    s_clause 0x1
9860 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9861 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9862 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9863 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9864 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9865 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9866 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9867 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9868 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9869 ; GFX11-CU-NEXT:    buffer_gl0_inv
9870 ; GFX11-CU-NEXT:    buffer_gl1_inv
9871 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9872 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9873 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9874 ; GFX11-CU-NEXT:    s_endpgm
9912 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9913 ; GFX10-CU:       ; %bb.0: ; %entry
9914 ; GFX10-CU-NEXT:    s_clause 0x1
9915 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9916 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9917 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9918 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9919 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9920 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9921 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9922 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9923 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9924 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9925 ; GFX10-CU-NEXT:    s_endpgm
9993 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9994 ; GFX11-CU:       ; %bb.0: ; %entry
9995 ; GFX11-CU-NEXT:    s_clause 0x1
9996 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9997 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9998 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9999 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10000 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10001 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10002 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10003 ; GFX11-CU-NEXT:    s_endpgm
10046 ; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10047 ; GFX10-CU:       ; %bb.0: ; %entry
10048 ; GFX10-CU-NEXT:    s_clause 0x1
10049 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10050 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10051 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10052 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10053 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10054 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10055 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10056 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10057 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10058 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10059 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10060 ; GFX10-CU-NEXT:    buffer_gl0_inv
10061 ; GFX10-CU-NEXT:    buffer_gl1_inv
10062 ; GFX10-CU-NEXT:    s_endpgm
10143 ; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10144 ; GFX11-CU:       ; %bb.0: ; %entry
10145 ; GFX11-CU-NEXT:    s_clause 0x1
10146 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10147 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10148 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10149 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10150 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10151 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10152 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10153 ; GFX11-CU-NEXT:    buffer_gl0_inv
10154 ; GFX11-CU-NEXT:    buffer_gl1_inv
10155 ; GFX11-CU-NEXT:    s_endpgm
10196 ; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10197 ; GFX10-CU:       ; %bb.0: ; %entry
10198 ; GFX10-CU-NEXT:    s_clause 0x1
10199 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10200 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10201 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10202 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10203 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10204 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10205 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10206 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10207 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10208 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10209 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10210 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10211 ; GFX10-CU-NEXT:    s_endpgm
10290 ; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10291 ; GFX11-CU:       ; %bb.0: ; %entry
10292 ; GFX11-CU-NEXT:    s_clause 0x1
10293 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10294 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10295 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10296 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10297 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10298 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10299 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10300 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10301 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10302 ; GFX11-CU-NEXT:    s_endpgm
10348 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10349 ; GFX10-CU:       ; %bb.0: ; %entry
10350 ; GFX10-CU-NEXT:    s_clause 0x1
10351 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10352 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10353 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10354 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10355 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10356 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10357 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10358 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10359 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10360 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10361 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10362 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10363 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10364 ; GFX10-CU-NEXT:    buffer_gl0_inv
10365 ; GFX10-CU-NEXT:    buffer_gl1_inv
10366 ; GFX10-CU-NEXT:    s_endpgm
10458 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10459 ; GFX11-CU:       ; %bb.0: ; %entry
10460 ; GFX11-CU-NEXT:    s_clause 0x1
10461 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10462 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10463 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10464 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10465 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10466 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10467 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10468 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10469 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10470 ; GFX11-CU-NEXT:    buffer_gl0_inv
10471 ; GFX11-CU-NEXT:    buffer_gl1_inv
10472 ; GFX11-CU-NEXT:    s_endpgm
10518 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10519 ; GFX10-CU:       ; %bb.0: ; %entry
10520 ; GFX10-CU-NEXT:    s_clause 0x1
10521 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10522 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10523 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10524 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10525 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10526 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10527 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10528 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10529 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10530 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10531 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10532 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10533 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10534 ; GFX10-CU-NEXT:    buffer_gl0_inv
10535 ; GFX10-CU-NEXT:    buffer_gl1_inv
10536 ; GFX10-CU-NEXT:    s_endpgm
10628 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10629 ; GFX11-CU:       ; %bb.0: ; %entry
10630 ; GFX11-CU-NEXT:    s_clause 0x1
10631 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10632 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10633 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10634 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10635 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10636 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10637 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10638 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10639 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10640 ; GFX11-CU-NEXT:    buffer_gl0_inv
10641 ; GFX11-CU-NEXT:    buffer_gl1_inv
10642 ; GFX11-CU-NEXT:    s_endpgm
10685 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10686 ; GFX10-CU:       ; %bb.0: ; %entry
10687 ; GFX10-CU-NEXT:    s_clause 0x1
10688 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10689 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10690 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10691 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10692 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10693 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10694 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10695 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10696 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10697 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10698 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10699 ; GFX10-CU-NEXT:    buffer_gl0_inv
10700 ; GFX10-CU-NEXT:    buffer_gl1_inv
10701 ; GFX10-CU-NEXT:    s_endpgm
10782 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10783 ; GFX11-CU:       ; %bb.0: ; %entry
10784 ; GFX11-CU-NEXT:    s_clause 0x1
10785 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10786 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10787 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10788 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10789 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10790 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10791 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10792 ; GFX11-CU-NEXT:    buffer_gl0_inv
10793 ; GFX11-CU-NEXT:    buffer_gl1_inv
10794 ; GFX11-CU-NEXT:    s_endpgm
10837 ; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10838 ; GFX10-CU:       ; %bb.0: ; %entry
10839 ; GFX10-CU-NEXT:    s_clause 0x1
10840 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10841 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10842 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10843 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10844 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10845 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10846 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10847 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10848 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10849 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10850 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10851 ; GFX10-CU-NEXT:    buffer_gl0_inv
10852 ; GFX10-CU-NEXT:    buffer_gl1_inv
10853 ; GFX10-CU-NEXT:    s_endpgm
10934 ; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10935 ; GFX11-CU:       ; %bb.0: ; %entry
10936 ; GFX11-CU-NEXT:    s_clause 0x1
10937 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10938 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10939 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10940 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10941 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10942 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10943 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10944 ; GFX11-CU-NEXT:    buffer_gl0_inv
10945 ; GFX11-CU-NEXT:    buffer_gl1_inv
10946 ; GFX11-CU-NEXT:    s_endpgm
10992 ; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
10993 ; GFX10-CU:       ; %bb.0: ; %entry
10994 ; GFX10-CU-NEXT:    s_clause 0x1
10995 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10996 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10997 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10998 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10999 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11000 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11001 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11002 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11003 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11004 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11005 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11006 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11007 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11008 ; GFX10-CU-NEXT:    buffer_gl0_inv
11009 ; GFX10-CU-NEXT:    buffer_gl1_inv
11010 ; GFX10-CU-NEXT:    s_endpgm
11102 ; GFX11-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11103 ; GFX11-CU:       ; %bb.0: ; %entry
11104 ; GFX11-CU-NEXT:    s_clause 0x1
11105 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11106 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11107 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11108 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11109 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11110 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11111 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11112 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11113 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11114 ; GFX11-CU-NEXT:    buffer_gl0_inv
11115 ; GFX11-CU-NEXT:    buffer_gl1_inv
11116 ; GFX11-CU-NEXT:    s_endpgm
11162 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11163 ; GFX10-CU:       ; %bb.0: ; %entry
11164 ; GFX10-CU-NEXT:    s_clause 0x1
11165 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11166 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11167 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11168 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11169 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11170 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11171 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11172 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11173 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11174 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11175 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11176 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11177 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11178 ; GFX10-CU-NEXT:    buffer_gl0_inv
11179 ; GFX10-CU-NEXT:    buffer_gl1_inv
11180 ; GFX10-CU-NEXT:    s_endpgm
11272 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11273 ; GFX11-CU:       ; %bb.0: ; %entry
11274 ; GFX11-CU-NEXT:    s_clause 0x1
11275 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11276 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11277 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11278 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11279 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11280 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11281 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11282 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11283 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11284 ; GFX11-CU-NEXT:    buffer_gl0_inv
11285 ; GFX11-CU-NEXT:    buffer_gl1_inv
11286 ; GFX11-CU-NEXT:    s_endpgm
11332 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11333 ; GFX10-CU:       ; %bb.0: ; %entry
11334 ; GFX10-CU-NEXT:    s_clause 0x1
11335 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11336 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11337 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11338 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11339 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11340 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11341 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11342 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11343 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11344 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11345 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11346 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11347 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11348 ; GFX10-CU-NEXT:    buffer_gl0_inv
11349 ; GFX10-CU-NEXT:    buffer_gl1_inv
11350 ; GFX10-CU-NEXT:    s_endpgm
11442 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11443 ; GFX11-CU:       ; %bb.0: ; %entry
11444 ; GFX11-CU-NEXT:    s_clause 0x1
11445 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11446 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11447 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11448 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11449 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11450 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11451 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11452 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11453 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11454 ; GFX11-CU-NEXT:    buffer_gl0_inv
11455 ; GFX11-CU-NEXT:    buffer_gl1_inv
11456 ; GFX11-CU-NEXT:    s_endpgm
11502 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11503 ; GFX10-CU:       ; %bb.0: ; %entry
11504 ; GFX10-CU-NEXT:    s_clause 0x1
11505 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11506 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11507 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11508 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11509 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11510 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11511 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11512 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11513 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11514 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11515 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11516 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11517 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11518 ; GFX10-CU-NEXT:    buffer_gl0_inv
11519 ; GFX10-CU-NEXT:    buffer_gl1_inv
11520 ; GFX10-CU-NEXT:    s_endpgm
11612 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11613 ; GFX11-CU:       ; %bb.0: ; %entry
11614 ; GFX11-CU-NEXT:    s_clause 0x1
11615 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11616 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11617 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11618 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11619 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11620 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11621 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11622 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11623 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11624 ; GFX11-CU-NEXT:    buffer_gl0_inv
11625 ; GFX11-CU-NEXT:    buffer_gl1_inv
11626 ; GFX11-CU-NEXT:    s_endpgm
11672 ; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11673 ; GFX10-CU:       ; %bb.0: ; %entry
11674 ; GFX10-CU-NEXT:    s_clause 0x1
11675 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11676 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11677 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11678 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11679 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11680 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11681 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11682 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11683 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11684 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11685 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11686 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11687 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11688 ; GFX10-CU-NEXT:    buffer_gl0_inv
11689 ; GFX10-CU-NEXT:    buffer_gl1_inv
11690 ; GFX10-CU-NEXT:    s_endpgm
11782 ; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11783 ; GFX11-CU:       ; %bb.0: ; %entry
11784 ; GFX11-CU-NEXT:    s_clause 0x1
11785 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11786 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11787 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11788 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11789 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11790 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11791 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11792 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11793 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11794 ; GFX11-CU-NEXT:    buffer_gl0_inv
11795 ; GFX11-CU-NEXT:    buffer_gl1_inv
11796 ; GFX11-CU-NEXT:    s_endpgm
11842 ; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11843 ; GFX10-CU:       ; %bb.0: ; %entry
11844 ; GFX10-CU-NEXT:    s_clause 0x1
11845 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11847 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11848 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11849 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11851 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11852 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11853 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11854 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11855 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11856 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11857 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11858 ; GFX10-CU-NEXT:    buffer_gl0_inv
11859 ; GFX10-CU-NEXT:    buffer_gl1_inv
11860 ; GFX10-CU-NEXT:    s_endpgm
11952 ; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11953 ; GFX11-CU:       ; %bb.0: ; %entry
11954 ; GFX11-CU-NEXT:    s_clause 0x1
11955 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11956 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11957 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11958 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11959 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11960 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11961 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11962 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11963 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11964 ; GFX11-CU-NEXT:    buffer_gl0_inv
11965 ; GFX11-CU-NEXT:    buffer_gl1_inv
11966 ; GFX11-CU-NEXT:    s_endpgm
12012 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12013 ; GFX10-CU:       ; %bb.0: ; %entry
12014 ; GFX10-CU-NEXT:    s_clause 0x1
12015 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12016 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12017 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12018 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12019 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12020 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12021 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12022 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12023 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12024 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12025 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12026 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12027 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12028 ; GFX10-CU-NEXT:    buffer_gl0_inv
12029 ; GFX10-CU-NEXT:    buffer_gl1_inv
12030 ; GFX10-CU-NEXT:    s_endpgm
12122 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12123 ; GFX11-CU:       ; %bb.0: ; %entry
12124 ; GFX11-CU-NEXT:    s_clause 0x1
12125 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12126 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12127 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12128 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12129 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12130 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12131 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12132 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12133 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12134 ; GFX11-CU-NEXT:    buffer_gl0_inv
12135 ; GFX11-CU-NEXT:    buffer_gl1_inv
12136 ; GFX11-CU-NEXT:    s_endpgm
12182 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12183 ; GFX10-CU:       ; %bb.0: ; %entry
12184 ; GFX10-CU-NEXT:    s_clause 0x1
12185 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12186 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12187 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12188 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12189 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12190 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12191 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12192 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12193 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12194 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12195 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12196 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12197 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12198 ; GFX10-CU-NEXT:    buffer_gl0_inv
12199 ; GFX10-CU-NEXT:    buffer_gl1_inv
12200 ; GFX10-CU-NEXT:    s_endpgm
12292 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12293 ; GFX11-CU:       ; %bb.0: ; %entry
12294 ; GFX11-CU-NEXT:    s_clause 0x1
12295 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12296 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12297 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12298 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12299 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12300 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12301 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12302 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12303 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12304 ; GFX11-CU-NEXT:    buffer_gl0_inv
12305 ; GFX11-CU-NEXT:    buffer_gl1_inv
12306 ; GFX11-CU-NEXT:    s_endpgm
12352 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12353 ; GFX10-CU:       ; %bb.0: ; %entry
12354 ; GFX10-CU-NEXT:    s_clause 0x1
12355 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12356 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12357 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12358 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12359 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12360 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12361 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12362 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12363 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12364 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12365 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12366 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12367 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12368 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12369 ; GFX10-CU-NEXT:    s_endpgm
12451 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12452 ; GFX11-CU:       ; %bb.0: ; %entry
12453 ; GFX11-CU-NEXT:    s_clause 0x1
12454 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12455 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12456 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12457 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12458 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12459 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12460 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12461 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12462 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12463 ; GFX11-CU-NEXT:    s_endpgm
12516 ; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12517 ; GFX10-CU:       ; %bb.0: ; %entry
12518 ; GFX10-CU-NEXT:    s_clause 0x1
12519 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12520 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12521 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12522 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12523 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12524 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12525 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12526 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12527 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12528 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12529 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12530 ; GFX10-CU-NEXT:    buffer_gl0_inv
12531 ; GFX10-CU-NEXT:    buffer_gl1_inv
12532 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12533 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12534 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12535 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12536 ; GFX10-CU-NEXT:    s_endpgm
12630 ; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12631 ; GFX11-CU:       ; %bb.0: ; %entry
12632 ; GFX11-CU-NEXT:    s_clause 0x1
12633 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12634 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12635 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12636 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12637 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12638 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12639 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12640 ; GFX11-CU-NEXT:    buffer_gl0_inv
12641 ; GFX11-CU-NEXT:    buffer_gl1_inv
12642 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12643 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12644 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12645 ; GFX11-CU-NEXT:    s_endpgm
12696 ; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12697 ; GFX10-CU:       ; %bb.0: ; %entry
12698 ; GFX10-CU-NEXT:    s_clause 0x1
12699 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12700 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12701 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12702 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12703 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12704 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12705 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12706 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12707 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12708 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12709 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12710 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12711 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12712 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12713 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12714 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12715 ; GFX10-CU-NEXT:    s_endpgm
12808 ; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12809 ; GFX11-CU:       ; %bb.0: ; %entry
12810 ; GFX11-CU-NEXT:    s_clause 0x1
12811 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12812 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12813 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12814 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12815 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12816 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12817 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12818 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12819 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12820 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12821 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12822 ; GFX11-CU-NEXT:    s_endpgm
12878 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12879 ; GFX10-CU:       ; %bb.0: ; %entry
12880 ; GFX10-CU-NEXT:    s_clause 0x1
12881 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12882 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12883 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12884 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12885 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12886 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12887 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12888 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12889 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12890 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12891 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12892 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12893 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12894 ; GFX10-CU-NEXT:    buffer_gl0_inv
12895 ; GFX10-CU-NEXT:    buffer_gl1_inv
12896 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12897 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12898 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12899 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12900 ; GFX10-CU-NEXT:    s_endpgm
13005 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13006 ; GFX11-CU:       ; %bb.0: ; %entry
13007 ; GFX11-CU-NEXT:    s_clause 0x1
13008 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13009 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13010 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13011 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13012 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13013 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13014 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13015 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13016 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13017 ; GFX11-CU-NEXT:    buffer_gl0_inv
13018 ; GFX11-CU-NEXT:    buffer_gl1_inv
13019 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13020 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13021 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13022 ; GFX11-CU-NEXT:    s_endpgm
13078 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13079 ; GFX10-CU:       ; %bb.0: ; %entry
13080 ; GFX10-CU-NEXT:    s_clause 0x1
13081 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13082 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13083 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13084 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13085 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13086 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13087 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13088 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13089 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13090 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13091 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13092 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13093 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13094 ; GFX10-CU-NEXT:    buffer_gl0_inv
13095 ; GFX10-CU-NEXT:    buffer_gl1_inv
13096 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13097 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13098 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13099 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13100 ; GFX10-CU-NEXT:    s_endpgm
13205 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13206 ; GFX11-CU:       ; %bb.0: ; %entry
13207 ; GFX11-CU-NEXT:    s_clause 0x1
13208 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13209 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13210 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13211 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13212 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13213 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13214 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13215 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13216 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13217 ; GFX11-CU-NEXT:    buffer_gl0_inv
13218 ; GFX11-CU-NEXT:    buffer_gl1_inv
13219 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13220 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13221 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13222 ; GFX11-CU-NEXT:    s_endpgm
13275 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13276 ; GFX10-CU:       ; %bb.0: ; %entry
13277 ; GFX10-CU-NEXT:    s_clause 0x1
13278 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13279 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13280 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13281 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13282 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13283 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13284 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13285 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13286 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13287 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13288 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13289 ; GFX10-CU-NEXT:    buffer_gl0_inv
13290 ; GFX10-CU-NEXT:    buffer_gl1_inv
13291 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13292 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13293 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13294 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13295 ; GFX10-CU-NEXT:    s_endpgm
13389 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13390 ; GFX11-CU:       ; %bb.0: ; %entry
13391 ; GFX11-CU-NEXT:    s_clause 0x1
13392 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13393 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13394 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13395 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13396 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13397 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13398 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13399 ; GFX11-CU-NEXT:    buffer_gl0_inv
13400 ; GFX11-CU-NEXT:    buffer_gl1_inv
13401 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13402 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13403 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13404 ; GFX11-CU-NEXT:    s_endpgm
13457 ; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13458 ; GFX10-CU:       ; %bb.0: ; %entry
13459 ; GFX10-CU-NEXT:    s_clause 0x1
13460 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13461 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13462 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13463 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13464 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13465 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13466 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13467 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13469 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13470 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13471 ; GFX10-CU-NEXT:    buffer_gl0_inv
13472 ; GFX10-CU-NEXT:    buffer_gl1_inv
13473 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13474 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13475 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13476 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13477 ; GFX10-CU-NEXT:    s_endpgm
13571 ; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13572 ; GFX11-CU:       ; %bb.0: ; %entry
13573 ; GFX11-CU-NEXT:    s_clause 0x1
13574 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13575 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13576 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13577 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13578 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13579 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13580 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13581 ; GFX11-CU-NEXT:    buffer_gl0_inv
13582 ; GFX11-CU-NEXT:    buffer_gl1_inv
13583 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13584 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13585 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13586 ; GFX11-CU-NEXT:    s_endpgm
13642 ; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13643 ; GFX10-CU:       ; %bb.0: ; %entry
13644 ; GFX10-CU-NEXT:    s_clause 0x1
13645 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13646 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13647 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13648 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13649 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13650 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13651 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13652 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13653 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13654 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13655 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13656 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13657 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13658 ; GFX10-CU-NEXT:    buffer_gl0_inv
13659 ; GFX10-CU-NEXT:    buffer_gl1_inv
13660 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13661 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13662 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13663 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13664 ; GFX10-CU-NEXT:    s_endpgm
13769 ; GFX11-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13770 ; GFX11-CU:       ; %bb.0: ; %entry
13771 ; GFX11-CU-NEXT:    s_clause 0x1
13772 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13773 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13774 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13775 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13776 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13777 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13778 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13779 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13780 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13781 ; GFX11-CU-NEXT:    buffer_gl0_inv
13782 ; GFX11-CU-NEXT:    buffer_gl1_inv
13783 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13784 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13785 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13786 ; GFX11-CU-NEXT:    s_endpgm
13842 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13843 ; GFX10-CU:       ; %bb.0: ; %entry
13844 ; GFX10-CU-NEXT:    s_clause 0x1
13845 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13847 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13848 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13849 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13851 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13852 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13853 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13854 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13855 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13856 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13857 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13858 ; GFX10-CU-NEXT:    buffer_gl0_inv
13859 ; GFX10-CU-NEXT:    buffer_gl1_inv
13860 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13861 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13862 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13863 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13864 ; GFX10-CU-NEXT:    s_endpgm
13969 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13970 ; GFX11-CU:       ; %bb.0: ; %entry
13971 ; GFX11-CU-NEXT:    s_clause 0x1
13972 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13973 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13974 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13975 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13976 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13977 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13978 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13979 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13980 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13981 ; GFX11-CU-NEXT:    buffer_gl0_inv
13982 ; GFX11-CU-NEXT:    buffer_gl1_inv
13983 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13984 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13985 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13986 ; GFX11-CU-NEXT:    s_endpgm
14042 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14043 ; GFX10-CU:       ; %bb.0: ; %entry
14044 ; GFX10-CU-NEXT:    s_clause 0x1
14045 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14046 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14047 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14048 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14049 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14050 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14051 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14052 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14053 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14054 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14055 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14056 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14057 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14058 ; GFX10-CU-NEXT:    buffer_gl0_inv
14059 ; GFX10-CU-NEXT:    buffer_gl1_inv
14060 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14061 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14062 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14063 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14064 ; GFX10-CU-NEXT:    s_endpgm
14169 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14170 ; GFX11-CU:       ; %bb.0: ; %entry
14171 ; GFX11-CU-NEXT:    s_clause 0x1
14172 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14173 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14174 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14175 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14176 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14177 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14178 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14179 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14180 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14181 ; GFX11-CU-NEXT:    buffer_gl0_inv
14182 ; GFX11-CU-NEXT:    buffer_gl1_inv
14183 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14184 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14185 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14186 ; GFX11-CU-NEXT:    s_endpgm
14242 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14243 ; GFX10-CU:       ; %bb.0: ; %entry
14244 ; GFX10-CU-NEXT:    s_clause 0x1
14245 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14246 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14247 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14248 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14249 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14250 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14251 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14252 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14253 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14254 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14255 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14256 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14257 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14258 ; GFX10-CU-NEXT:    buffer_gl0_inv
14259 ; GFX10-CU-NEXT:    buffer_gl1_inv
14260 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14261 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14262 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14263 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14264 ; GFX10-CU-NEXT:    s_endpgm
14369 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14370 ; GFX11-CU:       ; %bb.0: ; %entry
14371 ; GFX11-CU-NEXT:    s_clause 0x1
14372 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14373 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14374 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14375 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14376 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14377 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14378 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14379 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14380 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14381 ; GFX11-CU-NEXT:    buffer_gl0_inv
14382 ; GFX11-CU-NEXT:    buffer_gl1_inv
14383 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14384 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14385 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14386 ; GFX11-CU-NEXT:    s_endpgm
14442 ; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14443 ; GFX10-CU:       ; %bb.0: ; %entry
14444 ; GFX10-CU-NEXT:    s_clause 0x1
14445 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14446 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14447 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14448 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14449 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14450 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14451 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14452 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14453 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14454 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14455 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14456 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14457 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14458 ; GFX10-CU-NEXT:    buffer_gl0_inv
14459 ; GFX10-CU-NEXT:    buffer_gl1_inv
14460 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14461 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14462 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14463 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14464 ; GFX10-CU-NEXT:    s_endpgm
14569 ; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14570 ; GFX11-CU:       ; %bb.0: ; %entry
14571 ; GFX11-CU-NEXT:    s_clause 0x1
14572 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14573 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14574 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14575 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14576 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14577 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14578 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14579 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14580 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14581 ; GFX11-CU-NEXT:    buffer_gl0_inv
14582 ; GFX11-CU-NEXT:    buffer_gl1_inv
14583 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14584 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14585 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14586 ; GFX11-CU-NEXT:    s_endpgm
14642 ; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14643 ; GFX10-CU:       ; %bb.0: ; %entry
14644 ; GFX10-CU-NEXT:    s_clause 0x1
14645 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14646 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14647 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14648 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14649 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14650 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14651 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14652 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14653 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14654 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14655 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14656 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14657 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14658 ; GFX10-CU-NEXT:    buffer_gl0_inv
14659 ; GFX10-CU-NEXT:    buffer_gl1_inv
14660 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14661 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14662 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14663 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14664 ; GFX10-CU-NEXT:    s_endpgm
14769 ; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14770 ; GFX11-CU:       ; %bb.0: ; %entry
14771 ; GFX11-CU-NEXT:    s_clause 0x1
14772 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14773 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14774 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14775 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14776 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14777 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14778 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14779 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14780 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14781 ; GFX11-CU-NEXT:    buffer_gl0_inv
14782 ; GFX11-CU-NEXT:    buffer_gl1_inv
14783 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14784 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14785 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14786 ; GFX11-CU-NEXT:    s_endpgm
14842 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14843 ; GFX10-CU:       ; %bb.0: ; %entry
14844 ; GFX10-CU-NEXT:    s_clause 0x1
14845 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14847 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14848 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14849 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14851 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14852 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14853 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14854 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14855 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14856 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14857 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14858 ; GFX10-CU-NEXT:    buffer_gl0_inv
14859 ; GFX10-CU-NEXT:    buffer_gl1_inv
14860 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14861 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14862 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14863 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14864 ; GFX10-CU-NEXT:    s_endpgm
14969 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14970 ; GFX11-CU:       ; %bb.0: ; %entry
14971 ; GFX11-CU-NEXT:    s_clause 0x1
14972 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14973 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14974 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14975 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14976 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14977 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14978 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14979 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14980 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14981 ; GFX11-CU-NEXT:    buffer_gl0_inv
14982 ; GFX11-CU-NEXT:    buffer_gl1_inv
14983 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14984 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14985 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14986 ; GFX11-CU-NEXT:    s_endpgm
15042 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15043 ; GFX10-CU:       ; %bb.0: ; %entry
15044 ; GFX10-CU-NEXT:    s_clause 0x1
15045 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15046 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15047 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15048 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
15049 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
15050 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
15051 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
15052 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
15053 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
15054 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15055 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15056 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15057 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15058 ; GFX10-CU-NEXT:    buffer_gl0_inv
15059 ; GFX10-CU-NEXT:    buffer_gl1_inv
15060 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
15061 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
15062 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15063 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
15064 ; GFX10-CU-NEXT:    s_endpgm
15169 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15170 ; GFX11-CU:       ; %bb.0: ; %entry
15171 ; GFX11-CU-NEXT:    s_clause 0x1
15172 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15173 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15174 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15175 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
15176 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
15177 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15178 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15179 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15180 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15181 ; GFX11-CU-NEXT:    buffer_gl0_inv
15182 ; GFX11-CU-NEXT:    buffer_gl1_inv
15183 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15184 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
15185 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
15186 ; GFX11-CU-NEXT:    s_endpgm