memory-legalizer-flat-wavefront.ll - OpenGrok cross reference for /llvm-project-15.0.7/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll

Lines Matching refs:CU
4 …sa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
11 …sa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
40 ; GFX10-CU-LABEL: flat_wavefront_unordered_load:
41 ; GFX10-CU:       ; %bb.0: ; %entry
42 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51 ; GFX10-CU-NEXT:    s_endpgm
130 ; GFX11-CU-LABEL: flat_wavefront_unordered_load:
131 ; GFX11-CU:       ; %bb.0: ; %entry
132 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
133 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
135 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
136 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
137 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
138 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
139 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
140 ; GFX11-CU-NEXT:    s_endpgm
175 ; GFX10-CU-LABEL: flat_wavefront_monotonic_load:
176 ; GFX10-CU:       ; %bb.0: ; %entry
177 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
179 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
180 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
181 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
182 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
183 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
184 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
186 ; GFX10-CU-NEXT:    s_endpgm
265 ; GFX11-CU-LABEL: flat_wavefront_monotonic_load:
266 ; GFX11-CU:       ; %bb.0: ; %entry
267 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
268 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
270 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
271 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
272 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
273 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
274 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
275 ; GFX11-CU-NEXT:    s_endpgm
310 ; GFX10-CU-LABEL: flat_wavefront_acquire_load:
311 ; GFX10-CU:       ; %bb.0: ; %entry
312 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
313 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
314 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
315 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
316 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
317 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
318 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
319 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
320 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
321 ; GFX10-CU-NEXT:    s_endpgm
400 ; GFX11-CU-LABEL: flat_wavefront_acquire_load:
401 ; GFX11-CU:       ; %bb.0: ; %entry
402 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
403 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
404 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
405 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
406 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
407 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
408 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
409 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
410 ; GFX11-CU-NEXT:    s_endpgm
445 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_load:
446 ; GFX10-CU:       ; %bb.0: ; %entry
447 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
448 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
449 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
450 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
451 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
452 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
453 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
454 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
455 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
456 ; GFX10-CU-NEXT:    s_endpgm
535 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_load:
536 ; GFX11-CU:       ; %bb.0: ; %entry
537 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
538 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
539 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
540 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
541 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
542 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
543 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
544 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
545 ; GFX11-CU-NEXT:    s_endpgm
577 ; GFX10-CU-LABEL: flat_wavefront_unordered_store:
578 ; GFX10-CU:       ; %bb.0: ; %entry
579 ; GFX10-CU-NEXT:    s_clause 0x1
580 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
581 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
582 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
583 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
584 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
585 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
586 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
587 ; GFX10-CU-NEXT:    s_endpgm
652 ; GFX11-CU-LABEL: flat_wavefront_unordered_store:
653 ; GFX11-CU:       ; %bb.0: ; %entry
654 ; GFX11-CU-NEXT:    s_clause 0x1
655 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
656 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
657 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
658 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
659 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
660 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
661 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
662 ; GFX11-CU-NEXT:    s_endpgm
693 ; GFX10-CU-LABEL: flat_wavefront_monotonic_store:
694 ; GFX10-CU:       ; %bb.0: ; %entry
695 ; GFX10-CU-NEXT:    s_clause 0x1
696 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
697 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
698 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
699 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
700 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
701 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
702 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
703 ; GFX10-CU-NEXT:    s_endpgm
768 ; GFX11-CU-LABEL: flat_wavefront_monotonic_store:
769 ; GFX11-CU:       ; %bb.0: ; %entry
770 ; GFX11-CU-NEXT:    s_clause 0x1
771 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
772 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
773 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
774 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
775 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
776 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
777 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
778 ; GFX11-CU-NEXT:    s_endpgm
809 ; GFX10-CU-LABEL: flat_wavefront_release_store:
810 ; GFX10-CU:       ; %bb.0: ; %entry
811 ; GFX10-CU-NEXT:    s_clause 0x1
812 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
813 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
814 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
815 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
816 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
817 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
818 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
819 ; GFX10-CU-NEXT:    s_endpgm
884 ; GFX11-CU-LABEL: flat_wavefront_release_store:
885 ; GFX11-CU:       ; %bb.0: ; %entry
886 ; GFX11-CU-NEXT:    s_clause 0x1
887 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
888 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
889 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
890 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
891 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
892 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
893 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
894 ; GFX11-CU-NEXT:    s_endpgm
925 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_store:
926 ; GFX10-CU:       ; %bb.0: ; %entry
927 ; GFX10-CU-NEXT:    s_clause 0x1
928 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
929 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
930 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
931 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
932 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
933 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
934 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
935 ; GFX10-CU-NEXT:    s_endpgm
1000 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_store:
1001 ; GFX11-CU:       ; %bb.0: ; %entry
1002 ; GFX11-CU-NEXT:    s_clause 0x1
1003 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1004 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1005 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1006 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1007 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1008 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1009 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1010 ; GFX11-CU-NEXT:    s_endpgm
1041 ; GFX10-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
1042 ; GFX10-CU:       ; %bb.0: ; %entry
1043 ; GFX10-CU-NEXT:    s_clause 0x1
1044 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1045 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1046 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1047 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1048 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1049 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1050 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1051 ; GFX10-CU-NEXT:    s_endpgm
1116 ; GFX11-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
1117 ; GFX11-CU:       ; %bb.0: ; %entry
1118 ; GFX11-CU-NEXT:    s_clause 0x1
1119 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1120 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1121 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1122 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1123 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1124 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1125 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1126 ; GFX11-CU-NEXT:    s_endpgm
1157 ; GFX10-CU-LABEL: flat_wavefront_acquire_atomicrmw:
1158 ; GFX10-CU:       ; %bb.0: ; %entry
1159 ; GFX10-CU-NEXT:    s_clause 0x1
1160 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1161 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1162 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1163 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1164 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1165 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1166 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1167 ; GFX10-CU-NEXT:    s_endpgm
1232 ; GFX11-CU-LABEL: flat_wavefront_acquire_atomicrmw:
1233 ; GFX11-CU:       ; %bb.0: ; %entry
1234 ; GFX11-CU-NEXT:    s_clause 0x1
1235 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1236 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1237 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1238 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1239 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1240 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1241 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1242 ; GFX11-CU-NEXT:    s_endpgm
1273 ; GFX10-CU-LABEL: flat_wavefront_release_atomicrmw:
1274 ; GFX10-CU:       ; %bb.0: ; %entry
1275 ; GFX10-CU-NEXT:    s_clause 0x1
1276 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1277 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1278 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1279 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1280 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1281 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1282 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1283 ; GFX10-CU-NEXT:    s_endpgm
1348 ; GFX11-CU-LABEL: flat_wavefront_release_atomicrmw:
1349 ; GFX11-CU:       ; %bb.0: ; %entry
1350 ; GFX11-CU-NEXT:    s_clause 0x1
1351 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1352 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1353 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1354 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1355 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1356 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1357 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1358 ; GFX11-CU-NEXT:    s_endpgm
1389 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
1390 ; GFX10-CU:       ; %bb.0: ; %entry
1391 ; GFX10-CU-NEXT:    s_clause 0x1
1392 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1393 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1394 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1395 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1396 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1397 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1398 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1399 ; GFX10-CU-NEXT:    s_endpgm
1464 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
1465 ; GFX11-CU:       ; %bb.0: ; %entry
1466 ; GFX11-CU-NEXT:    s_clause 0x1
1467 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1468 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1469 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1470 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1471 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1472 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1473 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1474 ; GFX11-CU-NEXT:    s_endpgm
1505 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
1506 ; GFX10-CU:       ; %bb.0: ; %entry
1507 ; GFX10-CU-NEXT:    s_clause 0x1
1508 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1509 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1510 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1511 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1512 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1513 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1514 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1515 ; GFX10-CU-NEXT:    s_endpgm
1580 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
1581 ; GFX11-CU:       ; %bb.0: ; %entry
1582 ; GFX11-CU-NEXT:    s_clause 0x1
1583 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1584 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1585 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1586 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1587 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1588 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1589 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1590 ; GFX11-CU-NEXT:    s_endpgm
1625 ; GFX10-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1626 ; GFX10-CU:       ; %bb.0: ; %entry
1627 ; GFX10-CU-NEXT:    s_clause 0x1
1628 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1629 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1630 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1631 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1632 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1633 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1634 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1635 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1636 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1637 ; GFX10-CU-NEXT:    s_endpgm
1714 ; GFX11-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1715 ; GFX11-CU:       ; %bb.0: ; %entry
1716 ; GFX11-CU-NEXT:    s_clause 0x1
1717 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1718 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1719 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1720 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1721 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1722 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1723 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1724 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1725 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1726 ; GFX11-CU-NEXT:    s_endpgm
1762 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1763 ; GFX10-CU:       ; %bb.0: ; %entry
1764 ; GFX10-CU-NEXT:    s_clause 0x1
1765 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1766 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1767 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1768 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1769 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1770 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1771 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1772 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1773 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1774 ; GFX10-CU-NEXT:    s_endpgm
1851 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1852 ; GFX11-CU:       ; %bb.0: ; %entry
1853 ; GFX11-CU-NEXT:    s_clause 0x1
1854 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1855 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1856 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1857 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1858 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1859 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1860 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1861 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1862 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1863 ; GFX11-CU-NEXT:    s_endpgm
1899 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1900 ; GFX10-CU:       ; %bb.0: ; %entry
1901 ; GFX10-CU-NEXT:    s_clause 0x1
1902 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1903 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1904 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1905 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1906 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1907 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1908 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1909 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1910 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1911 ; GFX10-CU-NEXT:    s_endpgm
1988 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1989 ; GFX11-CU:       ; %bb.0: ; %entry
1990 ; GFX11-CU-NEXT:    s_clause 0x1
1991 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1992 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1993 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1994 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1995 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1996 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1997 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1998 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1999 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2000 ; GFX11-CU-NEXT:    s_endpgm
2038 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2039 ; GFX10-CU:       ; %bb.0: ; %entry
2040 ; GFX10-CU-NEXT:    s_clause 0x1
2041 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2042 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2043 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2044 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2045 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2046 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2047 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2048 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2049 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2050 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2051 ; GFX10-CU-NEXT:    s_endpgm
2119 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2120 ; GFX11-CU:       ; %bb.0: ; %entry
2121 ; GFX11-CU-NEXT:    s_clause 0x1
2122 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2123 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2124 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2125 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2126 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2127 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2128 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2129 ; GFX11-CU-NEXT:    s_endpgm
2167 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2168 ; GFX10-CU:       ; %bb.0: ; %entry
2169 ; GFX10-CU-NEXT:    s_clause 0x1
2170 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2171 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2172 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2173 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2174 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2175 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2176 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2177 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2178 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2179 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2180 ; GFX10-CU-NEXT:    s_endpgm
2248 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2249 ; GFX11-CU:       ; %bb.0: ; %entry
2250 ; GFX11-CU-NEXT:    s_clause 0x1
2251 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2252 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2253 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2254 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2255 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2256 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2257 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2258 ; GFX11-CU-NEXT:    s_endpgm
2296 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2297 ; GFX10-CU:       ; %bb.0: ; %entry
2298 ; GFX10-CU-NEXT:    s_clause 0x1
2299 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2300 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2301 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2302 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2303 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2304 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2305 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2306 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2307 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2308 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2309 ; GFX10-CU-NEXT:    s_endpgm
2377 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2378 ; GFX11-CU:       ; %bb.0: ; %entry
2379 ; GFX11-CU-NEXT:    s_clause 0x1
2380 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2381 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2382 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2383 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2384 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2385 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2386 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2387 ; GFX11-CU-NEXT:    s_endpgm
2425 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2426 ; GFX10-CU:       ; %bb.0: ; %entry
2427 ; GFX10-CU-NEXT:    s_clause 0x1
2428 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2429 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2430 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2431 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2432 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2433 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2434 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2435 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2436 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2437 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2438 ; GFX10-CU-NEXT:    s_endpgm
2506 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2507 ; GFX11-CU:       ; %bb.0: ; %entry
2508 ; GFX11-CU-NEXT:    s_clause 0x1
2509 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2510 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2511 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2512 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2513 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2514 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2515 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2516 ; GFX11-CU-NEXT:    s_endpgm
2554 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2555 ; GFX10-CU:       ; %bb.0: ; %entry
2556 ; GFX10-CU-NEXT:    s_clause 0x1
2557 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2558 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2559 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2560 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2561 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2562 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2563 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2564 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2565 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2566 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2567 ; GFX10-CU-NEXT:    s_endpgm
2635 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2636 ; GFX11-CU:       ; %bb.0: ; %entry
2637 ; GFX11-CU-NEXT:    s_clause 0x1
2638 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2639 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2640 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2641 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2642 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2643 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2644 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2645 ; GFX11-CU-NEXT:    s_endpgm
2683 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2684 ; GFX10-CU:       ; %bb.0: ; %entry
2685 ; GFX10-CU-NEXT:    s_clause 0x1
2686 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2687 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2688 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2689 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2690 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2691 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2692 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2693 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2694 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2695 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2696 ; GFX10-CU-NEXT:    s_endpgm
2764 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2765 ; GFX11-CU:       ; %bb.0: ; %entry
2766 ; GFX11-CU-NEXT:    s_clause 0x1
2767 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2768 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2769 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2770 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2771 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2772 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2773 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2774 ; GFX11-CU-NEXT:    s_endpgm
2812 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2813 ; GFX10-CU:       ; %bb.0: ; %entry
2814 ; GFX10-CU-NEXT:    s_clause 0x1
2815 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2816 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2817 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2818 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2819 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2820 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2821 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2822 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2823 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2824 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2825 ; GFX10-CU-NEXT:    s_endpgm
2893 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2894 ; GFX11-CU:       ; %bb.0: ; %entry
2895 ; GFX11-CU-NEXT:    s_clause 0x1
2896 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2897 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2898 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2899 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2900 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2901 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2902 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2903 ; GFX11-CU-NEXT:    s_endpgm
2941 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
2942 ; GFX10-CU:       ; %bb.0: ; %entry
2943 ; GFX10-CU-NEXT:    s_clause 0x1
2944 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2945 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2946 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2947 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2948 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2949 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2950 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2951 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2952 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2953 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2954 ; GFX10-CU-NEXT:    s_endpgm
3022 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
3023 ; GFX11-CU:       ; %bb.0: ; %entry
3024 ; GFX11-CU-NEXT:    s_clause 0x1
3025 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3026 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3027 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3028 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3029 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3030 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3031 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3032 ; GFX11-CU-NEXT:    s_endpgm
3070 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
3071 ; GFX10-CU:       ; %bb.0: ; %entry
3072 ; GFX10-CU-NEXT:    s_clause 0x1
3073 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3074 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3075 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3076 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3077 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3078 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3079 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3080 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3081 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3082 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3083 ; GFX10-CU-NEXT:    s_endpgm
3151 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
3152 ; GFX11-CU:       ; %bb.0: ; %entry
3153 ; GFX11-CU-NEXT:    s_clause 0x1
3154 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3155 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3156 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3157 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3158 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3159 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3160 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3161 ; GFX11-CU-NEXT:    s_endpgm
3199 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3200 ; GFX10-CU:       ; %bb.0: ; %entry
3201 ; GFX10-CU-NEXT:    s_clause 0x1
3202 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3203 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3204 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3205 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3206 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3207 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3208 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3209 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3210 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3211 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3212 ; GFX10-CU-NEXT:    s_endpgm
3280 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3281 ; GFX11-CU:       ; %bb.0: ; %entry
3282 ; GFX11-CU-NEXT:    s_clause 0x1
3283 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3284 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3285 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3286 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3287 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3288 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3289 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3290 ; GFX11-CU-NEXT:    s_endpgm
3328 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3329 ; GFX10-CU:       ; %bb.0: ; %entry
3330 ; GFX10-CU-NEXT:    s_clause 0x1
3331 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3332 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3333 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3334 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3335 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3336 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3337 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3338 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3339 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3340 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3341 ; GFX10-CU-NEXT:    s_endpgm
3409 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3410 ; GFX11-CU:       ; %bb.0: ; %entry
3411 ; GFX11-CU-NEXT:    s_clause 0x1
3412 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3413 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3414 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3415 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3416 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3417 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3418 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3419 ; GFX11-CU-NEXT:    s_endpgm
3457 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3458 ; GFX10-CU:       ; %bb.0: ; %entry
3459 ; GFX10-CU-NEXT:    s_clause 0x1
3460 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3461 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3462 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3463 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3464 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3465 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3466 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3467 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3469 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3470 ; GFX10-CU-NEXT:    s_endpgm
3538 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3539 ; GFX11-CU:       ; %bb.0: ; %entry
3540 ; GFX11-CU-NEXT:    s_clause 0x1
3541 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3542 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3543 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3544 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3545 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3546 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3547 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3548 ; GFX11-CU-NEXT:    s_endpgm
3586 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3587 ; GFX10-CU:       ; %bb.0: ; %entry
3588 ; GFX10-CU-NEXT:    s_clause 0x1
3589 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3590 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3591 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3592 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3593 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3594 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3595 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3596 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3597 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3598 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3599 ; GFX10-CU-NEXT:    s_endpgm
3667 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3668 ; GFX11-CU:       ; %bb.0: ; %entry
3669 ; GFX11-CU-NEXT:    s_clause 0x1
3670 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3671 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3672 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3673 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3674 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3675 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3676 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3677 ; GFX11-CU-NEXT:    s_endpgm
3715 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3716 ; GFX10-CU:       ; %bb.0: ; %entry
3717 ; GFX10-CU-NEXT:    s_clause 0x1
3718 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3719 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3720 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3721 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3722 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3723 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3724 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3725 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3726 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3727 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3728 ; GFX10-CU-NEXT:    s_endpgm
3796 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3797 ; GFX11-CU:       ; %bb.0: ; %entry
3798 ; GFX11-CU-NEXT:    s_clause 0x1
3799 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3800 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3801 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3802 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3803 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3804 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3805 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3806 ; GFX11-CU-NEXT:    s_endpgm
3844 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3845 ; GFX10-CU:       ; %bb.0: ; %entry
3846 ; GFX10-CU-NEXT:    s_clause 0x1
3847 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3848 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3849 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3850 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3851 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3852 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3853 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3854 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3855 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3856 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3857 ; GFX10-CU-NEXT:    s_endpgm
3925 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3926 ; GFX11-CU:       ; %bb.0: ; %entry
3927 ; GFX11-CU-NEXT:    s_clause 0x1
3928 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3929 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3930 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3931 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3932 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3933 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3934 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3935 ; GFX11-CU-NEXT:    s_endpgm
3981 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3982 ; GFX10-CU:       ; %bb.0: ; %entry
3983 ; GFX10-CU-NEXT:    s_clause 0x1
3984 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3985 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3986 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3987 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3988 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3989 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3990 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3991 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3992 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3993 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3994 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3995 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3996 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3997 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3998 ; GFX10-CU-NEXT:    s_endpgm
4080 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
4081 ; GFX11-CU:       ; %bb.0: ; %entry
4082 ; GFX11-CU-NEXT:    s_clause 0x1
4083 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4084 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4085 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4086 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4087 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4088 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4089 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4090 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4091 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4092 ; GFX11-CU-NEXT:    s_endpgm
4140 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
4141 ; GFX10-CU:       ; %bb.0: ; %entry
4142 ; GFX10-CU-NEXT:    s_clause 0x1
4143 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4144 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4145 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4146 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4147 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4148 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4149 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4150 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4151 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4152 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4153 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4154 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4155 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4156 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4157 ; GFX10-CU-NEXT:    s_endpgm
4239 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
4240 ; GFX11-CU:       ; %bb.0: ; %entry
4241 ; GFX11-CU-NEXT:    s_clause 0x1
4242 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4243 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4244 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4245 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4246 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4247 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4248 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4249 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4250 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4251 ; GFX11-CU-NEXT:    s_endpgm
4299 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4300 ; GFX10-CU:       ; %bb.0: ; %entry
4301 ; GFX10-CU-NEXT:    s_clause 0x1
4302 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4303 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4304 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4305 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4306 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4307 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4308 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4309 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4310 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4311 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4312 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4313 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4314 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4315 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4316 ; GFX10-CU-NEXT:    s_endpgm
4398 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4399 ; GFX11-CU:       ; %bb.0: ; %entry
4400 ; GFX11-CU-NEXT:    s_clause 0x1
4401 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4402 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4403 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4404 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4405 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4406 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4407 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4408 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4409 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4410 ; GFX11-CU-NEXT:    s_endpgm
4458 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4459 ; GFX10-CU:       ; %bb.0: ; %entry
4460 ; GFX10-CU-NEXT:    s_clause 0x1
4461 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4462 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4463 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4464 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4465 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4466 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4467 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4469 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4470 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4471 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4472 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4473 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4474 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4475 ; GFX10-CU-NEXT:    s_endpgm
4557 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4558 ; GFX11-CU:       ; %bb.0: ; %entry
4559 ; GFX11-CU-NEXT:    s_clause 0x1
4560 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4561 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4562 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4563 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4564 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4565 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4566 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4567 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4568 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4569 ; GFX11-CU-NEXT:    s_endpgm
4617 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4618 ; GFX10-CU:       ; %bb.0: ; %entry
4619 ; GFX10-CU-NEXT:    s_clause 0x1
4620 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4621 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4622 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4623 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4624 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4625 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4626 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4627 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4628 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4629 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4630 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4631 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4632 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4633 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4634 ; GFX10-CU-NEXT:    s_endpgm
4716 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4717 ; GFX11-CU:       ; %bb.0: ; %entry
4718 ; GFX11-CU-NEXT:    s_clause 0x1
4719 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4720 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4721 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4722 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4723 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4724 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4725 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4726 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4727 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4728 ; GFX11-CU-NEXT:    s_endpgm
4776 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4777 ; GFX10-CU:       ; %bb.0: ; %entry
4778 ; GFX10-CU-NEXT:    s_clause 0x1
4779 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4780 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4781 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4782 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4783 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4784 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4785 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4786 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4787 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4788 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4789 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4790 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4791 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4792 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4793 ; GFX10-CU-NEXT:    s_endpgm
4875 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4876 ; GFX11-CU:       ; %bb.0: ; %entry
4877 ; GFX11-CU-NEXT:    s_clause 0x1
4878 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4879 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4880 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4881 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4882 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4883 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4884 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4885 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4886 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4887 ; GFX11-CU-NEXT:    s_endpgm
4935 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4936 ; GFX10-CU:       ; %bb.0: ; %entry
4937 ; GFX10-CU-NEXT:    s_clause 0x1
4938 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4939 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4940 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4941 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4942 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4943 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4944 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4945 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4946 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4947 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4948 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4949 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4950 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4951 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4952 ; GFX10-CU-NEXT:    s_endpgm
5034 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
5035 ; GFX11-CU:       ; %bb.0: ; %entry
5036 ; GFX11-CU-NEXT:    s_clause 0x1
5037 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5038 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5039 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5040 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5041 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5042 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5043 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5044 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5045 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5046 ; GFX11-CU-NEXT:    s_endpgm
5094 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
5095 ; GFX10-CU:       ; %bb.0: ; %entry
5096 ; GFX10-CU-NEXT:    s_clause 0x1
5097 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5098 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5099 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5100 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5101 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5102 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5103 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5104 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5105 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5106 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5107 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5108 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5109 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5110 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5111 ; GFX10-CU-NEXT:    s_endpgm
5193 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
5194 ; GFX11-CU:       ; %bb.0: ; %entry
5195 ; GFX11-CU-NEXT:    s_clause 0x1
5196 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5197 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5198 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5199 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5200 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5201 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5202 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5203 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5204 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5205 ; GFX11-CU-NEXT:    s_endpgm
5253 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
5254 ; GFX10-CU:       ; %bb.0: ; %entry
5255 ; GFX10-CU-NEXT:    s_clause 0x1
5256 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5257 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5258 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5259 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5260 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5261 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5262 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5263 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5264 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5265 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5266 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5267 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5268 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5269 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5270 ; GFX10-CU-NEXT:    s_endpgm
5352 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
5353 ; GFX11-CU:       ; %bb.0: ; %entry
5354 ; GFX11-CU-NEXT:    s_clause 0x1
5355 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5356 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5357 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5358 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5359 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5360 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5361 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5362 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5363 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5364 ; GFX11-CU-NEXT:    s_endpgm
5412 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5413 ; GFX10-CU:       ; %bb.0: ; %entry
5414 ; GFX10-CU-NEXT:    s_clause 0x1
5415 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5416 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5417 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5418 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5419 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5420 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5421 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5422 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5423 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5424 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5425 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5426 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5427 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5428 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5429 ; GFX10-CU-NEXT:    s_endpgm
5511 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5512 ; GFX11-CU:       ; %bb.0: ; %entry
5513 ; GFX11-CU-NEXT:    s_clause 0x1
5514 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5515 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5516 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5517 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5518 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5519 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5520 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5521 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5522 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5523 ; GFX11-CU-NEXT:    s_endpgm
5571 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5572 ; GFX10-CU:       ; %bb.0: ; %entry
5573 ; GFX10-CU-NEXT:    s_clause 0x1
5574 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5575 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5576 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5577 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5578 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5579 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5580 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5581 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5582 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5583 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5584 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5585 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5586 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5587 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5588 ; GFX10-CU-NEXT:    s_endpgm
5670 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5671 ; GFX11-CU:       ; %bb.0: ; %entry
5672 ; GFX11-CU-NEXT:    s_clause 0x1
5673 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5674 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5675 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5676 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5677 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5678 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5679 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5680 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5681 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5682 ; GFX11-CU-NEXT:    s_endpgm
5730 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5731 ; GFX10-CU:       ; %bb.0: ; %entry
5732 ; GFX10-CU-NEXT:    s_clause 0x1
5733 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5734 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5735 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5736 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5737 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5738 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5739 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5740 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5741 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5742 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5743 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5744 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5745 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5746 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5747 ; GFX10-CU-NEXT:    s_endpgm
5829 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5830 ; GFX11-CU:       ; %bb.0: ; %entry
5831 ; GFX11-CU-NEXT:    s_clause 0x1
5832 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5833 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5834 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5835 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5836 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5837 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5838 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5839 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5840 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5841 ; GFX11-CU-NEXT:    s_endpgm
5889 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5890 ; GFX10-CU:       ; %bb.0: ; %entry
5891 ; GFX10-CU-NEXT:    s_clause 0x1
5892 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5893 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5894 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5895 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5896 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5897 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5898 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5899 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5900 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5901 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5902 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5903 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5904 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5905 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5906 ; GFX10-CU-NEXT:    s_endpgm
5988 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5989 ; GFX11-CU:       ; %bb.0: ; %entry
5990 ; GFX11-CU-NEXT:    s_clause 0x1
5991 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5992 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5993 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5994 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5995 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5996 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5997 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5998 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5999 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6000 ; GFX11-CU-NEXT:    s_endpgm
6048 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
6049 ; GFX10-CU:       ; %bb.0: ; %entry
6050 ; GFX10-CU-NEXT:    s_clause 0x1
6051 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6052 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6053 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6054 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6055 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6056 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6057 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6058 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6059 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6060 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6061 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6062 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6063 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6064 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6065 ; GFX10-CU-NEXT:    s_endpgm
6147 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
6148 ; GFX11-CU:       ; %bb.0: ; %entry
6149 ; GFX11-CU-NEXT:    s_clause 0x1
6150 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6151 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6152 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6153 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6154 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6155 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6156 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6157 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6158 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6159 ; GFX11-CU-NEXT:    s_endpgm
6207 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
6208 ; GFX10-CU:       ; %bb.0: ; %entry
6209 ; GFX10-CU-NEXT:    s_clause 0x1
6210 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6211 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6212 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6213 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6214 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6215 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6216 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6217 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6218 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6219 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6220 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6221 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6222 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6223 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6224 ; GFX10-CU-NEXT:    s_endpgm
6306 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
6307 ; GFX11-CU:       ; %bb.0: ; %entry
6308 ; GFX11-CU-NEXT:    s_clause 0x1
6309 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6310 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6311 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6312 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6313 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6314 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6315 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6316 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6317 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6318 ; GFX11-CU-NEXT:    s_endpgm
6355 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_load:
6356 ; GFX10-CU:       ; %bb.0: ; %entry
6357 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6358 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6359 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6360 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6361 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6362 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6363 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6364 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6365 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6366 ; GFX10-CU-NEXT:    s_endpgm
6445 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_load:
6446 ; GFX11-CU:       ; %bb.0: ; %entry
6447 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6448 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6449 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6450 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6451 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6452 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6453 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6454 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6455 ; GFX11-CU-NEXT:    s_endpgm
6490 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_load:
6491 ; GFX10-CU:       ; %bb.0: ; %entry
6492 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6493 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6494 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6495 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6496 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6497 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6498 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6499 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6500 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6501 ; GFX10-CU-NEXT:    s_endpgm
6580 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_load:
6581 ; GFX11-CU:       ; %bb.0: ; %entry
6582 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6583 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6584 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6585 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6586 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6587 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6588 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6589 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6590 ; GFX11-CU-NEXT:    s_endpgm
6625 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_load:
6626 ; GFX10-CU:       ; %bb.0: ; %entry
6627 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6628 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6629 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6630 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6631 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6632 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6633 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6634 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6635 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6636 ; GFX10-CU-NEXT:    s_endpgm
6715 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_load:
6716 ; GFX11-CU:       ; %bb.0: ; %entry
6717 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6718 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6719 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6720 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6721 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6722 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6723 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6724 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6725 ; GFX11-CU-NEXT:    s_endpgm
6760 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
6761 ; GFX10-CU:       ; %bb.0: ; %entry
6762 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6763 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6764 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6765 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6766 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6767 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6768 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6769 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6770 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6771 ; GFX10-CU-NEXT:    s_endpgm
6850 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
6851 ; GFX11-CU:       ; %bb.0: ; %entry
6852 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6853 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6854 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6855 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6856 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6857 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6858 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6859 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6860 ; GFX11-CU-NEXT:    s_endpgm
6892 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_store:
6893 ; GFX10-CU:       ; %bb.0: ; %entry
6894 ; GFX10-CU-NEXT:    s_clause 0x1
6895 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6896 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6897 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6898 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6899 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6900 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6901 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6902 ; GFX10-CU-NEXT:    s_endpgm
6967 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_store:
6968 ; GFX11-CU:       ; %bb.0: ; %entry
6969 ; GFX11-CU-NEXT:    s_clause 0x1
6970 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6971 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
6972 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6973 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6974 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6975 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6976 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6977 ; GFX11-CU-NEXT:    s_endpgm
7008 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_store:
7009 ; GFX10-CU:       ; %bb.0: ; %entry
7010 ; GFX10-CU-NEXT:    s_clause 0x1
7011 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7012 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7013 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7014 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7015 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7016 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7017 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7018 ; GFX10-CU-NEXT:    s_endpgm
7083 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_store:
7084 ; GFX11-CU:       ; %bb.0: ; %entry
7085 ; GFX11-CU-NEXT:    s_clause 0x1
7086 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7087 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7088 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7089 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7090 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7091 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7092 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7093 ; GFX11-CU-NEXT:    s_endpgm
7124 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_store:
7125 ; GFX10-CU:       ; %bb.0: ; %entry
7126 ; GFX10-CU-NEXT:    s_clause 0x1
7127 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7128 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7129 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7130 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7131 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7132 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7133 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7134 ; GFX10-CU-NEXT:    s_endpgm
7199 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_store:
7200 ; GFX11-CU:       ; %bb.0: ; %entry
7201 ; GFX11-CU-NEXT:    s_clause 0x1
7202 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7203 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7204 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7205 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7206 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7207 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7208 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7209 ; GFX11-CU-NEXT:    s_endpgm
7240 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
7241 ; GFX10-CU:       ; %bb.0: ; %entry
7242 ; GFX10-CU-NEXT:    s_clause 0x1
7243 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7244 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7245 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7246 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7247 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7248 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7249 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7250 ; GFX10-CU-NEXT:    s_endpgm
7315 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
7316 ; GFX11-CU:       ; %bb.0: ; %entry
7317 ; GFX11-CU-NEXT:    s_clause 0x1
7318 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7319 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7320 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7321 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7322 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7323 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7324 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7325 ; GFX11-CU-NEXT:    s_endpgm
7356 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
7357 ; GFX10-CU:       ; %bb.0: ; %entry
7358 ; GFX10-CU-NEXT:    s_clause 0x1
7359 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7360 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7361 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7362 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7363 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7364 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7365 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7366 ; GFX10-CU-NEXT:    s_endpgm
7431 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
7432 ; GFX11-CU:       ; %bb.0: ; %entry
7433 ; GFX11-CU-NEXT:    s_clause 0x1
7434 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7435 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7436 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7437 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7438 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7439 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7440 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7441 ; GFX11-CU-NEXT:    s_endpgm
7472 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
7473 ; GFX10-CU:       ; %bb.0: ; %entry
7474 ; GFX10-CU-NEXT:    s_clause 0x1
7475 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7476 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7477 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7478 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7479 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7480 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7481 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7482 ; GFX10-CU-NEXT:    s_endpgm
7547 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
7548 ; GFX11-CU:       ; %bb.0: ; %entry
7549 ; GFX11-CU-NEXT:    s_clause 0x1
7550 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7551 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7552 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7553 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7554 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7555 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7556 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7557 ; GFX11-CU-NEXT:    s_endpgm
7588 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
7589 ; GFX10-CU:       ; %bb.0: ; %entry
7590 ; GFX10-CU-NEXT:    s_clause 0x1
7591 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7592 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7593 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7594 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7595 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7596 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7597 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7598 ; GFX10-CU-NEXT:    s_endpgm
7663 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
7664 ; GFX11-CU:       ; %bb.0: ; %entry
7665 ; GFX11-CU-NEXT:    s_clause 0x1
7666 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7667 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7668 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7669 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7670 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7671 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7672 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7673 ; GFX11-CU-NEXT:    s_endpgm
7704 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7705 ; GFX10-CU:       ; %bb.0: ; %entry
7706 ; GFX10-CU-NEXT:    s_clause 0x1
7707 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7708 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7709 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7710 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7711 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7712 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7713 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7714 ; GFX10-CU-NEXT:    s_endpgm
7779 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7780 ; GFX11-CU:       ; %bb.0: ; %entry
7781 ; GFX11-CU-NEXT:    s_clause 0x1
7782 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7783 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7784 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7785 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7786 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7787 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7788 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7789 ; GFX11-CU-NEXT:    s_endpgm
7820 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7821 ; GFX10-CU:       ; %bb.0: ; %entry
7822 ; GFX10-CU-NEXT:    s_clause 0x1
7823 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7824 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7825 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7826 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7827 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7828 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7829 ; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7830 ; GFX10-CU-NEXT:    s_endpgm
7895 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7896 ; GFX11-CU:       ; %bb.0: ; %entry
7897 ; GFX11-CU-NEXT:    s_clause 0x1
7898 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7899 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7900 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7901 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7902 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7903 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7904 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7905 ; GFX11-CU-NEXT:    s_endpgm
7940 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7941 ; GFX10-CU:       ; %bb.0: ; %entry
7942 ; GFX10-CU-NEXT:    s_clause 0x1
7943 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7944 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7945 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7946 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7947 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7948 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7949 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7950 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7951 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7952 ; GFX10-CU-NEXT:    s_endpgm
8029 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
8030 ; GFX11-CU:       ; %bb.0: ; %entry
8031 ; GFX11-CU-NEXT:    s_clause 0x1
8032 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8033 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8034 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8035 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8036 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8037 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8038 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8039 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8040 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8041 ; GFX11-CU-NEXT:    s_endpgm
8077 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
8078 ; GFX10-CU:       ; %bb.0: ; %entry
8079 ; GFX10-CU-NEXT:    s_clause 0x1
8080 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8081 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8082 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8083 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8084 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8085 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8086 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8087 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8088 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8089 ; GFX10-CU-NEXT:    s_endpgm
8166 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
8167 ; GFX11-CU:       ; %bb.0: ; %entry
8168 ; GFX11-CU-NEXT:    s_clause 0x1
8169 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8170 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8171 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8172 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8173 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8174 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8175 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8176 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8177 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8178 ; GFX11-CU-NEXT:    s_endpgm
8214 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
8215 ; GFX10-CU:       ; %bb.0: ; %entry
8216 ; GFX10-CU-NEXT:    s_clause 0x1
8217 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8218 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8219 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8220 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8221 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8222 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8223 ; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8224 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8225 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8226 ; GFX10-CU-NEXT:    s_endpgm
8303 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
8304 ; GFX11-CU:       ; %bb.0: ; %entry
8305 ; GFX11-CU-NEXT:    s_clause 0x1
8306 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8307 ; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8308 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8309 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8310 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8311 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8312 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8313 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8314 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8315 ; GFX11-CU-NEXT:    s_endpgm
8353 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
8354 ; GFX10-CU:       ; %bb.0: ; %entry
8355 ; GFX10-CU-NEXT:    s_clause 0x1
8356 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8357 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8358 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8359 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8360 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8361 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8362 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8363 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8364 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8365 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8366 ; GFX10-CU-NEXT:    s_endpgm
8434 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
8435 ; GFX11-CU:       ; %bb.0: ; %entry
8436 ; GFX11-CU-NEXT:    s_clause 0x1
8437 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8438 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8439 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8440 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8441 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8442 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8443 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8444 ; GFX11-CU-NEXT:    s_endpgm
8482 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
8483 ; GFX10-CU:       ; %bb.0: ; %entry
8484 ; GFX10-CU-NEXT:    s_clause 0x1
8485 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8486 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8487 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8488 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8489 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8490 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8491 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8492 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8493 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8494 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8495 ; GFX10-CU-NEXT:    s_endpgm
8563 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
8564 ; GFX11-CU:       ; %bb.0: ; %entry
8565 ; GFX11-CU-NEXT:    s_clause 0x1
8566 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8567 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8568 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8569 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8570 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8571 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8572 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8573 ; GFX11-CU-NEXT:    s_endpgm
8611 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8612 ; GFX10-CU:       ; %bb.0: ; %entry
8613 ; GFX10-CU-NEXT:    s_clause 0x1
8614 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8615 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8616 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8617 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8618 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8619 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8620 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8621 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8622 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8623 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8624 ; GFX10-CU-NEXT:    s_endpgm
8692 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8693 ; GFX11-CU:       ; %bb.0: ; %entry
8694 ; GFX11-CU-NEXT:    s_clause 0x1
8695 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8696 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8697 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8698 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8699 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8700 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8701 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8702 ; GFX11-CU-NEXT:    s_endpgm
8740 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8741 ; GFX10-CU:       ; %bb.0: ; %entry
8742 ; GFX10-CU-NEXT:    s_clause 0x1
8743 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8744 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8745 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8746 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8747 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8748 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8749 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8750 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8751 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8752 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8753 ; GFX10-CU-NEXT:    s_endpgm
8821 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8822 ; GFX11-CU:       ; %bb.0: ; %entry
8823 ; GFX11-CU-NEXT:    s_clause 0x1
8824 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8825 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8826 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8827 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8828 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8829 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8830 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8831 ; GFX11-CU-NEXT:    s_endpgm
8869 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8870 ; GFX10-CU:       ; %bb.0: ; %entry
8871 ; GFX10-CU-NEXT:    s_clause 0x1
8872 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8873 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8874 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8875 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8876 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8877 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8878 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8879 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8880 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8881 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8882 ; GFX10-CU-NEXT:    s_endpgm
8950 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8951 ; GFX11-CU:       ; %bb.0: ; %entry
8952 ; GFX11-CU-NEXT:    s_clause 0x1
8953 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8954 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8955 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8956 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8957 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8958 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8959 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8960 ; GFX11-CU-NEXT:    s_endpgm
8998 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8999 ; GFX10-CU:       ; %bb.0: ; %entry
9000 ; GFX10-CU-NEXT:    s_clause 0x1
9001 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9002 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9003 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9004 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9005 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9006 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9007 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9008 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9009 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9010 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9011 ; GFX10-CU-NEXT:    s_endpgm
9079 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
9080 ; GFX11-CU:       ; %bb.0: ; %entry
9081 ; GFX11-CU-NEXT:    s_clause 0x1
9082 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9083 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9084 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9085 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9086 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9087 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9088 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9089 ; GFX11-CU-NEXT:    s_endpgm
9127 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
9128 ; GFX10-CU:       ; %bb.0: ; %entry
9129 ; GFX10-CU-NEXT:    s_clause 0x1
9130 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9131 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9132 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9133 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9134 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9135 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9136 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9137 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9138 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9139 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9140 ; GFX10-CU-NEXT:    s_endpgm
9208 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
9209 ; GFX11-CU:       ; %bb.0: ; %entry
9210 ; GFX11-CU-NEXT:    s_clause 0x1
9211 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9212 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9213 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9214 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9215 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9216 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9217 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9218 ; GFX11-CU-NEXT:    s_endpgm
9256 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
9257 ; GFX10-CU:       ; %bb.0: ; %entry
9258 ; GFX10-CU-NEXT:    s_clause 0x1
9259 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9260 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9261 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9262 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9263 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9264 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9265 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9266 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9267 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9268 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9269 ; GFX10-CU-NEXT:    s_endpgm
9337 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
9338 ; GFX11-CU:       ; %bb.0: ; %entry
9339 ; GFX11-CU-NEXT:    s_clause 0x1
9340 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9341 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9342 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9343 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9344 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9345 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9346 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9347 ; GFX11-CU-NEXT:    s_endpgm
9385 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
9386 ; GFX10-CU:       ; %bb.0: ; %entry
9387 ; GFX10-CU-NEXT:    s_clause 0x1
9388 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9389 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9390 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9391 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9392 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9393 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9394 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9395 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9396 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9397 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9398 ; GFX10-CU-NEXT:    s_endpgm
9466 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
9467 ; GFX11-CU:       ; %bb.0: ; %entry
9468 ; GFX11-CU-NEXT:    s_clause 0x1
9469 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9470 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9471 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9472 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9473 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9474 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9475 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9476 ; GFX11-CU-NEXT:    s_endpgm
9514 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
9515 ; GFX10-CU:       ; %bb.0: ; %entry
9516 ; GFX10-CU-NEXT:    s_clause 0x1
9517 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9518 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9519 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9520 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9521 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9522 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9523 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9524 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9525 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9526 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9527 ; GFX10-CU-NEXT:    s_endpgm
9595 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
9596 ; GFX11-CU:       ; %bb.0: ; %entry
9597 ; GFX11-CU-NEXT:    s_clause 0x1
9598 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9599 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9600 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9601 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9602 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9603 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9604 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9605 ; GFX11-CU-NEXT:    s_endpgm
9643 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
9644 ; GFX10-CU:       ; %bb.0: ; %entry
9645 ; GFX10-CU-NEXT:    s_clause 0x1
9646 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9647 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9648 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9649 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9650 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9651 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9652 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9653 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9654 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9655 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9656 ; GFX10-CU-NEXT:    s_endpgm
9724 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
9725 ; GFX11-CU:       ; %bb.0: ; %entry
9726 ; GFX11-CU-NEXT:    s_clause 0x1
9727 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9728 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9729 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9730 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9731 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9732 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9733 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9734 ; GFX11-CU-NEXT:    s_endpgm
9772 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9773 ; GFX10-CU:       ; %bb.0: ; %entry
9774 ; GFX10-CU-NEXT:    s_clause 0x1
9775 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9776 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9777 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9778 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9779 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9780 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9781 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9782 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9783 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9784 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9785 ; GFX10-CU-NEXT:    s_endpgm
9853 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9854 ; GFX11-CU:       ; %bb.0: ; %entry
9855 ; GFX11-CU-NEXT:    s_clause 0x1
9856 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9857 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9858 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9859 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9860 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9861 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9862 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9863 ; GFX11-CU-NEXT:    s_endpgm
9901 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9902 ; GFX10-CU:       ; %bb.0: ; %entry
9903 ; GFX10-CU-NEXT:    s_clause 0x1
9904 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9905 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9906 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9907 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9908 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9909 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9910 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9911 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9912 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9913 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9914 ; GFX10-CU-NEXT:    s_endpgm
9982 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9983 ; GFX11-CU:       ; %bb.0: ; %entry
9984 ; GFX11-CU-NEXT:    s_clause 0x1
9985 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9986 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9987 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9988 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9989 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9990 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9991 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9992 ; GFX11-CU-NEXT:    s_endpgm
10030 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
10031 ; GFX10-CU:       ; %bb.0: ; %entry
10032 ; GFX10-CU-NEXT:    s_clause 0x1
10033 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10034 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10035 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10036 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10037 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10038 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10039 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10040 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10041 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10042 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10043 ; GFX10-CU-NEXT:    s_endpgm
10111 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
10112 ; GFX11-CU:       ; %bb.0: ; %entry
10113 ; GFX11-CU-NEXT:    s_clause 0x1
10114 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10115 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10116 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10117 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10118 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10119 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10120 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10121 ; GFX11-CU-NEXT:    s_endpgm
10159 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
10160 ; GFX10-CU:       ; %bb.0: ; %entry
10161 ; GFX10-CU-NEXT:    s_clause 0x1
10162 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10163 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10164 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10165 ; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10166 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10167 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10168 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10169 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10170 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10171 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10172 ; GFX10-CU-NEXT:    s_endpgm
10240 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
10241 ; GFX11-CU:       ; %bb.0: ; %entry
10242 ; GFX11-CU-NEXT:    s_clause 0x1
10243 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10244 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10245 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10246 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10247 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10248 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10249 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10250 ; GFX11-CU-NEXT:    s_endpgm
10296 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
10297 ; GFX10-CU:       ; %bb.0: ; %entry
10298 ; GFX10-CU-NEXT:    s_clause 0x1
10299 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10300 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10301 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10302 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10303 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10304 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10305 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10306 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10307 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10308 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10309 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10310 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10311 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10312 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10313 ; GFX10-CU-NEXT:    s_endpgm
10395 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
10396 ; GFX11-CU:       ; %bb.0: ; %entry
10397 ; GFX11-CU-NEXT:    s_clause 0x1
10398 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10399 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10400 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10401 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10402 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10403 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10404 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10405 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10406 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10407 ; GFX11-CU-NEXT:    s_endpgm
10455 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
10456 ; GFX10-CU:       ; %bb.0: ; %entry
10457 ; GFX10-CU-NEXT:    s_clause 0x1
10458 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10459 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10460 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10461 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10462 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10463 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10464 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10465 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10466 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10467 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10469 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10470 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10471 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10472 ; GFX10-CU-NEXT:    s_endpgm
10554 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
10555 ; GFX11-CU:       ; %bb.0: ; %entry
10556 ; GFX11-CU-NEXT:    s_clause 0x1
10557 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10558 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10559 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10560 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10561 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10562 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10563 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10564 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10565 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10566 ; GFX11-CU-NEXT:    s_endpgm
10614 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
10615 ; GFX10-CU:       ; %bb.0: ; %entry
10616 ; GFX10-CU-NEXT:    s_clause 0x1
10617 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10618 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10619 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10620 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10621 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10622 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10623 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10624 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10625 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10626 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10627 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10628 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10629 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10630 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10631 ; GFX10-CU-NEXT:    s_endpgm
10713 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
10714 ; GFX11-CU:       ; %bb.0: ; %entry
10715 ; GFX11-CU-NEXT:    s_clause 0x1
10716 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10717 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10718 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10719 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10720 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10721 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10722 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10723 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10724 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10725 ; GFX11-CU-NEXT:    s_endpgm
10773 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
10774 ; GFX10-CU:       ; %bb.0: ; %entry
10775 ; GFX10-CU-NEXT:    s_clause 0x1
10776 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10777 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10778 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10779 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10780 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10781 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10782 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10783 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10784 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10785 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10786 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10787 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10788 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10789 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10790 ; GFX10-CU-NEXT:    s_endpgm
10872 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
10873 ; GFX11-CU:       ; %bb.0: ; %entry
10874 ; GFX11-CU-NEXT:    s_clause 0x1
10875 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10876 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10877 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10878 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10879 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10880 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10881 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10882 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10883 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10884 ; GFX11-CU-NEXT:    s_endpgm
10932 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10933 ; GFX10-CU:       ; %bb.0: ; %entry
10934 ; GFX10-CU-NEXT:    s_clause 0x1
10935 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10936 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10937 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10938 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10939 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10940 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10941 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10942 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10943 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10944 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10945 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10946 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10947 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10948 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10949 ; GFX10-CU-NEXT:    s_endpgm
11031 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
11032 ; GFX11-CU:       ; %bb.0: ; %entry
11033 ; GFX11-CU-NEXT:    s_clause 0x1
11034 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11035 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11036 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11037 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11038 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11039 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11040 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11041 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11042 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11043 ; GFX11-CU-NEXT:    s_endpgm
11091 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
11092 ; GFX10-CU:       ; %bb.0: ; %entry
11093 ; GFX10-CU-NEXT:    s_clause 0x1
11094 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11095 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11096 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11097 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11098 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11099 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11100 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11101 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11102 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11103 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11104 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11105 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11106 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11107 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11108 ; GFX10-CU-NEXT:    s_endpgm
11190 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
11191 ; GFX11-CU:       ; %bb.0: ; %entry
11192 ; GFX11-CU-NEXT:    s_clause 0x1
11193 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11194 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11195 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11196 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11197 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11198 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11199 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11200 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11201 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11202 ; GFX11-CU-NEXT:    s_endpgm
11250 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
11251 ; GFX10-CU:       ; %bb.0: ; %entry
11252 ; GFX10-CU-NEXT:    s_clause 0x1
11253 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11254 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11255 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11256 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11257 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11258 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11259 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11260 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11261 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11262 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11263 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11264 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11265 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11266 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11267 ; GFX10-CU-NEXT:    s_endpgm
11349 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
11350 ; GFX11-CU:       ; %bb.0: ; %entry
11351 ; GFX11-CU-NEXT:    s_clause 0x1
11352 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11353 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11354 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11355 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11356 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11357 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11358 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11359 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11360 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11361 ; GFX11-CU-NEXT:    s_endpgm
11409 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
11410 ; GFX10-CU:       ; %bb.0: ; %entry
11411 ; GFX10-CU-NEXT:    s_clause 0x1
11412 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11413 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11414 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11415 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11416 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11417 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11418 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11419 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11420 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11421 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11422 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11423 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11424 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11425 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11426 ; GFX10-CU-NEXT:    s_endpgm
11508 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
11509 ; GFX11-CU:       ; %bb.0: ; %entry
11510 ; GFX11-CU-NEXT:    s_clause 0x1
11511 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11512 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11513 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11514 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11515 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11516 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11517 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11518 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11519 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11520 ; GFX11-CU-NEXT:    s_endpgm
11568 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
11569 ; GFX10-CU:       ; %bb.0: ; %entry
11570 ; GFX10-CU-NEXT:    s_clause 0x1
11571 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11572 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11573 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11574 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11575 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11576 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11577 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11578 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11579 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11580 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11581 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11582 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11583 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11584 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11585 ; GFX10-CU-NEXT:    s_endpgm
11667 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
11668 ; GFX11-CU:       ; %bb.0: ; %entry
11669 ; GFX11-CU-NEXT:    s_clause 0x1
11670 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11671 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11672 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11673 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11674 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11675 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11676 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11677 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11678 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11679 ; GFX11-CU-NEXT:    s_endpgm
11727 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
11728 ; GFX10-CU:       ; %bb.0: ; %entry
11729 ; GFX10-CU-NEXT:    s_clause 0x1
11730 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11731 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11732 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11733 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11734 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11735 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11736 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11737 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11738 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11739 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11740 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11741 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11742 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11743 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11744 ; GFX10-CU-NEXT:    s_endpgm
11826 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
11827 ; GFX11-CU:       ; %bb.0: ; %entry
11828 ; GFX11-CU-NEXT:    s_clause 0x1
11829 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11830 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11831 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11832 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11833 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11834 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11835 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11836 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11837 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11838 ; GFX11-CU-NEXT:    s_endpgm
11886 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
11887 ; GFX10-CU:       ; %bb.0: ; %entry
11888 ; GFX10-CU-NEXT:    s_clause 0x1
11889 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11890 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11891 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11892 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11893 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11894 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11895 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11896 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11897 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11898 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11899 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11900 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11901 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11902 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11903 ; GFX10-CU-NEXT:    s_endpgm
11985 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
11986 ; GFX11-CU:       ; %bb.0: ; %entry
11987 ; GFX11-CU-NEXT:    s_clause 0x1
11988 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11989 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11990 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11991 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11992 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11993 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11994 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11995 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11996 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11997 ; GFX11-CU-NEXT:    s_endpgm
12045 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
12046 ; GFX10-CU:       ; %bb.0: ; %entry
12047 ; GFX10-CU-NEXT:    s_clause 0x1
12048 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12049 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12050 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12051 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12052 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12053 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12054 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12055 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12056 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12057 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12058 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12059 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12060 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12061 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12062 ; GFX10-CU-NEXT:    s_endpgm
12144 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
12145 ; GFX11-CU:       ; %bb.0: ; %entry
12146 ; GFX11-CU-NEXT:    s_clause 0x1
12147 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12148 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12149 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12150 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12151 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12152 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12153 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12154 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12155 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12156 ; GFX11-CU-NEXT:    s_endpgm
12204 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
12205 ; GFX10-CU:       ; %bb.0: ; %entry
12206 ; GFX10-CU-NEXT:    s_clause 0x1
12207 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12208 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12209 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12210 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12211 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12212 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12213 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12214 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12215 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12216 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12217 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12218 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12219 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12220 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12221 ; GFX10-CU-NEXT:    s_endpgm
12303 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
12304 ; GFX11-CU:       ; %bb.0: ; %entry
12305 ; GFX11-CU-NEXT:    s_clause 0x1
12306 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12307 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12308 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12309 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12310 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12311 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12312 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12313 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12314 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12315 ; GFX11-CU-NEXT:    s_endpgm
12363 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
12364 ; GFX10-CU:       ; %bb.0: ; %entry
12365 ; GFX10-CU-NEXT:    s_clause 0x1
12366 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12367 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12368 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12369 ; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12370 ; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12371 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12372 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12373 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12374 ; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12375 ; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12376 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12377 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12378 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12379 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12380 ; GFX10-CU-NEXT:    s_endpgm
12462 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
12463 ; GFX11-CU:       ; %bb.0: ; %entry
12464 ; GFX11-CU-NEXT:    s_clause 0x1
12465 ; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12466 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12467 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12468 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12469 ; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12470 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12471 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12472 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12473 ; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12474 ; GFX11-CU-NEXT:    s_endpgm