Lines Matching refs:CU

5 …sa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
12 …sa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
51 ; GFX10-CU-LABEL: local_singlethread_unordered_load:
52 ; GFX10-CU:       ; %bb.0: ; %entry
53 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
54 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
55 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
56 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
57 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
58 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
59 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
60 ; GFX10-CU-NEXT:    s_endpgm
128 ; GFX11-CU-LABEL: local_singlethread_unordered_load:
129 ; GFX11-CU:       ; %bb.0: ; %entry
130 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
131 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
132 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
133 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
134 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
136 ; GFX11-CU-NEXT:    s_endpgm
181 ; GFX10-CU-LABEL: local_singlethread_monotonic_load:
182 ; GFX10-CU:       ; %bb.0: ; %entry
183 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
184 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
185 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
186 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
187 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
188 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
189 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
190 ; GFX10-CU-NEXT:    s_endpgm
258 ; GFX11-CU-LABEL: local_singlethread_monotonic_load:
259 ; GFX11-CU:       ; %bb.0: ; %entry
260 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
261 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
262 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
263 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
264 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
265 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
266 ; GFX11-CU-NEXT:    s_endpgm
311 ; GFX10-CU-LABEL: local_singlethread_acquire_load:
312 ; GFX10-CU:       ; %bb.0: ; %entry
313 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
314 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
315 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
316 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
317 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
318 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
319 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
320 ; GFX10-CU-NEXT:    s_endpgm
388 ; GFX11-CU-LABEL: local_singlethread_acquire_load:
389 ; GFX11-CU:       ; %bb.0: ; %entry
390 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
391 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
392 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
393 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
394 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
395 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
396 ; GFX11-CU-NEXT:    s_endpgm
441 ; GFX10-CU-LABEL: local_singlethread_seq_cst_load:
442 ; GFX10-CU:       ; %bb.0: ; %entry
443 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
444 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
445 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
446 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
447 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
448 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
449 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
450 ; GFX10-CU-NEXT:    s_endpgm
518 ; GFX11-CU-LABEL: local_singlethread_seq_cst_load:
519 ; GFX11-CU:       ; %bb.0: ; %entry
520 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
521 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
522 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
523 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
524 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
525 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
526 ; GFX11-CU-NEXT:    s_endpgm
565 ; GFX10-CU-LABEL: local_singlethread_unordered_store:
566 ; GFX10-CU:       ; %bb.0: ; %entry
567 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
568 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
569 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
570 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
571 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
572 ; GFX10-CU-NEXT:    s_endpgm
628 ; GFX11-CU-LABEL: local_singlethread_unordered_store:
629 ; GFX11-CU:       ; %bb.0: ; %entry
630 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
631 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
632 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
633 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
634 ; GFX11-CU-NEXT:    s_endpgm
672 ; GFX10-CU-LABEL: local_singlethread_monotonic_store:
673 ; GFX10-CU:       ; %bb.0: ; %entry
674 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
675 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
676 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
677 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
678 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
679 ; GFX10-CU-NEXT:    s_endpgm
735 ; GFX11-CU-LABEL: local_singlethread_monotonic_store:
736 ; GFX11-CU:       ; %bb.0: ; %entry
737 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
738 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
739 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
740 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
741 ; GFX11-CU-NEXT:    s_endpgm
779 ; GFX10-CU-LABEL: local_singlethread_release_store:
780 ; GFX10-CU:       ; %bb.0: ; %entry
781 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
782 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
783 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
784 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
785 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
786 ; GFX10-CU-NEXT:    s_endpgm
842 ; GFX11-CU-LABEL: local_singlethread_release_store:
843 ; GFX11-CU:       ; %bb.0: ; %entry
844 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
845 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
846 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
847 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
848 ; GFX11-CU-NEXT:    s_endpgm
886 ; GFX10-CU-LABEL: local_singlethread_seq_cst_store:
887 ; GFX10-CU:       ; %bb.0: ; %entry
888 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
889 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
890 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
891 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
892 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
893 ; GFX10-CU-NEXT:    s_endpgm
949 ; GFX11-CU-LABEL: local_singlethread_seq_cst_store:
950 ; GFX11-CU:       ; %bb.0: ; %entry
951 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
952 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
953 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
954 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
955 ; GFX11-CU-NEXT:    s_endpgm
993 ; GFX10-CU-LABEL: local_singlethread_monotonic_atomicrmw:
994 ; GFX10-CU:       ; %bb.0: ; %entry
995 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
996 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
997 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
998 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
999 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1000 ; GFX10-CU-NEXT:    s_endpgm
1056 ; GFX11-CU-LABEL: local_singlethread_monotonic_atomicrmw:
1057 ; GFX11-CU:       ; %bb.0: ; %entry
1058 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1059 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1060 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1061 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1062 ; GFX11-CU-NEXT:    s_endpgm
1100 ; GFX10-CU-LABEL: local_singlethread_acquire_atomicrmw:
1101 ; GFX10-CU:       ; %bb.0: ; %entry
1102 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1103 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1104 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1105 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1106 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1107 ; GFX10-CU-NEXT:    s_endpgm
1163 ; GFX11-CU-LABEL: local_singlethread_acquire_atomicrmw:
1164 ; GFX11-CU:       ; %bb.0: ; %entry
1165 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1166 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1167 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1168 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1169 ; GFX11-CU-NEXT:    s_endpgm
1207 ; GFX10-CU-LABEL: local_singlethread_release_atomicrmw:
1208 ; GFX10-CU:       ; %bb.0: ; %entry
1209 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1210 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1211 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1212 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1213 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1214 ; GFX10-CU-NEXT:    s_endpgm
1270 ; GFX11-CU-LABEL: local_singlethread_release_atomicrmw:
1271 ; GFX11-CU:       ; %bb.0: ; %entry
1272 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1273 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1274 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1275 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1276 ; GFX11-CU-NEXT:    s_endpgm
1314 ; GFX10-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1315 ; GFX10-CU:       ; %bb.0: ; %entry
1316 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1317 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1318 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1319 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1320 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1321 ; GFX10-CU-NEXT:    s_endpgm
1377 ; GFX11-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1378 ; GFX11-CU:       ; %bb.0: ; %entry
1379 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1380 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1381 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1382 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1383 ; GFX11-CU-NEXT:    s_endpgm
1421 ; GFX10-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1422 ; GFX10-CU:       ; %bb.0: ; %entry
1423 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1424 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1425 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1426 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1427 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1428 ; GFX10-CU-NEXT:    s_endpgm
1484 ; GFX11-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1485 ; GFX11-CU:       ; %bb.0: ; %entry
1486 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1487 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1488 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1489 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1490 ; GFX11-CU-NEXT:    s_endpgm
1534 ; GFX10-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1535 ; GFX10-CU:       ; %bb.0: ; %entry
1536 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1537 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1538 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1539 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1540 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1541 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1542 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1543 ; GFX10-CU-NEXT:    s_endpgm
1611 ; GFX11-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1612 ; GFX11-CU:       ; %bb.0: ; %entry
1613 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1614 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1615 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1616 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1617 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1618 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1619 ; GFX11-CU-NEXT:    s_endpgm
1664 ; GFX10-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1665 ; GFX10-CU:       ; %bb.0: ; %entry
1666 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1667 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1668 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1669 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1670 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1671 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1672 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1673 ; GFX10-CU-NEXT:    s_endpgm
1741 ; GFX11-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1742 ; GFX11-CU:       ; %bb.0: ; %entry
1743 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1744 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1745 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1746 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1747 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1748 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1749 ; GFX11-CU-NEXT:    s_endpgm
1794 ; GFX10-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1795 ; GFX10-CU:       ; %bb.0: ; %entry
1796 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1797 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1798 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1799 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1800 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1801 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1802 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1803 ; GFX10-CU-NEXT:    s_endpgm
1871 ; GFX11-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1872 ; GFX11-CU:       ; %bb.0: ; %entry
1873 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1874 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1875 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1876 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1877 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1878 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1879 ; GFX11-CU-NEXT:    s_endpgm
1921 ; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1922 ; GFX10-CU:       ; %bb.0: ; %entry
1923 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1924 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1925 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1926 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1927 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1928 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1929 ; GFX10-CU-NEXT:    s_endpgm
1991 ; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1992 ; GFX11-CU:       ; %bb.0: ; %entry
1993 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
1994 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1995 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1996 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
1997 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
1998 ; GFX11-CU-NEXT:    s_endpgm
2040 ; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2041 ; GFX10-CU:       ; %bb.0: ; %entry
2042 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2043 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2044 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2045 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2046 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2047 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2048 ; GFX10-CU-NEXT:    s_endpgm
2110 ; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2111 ; GFX11-CU:       ; %bb.0: ; %entry
2112 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2113 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2114 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2115 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2116 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2117 ; GFX11-CU-NEXT:    s_endpgm
2159 ; GFX10-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2160 ; GFX10-CU:       ; %bb.0: ; %entry
2161 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2162 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2163 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2164 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2165 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2166 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2167 ; GFX10-CU-NEXT:    s_endpgm
2229 ; GFX11-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2230 ; GFX11-CU:       ; %bb.0: ; %entry
2231 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2232 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2233 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2234 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2235 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2236 ; GFX11-CU-NEXT:    s_endpgm
2278 ; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2279 ; GFX10-CU:       ; %bb.0: ; %entry
2280 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2281 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2282 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2283 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2284 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2285 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2286 ; GFX10-CU-NEXT:    s_endpgm
2348 ; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2349 ; GFX11-CU:       ; %bb.0: ; %entry
2350 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2351 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2352 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2353 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2354 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2355 ; GFX11-CU-NEXT:    s_endpgm
2397 ; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2398 ; GFX10-CU:       ; %bb.0: ; %entry
2399 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2400 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2401 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2402 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2403 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2404 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2405 ; GFX10-CU-NEXT:    s_endpgm
2467 ; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2468 ; GFX11-CU:       ; %bb.0: ; %entry
2469 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2470 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2471 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2472 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2473 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2474 ; GFX11-CU-NEXT:    s_endpgm
2516 ; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2517 ; GFX10-CU:       ; %bb.0: ; %entry
2518 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2519 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2520 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2521 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2522 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2523 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2524 ; GFX10-CU-NEXT:    s_endpgm
2586 ; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2587 ; GFX11-CU:       ; %bb.0: ; %entry
2588 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2589 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2590 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2591 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2592 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2593 ; GFX11-CU-NEXT:    s_endpgm
2635 ; GFX10-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2636 ; GFX10-CU:       ; %bb.0: ; %entry
2637 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2638 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2639 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2640 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2641 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2642 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2643 ; GFX10-CU-NEXT:    s_endpgm
2705 ; GFX11-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2706 ; GFX11-CU:       ; %bb.0: ; %entry
2707 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2708 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2709 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2710 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2711 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2712 ; GFX11-CU-NEXT:    s_endpgm
2754 ; GFX10-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2755 ; GFX10-CU:       ; %bb.0: ; %entry
2756 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2757 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2758 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2759 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2760 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2761 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2762 ; GFX10-CU-NEXT:    s_endpgm
2824 ; GFX11-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2825 ; GFX11-CU:       ; %bb.0: ; %entry
2826 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2827 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2828 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2829 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2830 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2831 ; GFX11-CU-NEXT:    s_endpgm
2873 ; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2874 ; GFX10-CU:       ; %bb.0: ; %entry
2875 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2876 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2877 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2878 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2879 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2880 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2881 ; GFX10-CU-NEXT:    s_endpgm
2943 ; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2944 ; GFX11-CU:       ; %bb.0: ; %entry
2945 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2946 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2947 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2948 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2949 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2950 ; GFX11-CU-NEXT:    s_endpgm
2992 ; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2993 ; GFX10-CU:       ; %bb.0: ; %entry
2994 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2995 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2996 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2997 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2998 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2999 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3000 ; GFX10-CU-NEXT:    s_endpgm
3062 ; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3063 ; GFX11-CU:       ; %bb.0: ; %entry
3064 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3065 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3066 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3067 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3068 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3069 ; GFX11-CU-NEXT:    s_endpgm
3111 ; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3112 ; GFX10-CU:       ; %bb.0: ; %entry
3113 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3114 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3115 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3116 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3117 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3118 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3119 ; GFX10-CU-NEXT:    s_endpgm
3181 ; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3182 ; GFX11-CU:       ; %bb.0: ; %entry
3183 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3184 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3185 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3186 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3187 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3188 ; GFX11-CU-NEXT:    s_endpgm
3230 ; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3231 ; GFX10-CU:       ; %bb.0: ; %entry
3232 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3233 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3234 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3235 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3236 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3237 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3238 ; GFX10-CU-NEXT:    s_endpgm
3300 ; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3301 ; GFX11-CU:       ; %bb.0: ; %entry
3302 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3303 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3304 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3305 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3306 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3307 ; GFX11-CU-NEXT:    s_endpgm
3349 ; GFX10-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3350 ; GFX10-CU:       ; %bb.0: ; %entry
3351 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3352 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3353 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3354 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3355 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3356 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3357 ; GFX10-CU-NEXT:    s_endpgm
3419 ; GFX11-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3420 ; GFX11-CU:       ; %bb.0: ; %entry
3421 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3422 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3423 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3424 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3425 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3426 ; GFX11-CU-NEXT:    s_endpgm
3468 ; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3469 ; GFX10-CU:       ; %bb.0: ; %entry
3470 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3471 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3472 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3473 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3474 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3475 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3476 ; GFX10-CU-NEXT:    s_endpgm
3538 ; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3539 ; GFX11-CU:       ; %bb.0: ; %entry
3540 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3541 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3542 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3543 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3544 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3545 ; GFX11-CU-NEXT:    s_endpgm
3587 ; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3588 ; GFX10-CU:       ; %bb.0: ; %entry
3589 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3590 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3591 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3592 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3593 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3594 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3595 ; GFX10-CU-NEXT:    s_endpgm
3657 ; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3658 ; GFX11-CU:       ; %bb.0: ; %entry
3659 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3660 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3661 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3662 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3663 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3664 ; GFX11-CU-NEXT:    s_endpgm
3712 ; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3713 ; GFX10-CU:       ; %bb.0: ; %entry
3714 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3715 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3716 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3717 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3718 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3719 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3720 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3721 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3722 ; GFX10-CU-NEXT:    s_endpgm
3796 ; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3797 ; GFX11-CU:       ; %bb.0: ; %entry
3798 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3799 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3800 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3801 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3802 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3803 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3804 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
3805 ; GFX11-CU-NEXT:    s_endpgm
3855 ; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3856 ; GFX10-CU:       ; %bb.0: ; %entry
3857 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3858 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3859 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3860 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3861 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3862 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3863 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3864 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3865 ; GFX10-CU-NEXT:    s_endpgm
3939 ; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3940 ; GFX11-CU:       ; %bb.0: ; %entry
3941 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3942 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3943 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3944 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3945 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3946 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3947 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
3948 ; GFX11-CU-NEXT:    s_endpgm
3998 ; GFX10-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
3999 ; GFX10-CU:       ; %bb.0: ; %entry
4000 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4001 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4002 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4003 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4004 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4005 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4006 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4007 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4008 ; GFX10-CU-NEXT:    s_endpgm
4082 ; GFX11-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4083 ; GFX11-CU:       ; %bb.0: ; %entry
4084 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4085 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4086 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4087 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4088 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4089 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4090 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4091 ; GFX11-CU-NEXT:    s_endpgm
4141 ; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4142 ; GFX10-CU:       ; %bb.0: ; %entry
4143 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4144 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4145 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4146 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4147 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4148 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4149 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4150 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4151 ; GFX10-CU-NEXT:    s_endpgm
4225 ; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4226 ; GFX11-CU:       ; %bb.0: ; %entry
4227 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4228 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4229 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4230 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4231 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4232 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4233 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4234 ; GFX11-CU-NEXT:    s_endpgm
4284 ; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4285 ; GFX10-CU:       ; %bb.0: ; %entry
4286 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4287 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4288 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4289 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4290 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4291 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4292 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4293 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4294 ; GFX10-CU-NEXT:    s_endpgm
4368 ; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4369 ; GFX11-CU:       ; %bb.0: ; %entry
4370 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4371 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4372 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4373 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4374 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4375 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4376 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4377 ; GFX11-CU-NEXT:    s_endpgm
4427 ; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4428 ; GFX10-CU:       ; %bb.0: ; %entry
4429 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4430 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4431 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4432 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4433 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4434 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4435 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4436 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4437 ; GFX10-CU-NEXT:    s_endpgm
4511 ; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4512 ; GFX11-CU:       ; %bb.0: ; %entry
4513 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4514 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4515 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4516 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4517 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4518 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4519 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4520 ; GFX11-CU-NEXT:    s_endpgm
4570 ; GFX10-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4571 ; GFX10-CU:       ; %bb.0: ; %entry
4572 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4573 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4574 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4575 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4576 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4577 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4578 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4579 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4580 ; GFX10-CU-NEXT:    s_endpgm
4654 ; GFX11-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4655 ; GFX11-CU:       ; %bb.0: ; %entry
4656 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4657 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4658 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4659 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4660 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4661 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4662 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4663 ; GFX11-CU-NEXT:    s_endpgm
4713 ; GFX10-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4714 ; GFX10-CU:       ; %bb.0: ; %entry
4715 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4716 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4717 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4718 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4719 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4720 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4721 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4722 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4723 ; GFX10-CU-NEXT:    s_endpgm
4797 ; GFX11-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4798 ; GFX11-CU:       ; %bb.0: ; %entry
4799 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4800 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4801 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4802 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4803 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4804 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4805 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4806 ; GFX11-CU-NEXT:    s_endpgm
4856 ; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4857 ; GFX10-CU:       ; %bb.0: ; %entry
4858 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4859 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4860 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4861 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4862 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4863 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4864 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4865 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4866 ; GFX10-CU-NEXT:    s_endpgm
4940 ; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4941 ; GFX11-CU:       ; %bb.0: ; %entry
4942 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4943 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4944 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4945 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4946 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4947 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4948 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4949 ; GFX11-CU-NEXT:    s_endpgm
4999 ; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5000 ; GFX10-CU:       ; %bb.0: ; %entry
5001 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5002 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5003 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5004 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5005 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5006 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5007 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5008 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5009 ; GFX10-CU-NEXT:    s_endpgm
5083 ; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5084 ; GFX11-CU:       ; %bb.0: ; %entry
5085 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5086 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5087 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5088 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5089 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5090 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5091 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5092 ; GFX11-CU-NEXT:    s_endpgm
5142 ; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5143 ; GFX10-CU:       ; %bb.0: ; %entry
5144 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5145 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5146 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5147 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5148 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5149 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5150 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5151 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5152 ; GFX10-CU-NEXT:    s_endpgm
5226 ; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5227 ; GFX11-CU:       ; %bb.0: ; %entry
5228 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5229 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5230 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5231 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5232 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5233 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5234 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5235 ; GFX11-CU-NEXT:    s_endpgm
5285 ; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5286 ; GFX10-CU:       ; %bb.0: ; %entry
5287 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5288 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5289 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5290 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5291 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5292 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5293 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5294 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5295 ; GFX10-CU-NEXT:    s_endpgm
5369 ; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5370 ; GFX11-CU:       ; %bb.0: ; %entry
5371 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5372 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5373 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5374 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5375 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5376 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5377 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5378 ; GFX11-CU-NEXT:    s_endpgm
5428 ; GFX10-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5429 ; GFX10-CU:       ; %bb.0: ; %entry
5430 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5431 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5432 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5433 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5434 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5435 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5436 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5437 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5438 ; GFX10-CU-NEXT:    s_endpgm
5512 ; GFX11-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5513 ; GFX11-CU:       ; %bb.0: ; %entry
5514 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5515 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5516 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5517 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5518 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5519 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5520 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5521 ; GFX11-CU-NEXT:    s_endpgm
5571 ; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5572 ; GFX10-CU:       ; %bb.0: ; %entry
5573 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5574 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5575 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5576 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5577 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5578 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5579 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5580 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5581 ; GFX10-CU-NEXT:    s_endpgm
5655 ; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5656 ; GFX11-CU:       ; %bb.0: ; %entry
5657 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5658 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5659 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5660 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5661 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5662 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5663 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5664 ; GFX11-CU-NEXT:    s_endpgm
5714 ; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5715 ; GFX10-CU:       ; %bb.0: ; %entry
5716 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5717 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5718 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5719 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5720 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5721 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5722 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5723 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5724 ; GFX10-CU-NEXT:    s_endpgm
5798 ; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5799 ; GFX11-CU:       ; %bb.0: ; %entry
5800 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5801 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5802 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5803 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5804 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5805 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5806 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5807 ; GFX11-CU-NEXT:    s_endpgm
5854 ; GFX10-CU-LABEL: local_singlethread_one_as_unordered_load:
5855 ; GFX10-CU:       ; %bb.0: ; %entry
5856 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5857 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5858 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5859 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5860 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
5861 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5862 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
5863 ; GFX10-CU-NEXT:    s_endpgm
5931 ; GFX11-CU-LABEL: local_singlethread_one_as_unordered_load:
5932 ; GFX11-CU:       ; %bb.0: ; %entry
5933 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5934 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5935 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5936 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
5937 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5938 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
5939 ; GFX11-CU-NEXT:    s_endpgm
5984 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_load:
5985 ; GFX10-CU:       ; %bb.0: ; %entry
5986 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5987 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5988 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5989 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5990 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
5991 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5992 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
5993 ; GFX10-CU-NEXT:    s_endpgm
6061 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_load:
6062 ; GFX11-CU:       ; %bb.0: ; %entry
6063 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6064 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6065 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6066 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6067 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6068 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6069 ; GFX11-CU-NEXT:    s_endpgm
6114 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_load:
6115 ; GFX10-CU:       ; %bb.0: ; %entry
6116 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6117 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6118 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6119 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6120 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6121 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6122 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6123 ; GFX10-CU-NEXT:    s_endpgm
6191 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_load:
6192 ; GFX11-CU:       ; %bb.0: ; %entry
6193 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6194 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6195 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6196 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6197 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6198 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6199 ; GFX11-CU-NEXT:    s_endpgm
6244 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6245 ; GFX10-CU:       ; %bb.0: ; %entry
6246 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6247 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6248 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6249 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6250 ; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6251 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6252 ; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6253 ; GFX10-CU-NEXT:    s_endpgm
6321 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6322 ; GFX11-CU:       ; %bb.0: ; %entry
6323 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6324 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6325 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6326 ; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6327 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6328 ; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6329 ; GFX11-CU-NEXT:    s_endpgm
6368 ; GFX10-CU-LABEL: local_singlethread_one_as_unordered_store:
6369 ; GFX10-CU:       ; %bb.0: ; %entry
6370 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6371 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6372 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6373 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6374 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6375 ; GFX10-CU-NEXT:    s_endpgm
6431 ; GFX11-CU-LABEL: local_singlethread_one_as_unordered_store:
6432 ; GFX11-CU:       ; %bb.0: ; %entry
6433 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6434 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6435 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6436 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6437 ; GFX11-CU-NEXT:    s_endpgm
6475 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_store:
6476 ; GFX10-CU:       ; %bb.0: ; %entry
6477 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6478 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6479 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6480 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6481 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6482 ; GFX10-CU-NEXT:    s_endpgm
6538 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_store:
6539 ; GFX11-CU:       ; %bb.0: ; %entry
6540 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6541 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6542 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6543 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6544 ; GFX11-CU-NEXT:    s_endpgm
6582 ; GFX10-CU-LABEL: local_singlethread_one_as_release_store:
6583 ; GFX10-CU:       ; %bb.0: ; %entry
6584 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6585 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6586 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6587 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6588 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6589 ; GFX10-CU-NEXT:    s_endpgm
6645 ; GFX11-CU-LABEL: local_singlethread_one_as_release_store:
6646 ; GFX11-CU:       ; %bb.0: ; %entry
6647 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6648 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6649 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6650 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6651 ; GFX11-CU-NEXT:    s_endpgm
6689 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6690 ; GFX10-CU:       ; %bb.0: ; %entry
6691 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6692 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6693 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6694 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6695 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6696 ; GFX10-CU-NEXT:    s_endpgm
6752 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6753 ; GFX11-CU:       ; %bb.0: ; %entry
6754 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6755 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6756 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6757 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6758 ; GFX11-CU-NEXT:    s_endpgm
6796 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6797 ; GFX10-CU:       ; %bb.0: ; %entry
6798 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6799 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6800 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6801 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6802 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6803 ; GFX10-CU-NEXT:    s_endpgm
6859 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6860 ; GFX11-CU:       ; %bb.0: ; %entry
6861 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6862 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6863 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6864 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6865 ; GFX11-CU-NEXT:    s_endpgm
6903 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6904 ; GFX10-CU:       ; %bb.0: ; %entry
6905 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6906 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6908 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6909 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6910 ; GFX10-CU-NEXT:    s_endpgm
6966 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6967 ; GFX11-CU:       ; %bb.0: ; %entry
6968 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6969 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6970 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6971 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6972 ; GFX11-CU-NEXT:    s_endpgm
7010 ; GFX10-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7011 ; GFX10-CU:       ; %bb.0: ; %entry
7012 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7013 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7014 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7015 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7016 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7017 ; GFX10-CU-NEXT:    s_endpgm
7073 ; GFX11-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7074 ; GFX11-CU:       ; %bb.0: ; %entry
7075 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7076 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7077 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7078 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7079 ; GFX11-CU-NEXT:    s_endpgm
7117 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7118 ; GFX10-CU:       ; %bb.0: ; %entry
7119 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7120 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7121 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7122 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7123 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7124 ; GFX10-CU-NEXT:    s_endpgm
7180 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7181 ; GFX11-CU:       ; %bb.0: ; %entry
7182 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7183 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7184 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7185 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7186 ; GFX11-CU-NEXT:    s_endpgm
7224 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7225 ; GFX10-CU:       ; %bb.0: ; %entry
7226 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7227 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7228 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7229 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7230 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7231 ; GFX10-CU-NEXT:    s_endpgm
7287 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7288 ; GFX11-CU:       ; %bb.0: ; %entry
7289 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7290 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7291 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7292 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7293 ; GFX11-CU-NEXT:    s_endpgm
7337 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7338 ; GFX10-CU:       ; %bb.0: ; %entry
7339 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7340 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7341 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7342 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7343 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7344 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7345 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7346 ; GFX10-CU-NEXT:    s_endpgm
7414 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7415 ; GFX11-CU:       ; %bb.0: ; %entry
7416 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7417 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7418 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7419 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7420 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7421 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7422 ; GFX11-CU-NEXT:    s_endpgm
7467 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7468 ; GFX10-CU:       ; %bb.0: ; %entry
7469 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7470 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7471 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7472 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7473 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7474 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7475 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7476 ; GFX10-CU-NEXT:    s_endpgm
7544 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7545 ; GFX11-CU:       ; %bb.0: ; %entry
7546 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7547 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7548 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7549 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7550 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7551 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7552 ; GFX11-CU-NEXT:    s_endpgm
7597 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7598 ; GFX10-CU:       ; %bb.0: ; %entry
7599 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7600 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7601 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7602 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7603 ; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7604 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7605 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7606 ; GFX10-CU-NEXT:    s_endpgm
7674 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7675 ; GFX11-CU:       ; %bb.0: ; %entry
7676 ; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7677 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7678 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7679 ; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7680 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7681 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7682 ; GFX11-CU-NEXT:    s_endpgm
7724 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7725 ; GFX10-CU:       ; %bb.0: ; %entry
7726 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7727 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7728 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7729 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7730 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7731 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7732 ; GFX10-CU-NEXT:    s_endpgm
7794 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7795 ; GFX11-CU:       ; %bb.0: ; %entry
7796 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7797 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7798 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7799 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
7800 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7801 ; GFX11-CU-NEXT:    s_endpgm
7843 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7844 ; GFX10-CU:       ; %bb.0: ; %entry
7845 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7846 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7847 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7848 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7849 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7850 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7851 ; GFX10-CU-NEXT:    s_endpgm
7913 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7914 ; GFX11-CU:       ; %bb.0: ; %entry
7915 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7916 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7917 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7918 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
7919 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7920 ; GFX11-CU-NEXT:    s_endpgm
7962 ; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7963 ; GFX10-CU:       ; %bb.0: ; %entry
7964 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7965 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7966 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7967 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7968 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7969 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7970 ; GFX10-CU-NEXT:    s_endpgm
8032 ; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8033 ; GFX11-CU:       ; %bb.0: ; %entry
8034 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8035 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8036 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8037 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8038 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8039 ; GFX11-CU-NEXT:    s_endpgm
8081 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8082 ; GFX10-CU:       ; %bb.0: ; %entry
8083 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8084 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8085 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8086 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8087 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8088 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8089 ; GFX10-CU-NEXT:    s_endpgm
8151 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8152 ; GFX11-CU:       ; %bb.0: ; %entry
8153 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8154 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8155 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8156 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8157 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8158 ; GFX11-CU-NEXT:    s_endpgm
8200 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8201 ; GFX10-CU:       ; %bb.0: ; %entry
8202 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8203 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8204 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8205 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8206 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8207 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8208 ; GFX10-CU-NEXT:    s_endpgm
8270 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8271 ; GFX11-CU:       ; %bb.0: ; %entry
8272 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8273 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8274 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8275 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8276 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8277 ; GFX11-CU-NEXT:    s_endpgm
8319 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8320 ; GFX10-CU:       ; %bb.0: ; %entry
8321 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8322 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8323 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8324 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8325 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8326 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8327 ; GFX10-CU-NEXT:    s_endpgm
8389 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8390 ; GFX11-CU:       ; %bb.0: ; %entry
8391 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8392 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8393 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8394 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8395 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8396 ; GFX11-CU-NEXT:    s_endpgm
8438 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8439 ; GFX10-CU:       ; %bb.0: ; %entry
8440 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8441 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8442 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8443 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8444 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8445 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8446 ; GFX10-CU-NEXT:    s_endpgm
8508 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8509 ; GFX11-CU:       ; %bb.0: ; %entry
8510 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8511 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8512 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8513 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8514 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8515 ; GFX11-CU-NEXT:    s_endpgm
8557 ; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8558 ; GFX10-CU:       ; %bb.0: ; %entry
8559 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8560 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8561 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8562 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8563 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8564 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8565 ; GFX10-CU-NEXT:    s_endpgm
8627 ; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8628 ; GFX11-CU:       ; %bb.0: ; %entry
8629 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8630 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8631 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8632 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8633 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8634 ; GFX11-CU-NEXT:    s_endpgm
8676 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8677 ; GFX10-CU:       ; %bb.0: ; %entry
8678 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8679 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8680 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8681 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8682 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8683 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8684 ; GFX10-CU-NEXT:    s_endpgm
8746 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8747 ; GFX11-CU:       ; %bb.0: ; %entry
8748 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8749 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8750 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8751 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8752 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8753 ; GFX11-CU-NEXT:    s_endpgm
8795 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8796 ; GFX10-CU:       ; %bb.0: ; %entry
8797 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8798 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8799 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8800 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8801 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8802 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8803 ; GFX10-CU-NEXT:    s_endpgm
8865 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8866 ; GFX11-CU:       ; %bb.0: ; %entry
8867 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8868 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8869 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8870 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8871 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8872 ; GFX11-CU-NEXT:    s_endpgm
8914 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8915 ; GFX10-CU:       ; %bb.0: ; %entry
8916 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8917 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8918 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8919 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8920 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8921 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8922 ; GFX10-CU-NEXT:    s_endpgm
8984 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8985 ; GFX11-CU:       ; %bb.0: ; %entry
8986 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8987 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8988 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8989 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8990 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8991 ; GFX11-CU-NEXT:    s_endpgm
9033 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9034 ; GFX10-CU:       ; %bb.0: ; %entry
9035 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9036 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9037 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9038 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9039 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9040 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9041 ; GFX10-CU-NEXT:    s_endpgm
9103 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9104 ; GFX11-CU:       ; %bb.0: ; %entry
9105 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9106 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9107 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9108 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9109 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9110 ; GFX11-CU-NEXT:    s_endpgm
9152 ; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9153 ; GFX10-CU:       ; %bb.0: ; %entry
9154 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9155 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9156 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9157 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9158 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9159 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9160 ; GFX10-CU-NEXT:    s_endpgm
9222 ; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9223 ; GFX11-CU:       ; %bb.0: ; %entry
9224 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9225 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9226 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9227 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9228 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9229 ; GFX11-CU-NEXT:    s_endpgm
9271 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9272 ; GFX10-CU:       ; %bb.0: ; %entry
9273 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9274 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9275 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9276 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9277 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9278 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9279 ; GFX10-CU-NEXT:    s_endpgm
9341 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9342 ; GFX11-CU:       ; %bb.0: ; %entry
9343 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9344 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9345 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9346 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9347 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9348 ; GFX11-CU-NEXT:    s_endpgm
9390 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9391 ; GFX10-CU:       ; %bb.0: ; %entry
9392 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9393 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9394 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9395 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9396 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9397 ; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9398 ; GFX10-CU-NEXT:    s_endpgm
9460 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9461 ; GFX11-CU:       ; %bb.0: ; %entry
9462 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9463 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9464 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9465 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9466 ; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9467 ; GFX11-CU-NEXT:    s_endpgm
9515 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9516 ; GFX10-CU:       ; %bb.0: ; %entry
9517 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9518 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9519 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9520 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9521 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9522 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9523 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9524 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9525 ; GFX10-CU-NEXT:    s_endpgm
9599 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9600 ; GFX11-CU:       ; %bb.0: ; %entry
9601 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9602 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9603 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9604 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9605 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9606 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9607 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9608 ; GFX11-CU-NEXT:    s_endpgm
9658 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9659 ; GFX10-CU:       ; %bb.0: ; %entry
9660 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9661 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9662 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9663 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9664 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9665 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9666 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9667 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9668 ; GFX10-CU-NEXT:    s_endpgm
9742 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9743 ; GFX11-CU:       ; %bb.0: ; %entry
9744 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9745 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9746 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9747 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9748 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9749 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9750 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9751 ; GFX11-CU-NEXT:    s_endpgm
9801 ; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9802 ; GFX10-CU:       ; %bb.0: ; %entry
9803 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9804 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9805 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9806 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9807 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9808 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9809 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9810 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9811 ; GFX10-CU-NEXT:    s_endpgm
9885 ; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9886 ; GFX11-CU:       ; %bb.0: ; %entry
9887 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9888 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9889 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9890 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9891 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9892 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9893 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9894 ; GFX11-CU-NEXT:    s_endpgm
9944 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9945 ; GFX10-CU:       ; %bb.0: ; %entry
9946 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9947 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9948 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9949 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9950 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9951 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9952 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9953 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9954 ; GFX10-CU-NEXT:    s_endpgm
10028 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10029 ; GFX11-CU:       ; %bb.0: ; %entry
10030 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10031 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10032 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10033 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10034 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10035 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10036 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10037 ; GFX11-CU-NEXT:    s_endpgm
10087 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10088 ; GFX10-CU:       ; %bb.0: ; %entry
10089 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10090 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10091 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10092 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10093 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10094 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10095 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10096 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10097 ; GFX10-CU-NEXT:    s_endpgm
10171 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10172 ; GFX11-CU:       ; %bb.0: ; %entry
10173 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10174 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10175 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10176 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10177 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10178 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10179 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10180 ; GFX11-CU-NEXT:    s_endpgm
10230 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10231 ; GFX10-CU:       ; %bb.0: ; %entry
10232 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10233 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10234 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10235 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10236 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10237 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10238 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10239 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10240 ; GFX10-CU-NEXT:    s_endpgm
10314 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10315 ; GFX11-CU:       ; %bb.0: ; %entry
10316 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10317 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10318 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10319 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10320 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10321 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10322 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10323 ; GFX11-CU-NEXT:    s_endpgm
10373 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10374 ; GFX10-CU:       ; %bb.0: ; %entry
10375 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10376 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10377 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10378 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10379 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10380 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10381 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10382 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10383 ; GFX10-CU-NEXT:    s_endpgm
10457 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10458 ; GFX11-CU:       ; %bb.0: ; %entry
10459 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10460 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10461 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10462 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10463 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10464 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10465 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10466 ; GFX11-CU-NEXT:    s_endpgm
10516 ; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10517 ; GFX10-CU:       ; %bb.0: ; %entry
10518 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10519 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10520 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10521 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10522 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10523 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10524 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10525 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10526 ; GFX10-CU-NEXT:    s_endpgm
10600 ; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10601 ; GFX11-CU:       ; %bb.0: ; %entry
10602 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10603 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10604 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10605 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10606 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10607 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10608 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10609 ; GFX11-CU-NEXT:    s_endpgm
10659 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10660 ; GFX10-CU:       ; %bb.0: ; %entry
10661 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10662 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10663 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10664 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10665 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10666 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10667 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10668 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10669 ; GFX10-CU-NEXT:    s_endpgm
10743 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10744 ; GFX11-CU:       ; %bb.0: ; %entry
10745 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10746 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10747 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10748 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10749 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10750 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10751 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10752 ; GFX11-CU-NEXT:    s_endpgm
10802 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10803 ; GFX10-CU:       ; %bb.0: ; %entry
10804 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10805 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10806 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10807 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10808 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10809 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10810 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10811 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10812 ; GFX10-CU-NEXT:    s_endpgm
10886 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10887 ; GFX11-CU:       ; %bb.0: ; %entry
10888 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10889 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10890 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10891 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10892 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10893 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10894 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10895 ; GFX11-CU-NEXT:    s_endpgm
10945 ; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10946 ; GFX10-CU:       ; %bb.0: ; %entry
10947 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10948 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10949 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10950 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10951 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10952 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10953 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10954 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10955 ; GFX10-CU-NEXT:    s_endpgm
11029 ; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11030 ; GFX11-CU:       ; %bb.0: ; %entry
11031 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11032 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11033 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11034 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11035 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11036 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11037 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11038 ; GFX11-CU-NEXT:    s_endpgm
11088 ; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11089 ; GFX10-CU:       ; %bb.0: ; %entry
11090 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11091 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11092 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11093 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11094 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11095 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11096 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11097 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11098 ; GFX10-CU-NEXT:    s_endpgm
11172 ; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11173 ; GFX11-CU:       ; %bb.0: ; %entry
11174 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11175 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11176 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11177 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11178 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11179 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11180 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11181 ; GFX11-CU-NEXT:    s_endpgm
11231 ; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11232 ; GFX10-CU:       ; %bb.0: ; %entry
11233 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11234 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11235 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11236 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11237 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11238 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11239 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11240 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11241 ; GFX10-CU-NEXT:    s_endpgm
11315 ; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11316 ; GFX11-CU:       ; %bb.0: ; %entry
11317 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11318 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11319 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11320 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11321 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11322 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11323 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11324 ; GFX11-CU-NEXT:    s_endpgm
11374 ; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11375 ; GFX10-CU:       ; %bb.0: ; %entry
11376 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11377 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11378 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11379 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11380 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11381 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11382 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11383 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11384 ; GFX10-CU-NEXT:    s_endpgm
11458 ; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11459 ; GFX11-CU:       ; %bb.0: ; %entry
11460 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11461 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11462 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11463 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11464 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11465 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11466 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11467 ; GFX11-CU-NEXT:    s_endpgm
11517 ; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11518 ; GFX10-CU:       ; %bb.0: ; %entry
11519 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11520 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11521 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11522 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11523 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11524 ; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11525 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11526 ; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11527 ; GFX10-CU-NEXT:    s_endpgm
11601 ; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11602 ; GFX11-CU:       ; %bb.0: ; %entry
11603 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11604 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11605 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11606 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11607 ; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11608 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11609 ; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11610 ; GFX11-CU-NEXT:    s_endpgm