Lines Matching refs:DL
5 … llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6 …lc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7 …lc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
165 ; GFX9-DL-LABEL: udot8_acc32:
166 ; GFX9-DL: ; %bb.0: ; %entry
167 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
168 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
169 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
170 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
171 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
172 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
173 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
174 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
175 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
176 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
177 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
178 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
179 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
180 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
181 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
182 ; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s0
183 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
184 ; GFX9-DL-NEXT: s_endpgm
186 ; GFX10-DL-LABEL: udot8_acc32:
187 ; GFX10-DL: ; %bb.0: ; %entry
188 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
189 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
190 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
191 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
192 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
193 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
194 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
195 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
196 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
197 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
198 ; GFX10-DL-NEXT: s_clause 0x1
199 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
200 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
201 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
202 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
203 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
204 ; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
205 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
206 ; GFX10-DL-NEXT: s_endpgm
432 ; GFX9-DL-LABEL: udot8_acc16:
433 ; GFX9-DL: ; %bb.0: ; %entry
434 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
435 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
436 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
437 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
438 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
439 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
440 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
441 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
442 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
443 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
444 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
445 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
446 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
447 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
448 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
449 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
450 ; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
451 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
452 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
453 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
454 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
455 ; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4
456 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
457 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
458 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
459 ; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4
460 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
461 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
462 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
463 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
464 ; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4
465 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
466 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
467 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
468 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1
469 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1
470 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1
471 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1
472 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1
473 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1
474 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1
475 ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
476 ; GFX9-DL-NEXT: s_endpgm
478 ; GFX10-DL-LABEL: udot8_acc16:
479 ; GFX10-DL: ; %bb.0: ; %entry
480 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
481 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
482 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
483 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
484 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
485 ; GFX10-DL-NEXT: s_clause 0x1
486 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
487 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
488 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
489 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
490 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
491 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
492 ; GFX10-DL-NEXT: s_clause 0x1
493 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
494 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
495 ; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
496 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
497 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2
498 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
499 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
500 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
501 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
502 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
503 ; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
504 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4
505 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
506 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
507 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
508 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
509 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
510 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4
511 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
512 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
513 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
514 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
515 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
516 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
517 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
518 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
519 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
520 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
521 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
522 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
523 ; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
524 ; GFX10-DL-NEXT: s_endpgm
750 ; GFX9-DL-LABEL: udot8_acc8:
751 ; GFX9-DL: ; %bb.0: ; %entry
752 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
753 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
754 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
755 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
756 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
757 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
758 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
759 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
760 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
761 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
762 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
763 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
764 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
765 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
766 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
767 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
768 ; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
769 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
770 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
771 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
772 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
773 ; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4
774 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
775 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
776 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
777 ; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4
778 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
779 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
780 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
781 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
782 ; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4
783 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
784 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
785 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
786 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1
787 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1
788 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1
789 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1
790 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1
791 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1
792 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1
793 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
794 ; GFX9-DL-NEXT: s_endpgm
796 ; GFX10-DL-LABEL: udot8_acc8:
797 ; GFX10-DL: ; %bb.0: ; %entry
798 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
799 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
800 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
801 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
802 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
803 ; GFX10-DL-NEXT: s_clause 0x1
804 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
805 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
806 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
807 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
808 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
809 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
810 ; GFX10-DL-NEXT: s_clause 0x1
811 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
812 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
813 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
814 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
815 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2
816 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
817 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
818 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
819 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
820 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
821 ; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
822 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4
823 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
824 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
825 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
826 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
827 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
828 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4
829 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
830 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
831 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
832 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
833 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
834 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
835 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
836 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
837 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
838 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
839 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
840 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
841 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
842 ; GFX10-DL-NEXT: s_endpgm
1071 ; GFX9-DL-LABEL: udot8_acc4:
1072 ; GFX9-DL: ; %bb.0: ; %entry
1073 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1074 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1075 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
1076 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
1077 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
1078 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1079 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1080 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1081 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
1082 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1083 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1084 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1085 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1086 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
1087 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
1088 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
1089 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
1090 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
1091 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
1092 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
1093 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
1094 ; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4
1095 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
1096 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
1097 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
1098 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 24, v2
1099 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
1100 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
1101 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
1102 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
1103 ; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4
1104 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
1105 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1106 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
1107 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1
1108 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1
1109 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1
1110 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1
1111 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1
1112 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1
1113 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1
1114 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
1115 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
1116 ; GFX9-DL-NEXT: s_endpgm
1118 ; GFX10-DL-LABEL: udot8_acc4:
1119 ; GFX10-DL: ; %bb.0: ; %entry
1120 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1121 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1122 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
1123 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
1124 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
1125 ; GFX10-DL-NEXT: s_clause 0x1
1126 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1127 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1128 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1129 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
1130 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
1131 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1132 ; GFX10-DL-NEXT: s_clause 0x1
1133 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
1134 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
1135 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
1136 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
1137 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2
1138 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
1139 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
1140 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
1141 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
1142 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1143 ; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
1144 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4
1145 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
1146 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1147 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
1148 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
1149 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1150 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4
1151 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
1152 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1153 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
1154 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
1155 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1156 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
1157 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
1158 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
1159 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
1160 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1161 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1162 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
1163 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
1164 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
1165 ; GFX10-DL-NEXT: s_endpgm
1378 ; GFX9-DL-LABEL: udot8_CommutationInsideMAD:
1379 ; GFX9-DL: ; %bb.0: ; %entry
1380 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1381 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1382 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
1383 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
1384 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
1385 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1386 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1387 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1388 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
1389 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1390 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1391 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1392 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1393 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
1394 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
1395 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
1396 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
1397 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
1398 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
1399 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
1400 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
1401 ; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4
1402 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
1403 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
1404 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
1405 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 24, v2
1406 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
1407 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
1408 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
1409 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
1410 ; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4
1411 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
1412 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1413 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
1414 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1
1415 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1
1416 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1
1417 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1
1418 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1
1419 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1
1420 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1
1421 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
1422 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
1423 ; GFX9-DL-NEXT: s_endpgm
1425 ; GFX10-DL-LABEL: udot8_CommutationInsideMAD:
1426 ; GFX10-DL: ; %bb.0: ; %entry
1427 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1428 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1429 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
1430 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
1431 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
1432 ; GFX10-DL-NEXT: s_clause 0x1
1433 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1434 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1435 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1436 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
1437 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
1438 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1439 ; GFX10-DL-NEXT: s_clause 0x1
1440 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
1441 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
1442 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
1443 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
1444 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2
1445 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
1446 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
1447 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
1448 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
1449 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1450 ; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
1451 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4
1452 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
1453 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1454 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
1455 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
1456 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1457 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4
1458 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
1459 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1460 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
1461 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
1462 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1463 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
1464 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
1465 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
1466 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
1467 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
1468 ; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0
1469 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
1470 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
1471 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
1472 ; GFX10-DL-NEXT: s_endpgm
1689 ; GFX9-DL-LABEL: udot8_multiuses_mul1:
1690 ; GFX9-DL: ; %bb.0: ; %entry
1691 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1692 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1693 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
1694 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
1695 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
1696 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1697 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1698 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1699 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
1700 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1701 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1702 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1703 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
1704 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1705 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
1706 ; GFX9-DL-NEXT: v_bfe_u32 v3, v1, 4, 4
1707 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
1708 ; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
1709 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
1710 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
1711 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
1712 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
1713 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
1714 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1715 ; GFX9-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
1716 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
1717 ; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4
1718 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
1719 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
1720 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
1721 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
1722 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
1723 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v17, v1, v2
1724 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1725 ; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, s0
1726 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v9, v9, v16
1727 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v15
1728 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v10, v1
1729 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v14
1730 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v13
1731 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v9, v8
1732 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v12
1733 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v11
1734 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v7, v6
1735 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v5, v4
1736 ; GFX9-DL-NEXT: v_add3_u32 v1, v17, v1, v2
1737 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
1738 ; GFX9-DL-NEXT: s_endpgm
1740 ; GFX10-DL-LABEL: udot8_multiuses_mul1:
1741 ; GFX10-DL: ; %bb.0: ; %entry
1742 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1743 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1744 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1745 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1746 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1747 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
1748 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
1749 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
1750 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
1751 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1752 ; GFX10-DL-NEXT: s_clause 0x1
1753 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
1754 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
1755 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
1756 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
1757 ; GFX10-DL-NEXT: v_and_b32_e32 v8, 15, v1
1758 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1759 ; GFX10-DL-NEXT: v_and_b32_e32 v9, 15, v2
1760 ; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
1761 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v1
1762 ; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 24, 4
1763 ; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 4
1764 ; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 4
1765 ; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 4
1766 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 8, 4
1767 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
1768 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
1769 ; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 12, 4
1770 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1771 ; GFX10-DL-NEXT: v_mad_u32_u24 v13, v8, v9, s2
1772 ; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 4
1773 ; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 16, 4
1774 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v11
1775 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v12
1776 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v10, v13
1777 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
1778 ; GFX10-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
1779 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v15
1780 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v14
1781 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v7
1782 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v4, v2
1783 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v10
1784 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v3, v8, v9
1785 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v6, v5
1786 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
1787 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
1788 ; GFX10-DL-NEXT: v_add3_u32 v0, v3, v13, v0
1789 ; GFX10-DL-NEXT: global_store_dword v1, v0, s[0:1]
1790 ; GFX10-DL-NEXT: s_endpgm
2020 ; GFX9-DL-LABEL: udot8_acc32_vecMul:
2021 ; GFX9-DL: ; %bb.0: ; %entry
2022 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2023 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2024 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
2025 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
2026 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
2027 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2028 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2029 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2030 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
2031 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2032 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
2033 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
2034 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
2035 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2036 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2037 ; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s0
2038 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
2039 ; GFX9-DL-NEXT: s_endpgm
2041 ; GFX10-DL-LABEL: udot8_acc32_vecMul:
2042 ; GFX10-DL: ; %bb.0: ; %entry
2043 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2044 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2045 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2046 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2047 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2048 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
2049 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
2050 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
2051 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
2052 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2053 ; GFX10-DL-NEXT: s_clause 0x1
2054 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
2055 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
2056 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
2057 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
2058 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2059 ; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
2060 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
2061 ; GFX10-DL-NEXT: s_endpgm
2284 ; GFX9-DL-LABEL: udot8_acc16_vecMul:
2285 ; GFX9-DL: ; %bb.0: ; %entry
2286 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2287 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2288 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
2289 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
2290 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
2291 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2292 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2293 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2294 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
2295 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2296 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
2297 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
2298 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2299 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
2300 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
2301 ; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v1
2302 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
2303 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2304 ; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v2
2305 ; GFX9-DL-NEXT: v_bfe_u32 v4, v1, 4, 4
2306 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
2307 ; GFX9-DL-NEXT: v_bfe_u32 v11, v2, 4, 4
2308 ; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xffff, v7
2309 ; GFX9-DL-NEXT: v_and_b32_e32 v12, 0xffff, v12
2310 ; GFX9-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
2311 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
2312 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
2313 ; GFX9-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
2314 ; GFX9-DL-NEXT: v_lshl_or_b32 v7, v11, 16, v12
2315 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
2316 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
2317 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
2318 ; GFX9-DL-NEXT: v_and_b32_e32 v9, 0xffff, v9
2319 ; GFX9-DL-NEXT: v_and_b32_e32 v14, 0xffff, v14
2320 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v7
2321 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
2322 ; GFX9-DL-NEXT: v_bfe_u32 v1, v1, 24, 4
2323 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
2324 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
2325 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
2326 ; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
2327 ; GFX9-DL-NEXT: v_lshl_or_b32 v8, v8, 16, v9
2328 ; GFX9-DL-NEXT: v_lshl_or_b32 v9, v13, 16, v14
2329 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2330 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v4, v3
2331 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
2332 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
2333 ; GFX9-DL-NEXT: v_and_b32_e32 v16, 0xffff, v16
2334 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v6, v9
2335 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
2336 ; GFX9-DL-NEXT: v_lshl_or_b32 v2, v17, 16, v2
2337 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v10, 16, v1
2338 ; GFX9-DL-NEXT: v_lshl_or_b32 v10, v15, 16, v16
2339 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v5
2340 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
2341 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v8, v10
2342 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
2343 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v2
2344 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
2345 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v1
2346 ; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
2347 ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
2348 ; GFX9-DL-NEXT: s_endpgm
2350 ; GFX10-DL-LABEL: udot8_acc16_vecMul:
2351 ; GFX10-DL: ; %bb.0: ; %entry
2352 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2353 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2354 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2355 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2356 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2357 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
2358 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
2359 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
2360 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
2361 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2362 ; GFX10-DL-NEXT: s_clause 0x1
2363 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
2364 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
2365 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
2366 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
2367 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
2368 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 15, v1
2369 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2370 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v2
2371 ; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 4, 4
2372 ; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
2373 ; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
2374 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 0xffff, v6
2375 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
2376 ; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 8, 4
2377 ; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 4
2378 ; GFX10-DL-NEXT: v_and_b32_e32 v8, 0xffff, v8
2379 ; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v6
2380 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v9, 16, v5
2381 ; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 12, 4
2382 ; GFX10-DL-NEXT: v_and_b32_e32 v12, 0xffff, v12
2383 ; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 16, 4
2384 ; GFX10-DL-NEXT: v_lshl_or_b32 v7, v7, 16, v8
2385 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5
2386 ; GFX10-DL-NEXT: v_bfe_u32 v5, v2, 16, 4
2387 ; GFX10-DL-NEXT: v_lshl_or_b32 v9, v9, 16, v12
2388 ; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 20, 4
2389 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 0xffff, v11
2390 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v4
2391 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2392 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v3
2393 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 20, 4
2394 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
2395 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v7, v7, v9
2396 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 28, v1
2397 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v8
2398 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 24, 4
2399 ; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 24, 4
2400 ; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
2401 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v10, 16, v11
2402 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v7
2403 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v7
2404 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
2405 ; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xffff, v8
2406 ; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
2407 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v4
2408 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v9
2409 ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v7
2410 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v6, 16, v1
2411 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
2412 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v4
2413 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
2414 ; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v5
2415 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
2416 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v1
2417 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v3
2418 ; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
2419 ; GFX10-DL-NEXT: s_endpgm
2687 ; GFX9-DL-LABEL: udot8_acc8_vecMul:
2688 ; GFX9-DL: ; %bb.0: ; %entry
2689 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2690 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2691 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
2692 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
2693 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
2694 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2695 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2696 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2697 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
2698 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2699 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
2700 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
2701 ; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[2:3]
2702 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
2703 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
2704 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 28, v1
2705 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 24, 4
2706 ; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 20, 4
2707 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2708 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v15, 28, v2
2709 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 24, 4
2710 ; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 20, 4
2711 ; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
2712 ; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v1
2713 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
2714 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
2715 ; GFX9-DL-NEXT: v_bfe_u32 v11, v1, 16, 4
2716 ; GFX9-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
2717 ; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v2
2718 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
2719 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
2720 ; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 16, 4
2721 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DW…
2722 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v17, v9, v16
2723 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWOR…
2724 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v2
2725 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v14
2726 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWOR…
2727 ; GFX9-DL-NEXT: v_or_b32_e32 v8, v17, v8
2728 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, v5, v12
2729 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWOR…
2730 ; GFX9-DL-NEXT: v_or_b32_e32 v1, v18, v10
2731 ; GFX9-DL-NEXT: v_or_b32_e32 v6, v7, v6
2732 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v7, 16, v8
2733 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
2734 ; GFX9-DL-NEXT: v_or_b32_sdwa v1, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src…
2735 ; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v12
2736 ; GFX9-DL-NEXT: v_or_b32_e32 v7, v12, v0
2737 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v1
2738 ; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
2739 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
2740 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2741 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v5, v4
2742 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v7
2743 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
2744 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
2745 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v11, v2, v0
2746 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v10
2747 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v8
2748 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
2749 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
2750 ; GFX9-DL-NEXT: global_store_byte v3, v0, s[2:3]
2751 ; GFX9-DL-NEXT: s_endpgm
2753 ; GFX10-DL-LABEL: udot8_acc8_vecMul:
2754 ; GFX10-DL: ; %bb.0: ; %entry
2755 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2756 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2757 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2758 ; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0
2759 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
2760 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
2761 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
2762 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
2763 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
2764 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
2765 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2766 ; GFX10-DL-NEXT: s_clause 0x1
2767 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
2768 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
2769 ; GFX10-DL-NEXT: global_load_ubyte v3, v4, s[0:1]
2770 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
2771 ; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
2772 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2773 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 12, 4
2774 ; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
2775 ; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 8, 4
2776 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 28, v1
2777 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v2
2778 ; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v10
2779 ; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
2780 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v1
2781 ; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 24, 4
2782 ; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 20, 4
2783 ; GFX10-DL-NEXT: v_bfe_u32 v12, v1, 16, 4
2784 ; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
2785 ; GFX10-DL-NEXT: v_mul_lo_u16 v7, v7, v13
2786 ; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
2787 ; GFX10-DL-NEXT: v_and_b32_e32 v10, 15, v2
2788 ; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 24, 4
2789 ; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
2790 ; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
2791 ; GFX10-DL-NEXT: v_mul_lo_u16 v2, v8, v14
2792 ; GFX10-DL-NEXT: v_mul_lo_u16 v0, v0, v1
2793 ; GFX10-DL-NEXT: v_or_b32_e32 v6, v7, v6
2794 ; GFX10-DL-NEXT: v_mul_lo_u16 v1, v11, v13
2795 ; GFX10-DL-NEXT: v_mul_lo_u16 v7, v9, v15
2796 ; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v2
2797 ; GFX10-DL-NEXT: v_lshlrev_b16 v8, 8, v0
2798 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
2799 ; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v10
2800 ; GFX10-DL-NEXT: v_mul_lo_u16 v10, v12, v16
2801 ; GFX10-DL-NEXT: v_lshlrev_b16 v1, 8, v1
2802 ; GFX10-DL-NEXT: v_or_b32_e32 v7, v7, v2
2803 ; GFX10-DL-NEXT: v_or_b32_sdwa v2, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 sr…
2804 ; GFX10-DL-NEXT: v_or_b32_e32 v5, v5, v8
2805 ; GFX10-DL-NEXT: v_or_b32_e32 v1, v10, v1
2806 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v7
2807 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v2
2808 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2809 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v5, v3
2810 ; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 sr…
2811 ; GFX10-DL-NEXT: v_add_nc_u16 v5, v3, v2
2812 ; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
2813 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
2814 ; GFX10-DL-NEXT: v_add_nc_u16 v0, v5, v6
2815 ; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v2
2816 ; GFX10-DL-NEXT: v_mad_u16 v0, v12, v16, v0
2817 ; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
2818 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v7
2819 ; GFX10-DL-NEXT: v_mad_u16 v0, v9, v15, v0
2820 ; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
2821 ; GFX10-DL-NEXT: global_store_byte v4, v0, s[0:1]
2822 ; GFX10-DL-NEXT: s_endpgm
3035 ; GFX9-DL-LABEL: udot8_acc4_vecMul:
3036 ; GFX9-DL: ; %bb.0: ; %entry
3037 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
3038 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
3039 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
3040 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
3041 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
3042 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3043 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
3044 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3045 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
3046 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3047 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
3048 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
3049 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3050 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
3051 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
3052 ; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v1
3053 ; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
3054 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
3055 ; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v2
3056 ; GFX9-DL-NEXT: v_bfe_u32 v4, v1, 4, 4
3057 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
3058 ; GFX9-DL-NEXT: v_bfe_u32 v11, v2, 4, 4
3059 ; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xffff, v7
3060 ; GFX9-DL-NEXT: v_and_b32_e32 v12, 0xffff, v12
3061 ; GFX9-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
3062 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
3063 ; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
3064 ; GFX9-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
3065 ; GFX9-DL-NEXT: v_lshl_or_b32 v7, v11, 16, v12
3066 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
3067 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
3068 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
3069 ; GFX9-DL-NEXT: v_and_b32_e32 v9, 0xffff, v9
3070 ; GFX9-DL-NEXT: v_and_b32_e32 v14, 0xffff, v14
3071 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v7
3072 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
3073 ; GFX9-DL-NEXT: v_bfe_u32 v1, v1, 24, 4
3074 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
3075 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
3076 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
3077 ; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
3078 ; GFX9-DL-NEXT: v_lshl_or_b32 v8, v8, 16, v9
3079 ; GFX9-DL-NEXT: v_lshl_or_b32 v9, v13, 16, v14
3080 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3081 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v4, v3
3082 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
3083 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
3084 ; GFX9-DL-NEXT: v_and_b32_e32 v16, 0xffff, v16
3085 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v6, v9
3086 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
3087 ; GFX9-DL-NEXT: v_lshl_or_b32 v2, v17, 16, v2
3088 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v10, 16, v1
3089 ; GFX9-DL-NEXT: v_lshl_or_b32 v10, v15, 16, v16
3090 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v5
3091 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
3092 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v8, v10
3093 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
3094 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v2
3095 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
3096 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v1
3097 ; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src…
3098 ; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
3099 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
3100 ; GFX9-DL-NEXT: s_endpgm
3102 ; GFX10-DL-LABEL: udot8_acc4_vecMul:
3103 ; GFX10-DL: ; %bb.0: ; %entry
3104 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3105 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3106 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3107 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
3108 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
3109 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
3110 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
3111 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
3112 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
3113 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3114 ; GFX10-DL-NEXT: s_clause 0x1
3115 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
3116 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
3117 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
3118 ; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[0:1]
3119 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
3120 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 15, v1
3121 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
3122 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v2
3123 ; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 4, 4
3124 ; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
3125 ; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
3126 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 0xffff, v6
3127 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
3128 ; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 8, 4
3129 ; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 4
3130 ; GFX10-DL-NEXT: v_and_b32_e32 v8, 0xffff, v8
3131 ; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v6
3132 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v9, 16, v5
3133 ; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 12, 4
3134 ; GFX10-DL-NEXT: v_and_b32_e32 v12, 0xffff, v12
3135 ; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 16, 4
3136 ; GFX10-DL-NEXT: v_lshl_or_b32 v7, v7, 16, v8
3137 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5
3138 ; GFX10-DL-NEXT: v_bfe_u32 v5, v2, 16, 4
3139 ; GFX10-DL-NEXT: v_lshl_or_b32 v9, v9, 16, v12
3140 ; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 20, 4
3141 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 0xffff, v11
3142 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v4
3143 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3144 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v3
3145 ; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 20, 4
3146 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5
3147 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v7, v7, v9
3148 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 28, v1
3149 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v8
3150 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 24, 4
3151 ; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 24, 4
3152 ; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5
3153 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v10, 16, v11
3154 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v7
3155 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v7
3156 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
3157 ; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xffff, v8
3158 ; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
3159 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v4
3160 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v9
3161 ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v7
3162 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v6, 16, v1
3163 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
3164 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v4
3165 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
3166 ; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v5
3167 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
3168 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v1
3169 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v3
3170 ; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
3171 ; GFX10-DL-NEXT: global_store_byte v0, v1, s[0:1]
3172 ; GFX10-DL-NEXT: s_endpgm
3344 ; GFX9-DL-LABEL: udot8_variant1:
3345 ; GFX9-DL: ; %bb.0: ; %entry
3346 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3347 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
3348 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3349 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3350 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
3351 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
3352 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
3353 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3354 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3355 ; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s0
3356 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
3357 ; GFX9-DL-NEXT: s_endpgm
3359 ; GFX10-DL-LABEL: udot8_variant1:
3360 ; GFX10-DL: ; %bb.0: ; %entry
3361 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3362 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3363 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3364 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3365 ; GFX10-DL-NEXT: s_clause 0x1
3366 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
3367 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
3368 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
3369 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
3370 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3371 ; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s2
3372 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
3373 ; GFX10-DL-NEXT: s_endpgm