1//===-- VOP3PInstructions.td - Vector Instruction Definitions -------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9//===----------------------------------------------------------------------===// 10// VOP3P Classes 11//===----------------------------------------------------------------------===// 12 13class VOP3P_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR, 14 bit HasDPP = 0> : VOP3_Profile<P, Features> { 15 let IsVOP3P = 1; 16 let HasExtVOP3DPP = HasDPP; 17 // We do not want to print src modifiers for vop3p because the bits are 18 // overloaded in meaning and the logic in printOperandAndFPInputMods is 19 // wrong for vop3p 20 let AsmVOP3DPPBase = AsmVOP3P; 21} 22 23// Used for FMA_MIX* and MAD_MIX* insts 24// Their operands are only sort of f16 operands. Depending on 25// op_sel_hi, these may be interpreted as f32. The inline immediate 26// values are really f16 converted to f32, so we treat these as f16 27// operands. 28class VOP3P_Mix_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR, 29 bit useTiedOutput = 0> : VOP3P_Profile<P, Features, 1> { 30 bit UseTiedOutput = useTiedOutput; 31 32 dag srcs = 33 (ins FP16InputMods:$src0_modifiers, VCSrc_f16:$src0, 34 FP16InputMods:$src1_modifiers, VCSrc_f16:$src1, 35 FP16InputMods:$src2_modifiers, VCSrc_f16:$src2); 36 dag dpp_srcs = 37 (ins FPVRegInputMods:$src0_modifiers, VGPRSrc_32:$src0, 38 FP16InputMods:$src1_modifiers, VCSrc_f16:$src1, 39 FP16InputMods:$src2_modifiers, VCSrc_f16:$src2); 40 41 // FIXME: clampmod0 misbehaves with the non-default vdst_in 42 // following it. For now workaround this by requiring clamp 43 // in tied patterns. This should use undef_tied_input, but it 44 // seems underdeveloped and doesn't apply the right register 45 // class constraints. 46 dag mods = !con(!if(UseTiedOutput, (ins clampmod:$clamp, VGPR_32:$vdst_in), 47 (ins clampmod0:$clamp)), 48 (ins op_sel0:$op_sel, op_sel_hi0:$op_sel_hi)); 49 // We use Ins64 because that is the one which populates InOperandList 50 // due to the logic in class VOP3_Pseudo 51 let Ins64 = !con(srcs, mods); 52 let InsVOP3Base = !con(dpp_srcs, mods); 53 let Asm64 = 54 "$vdst, $src0_modifiers, $src1_modifiers, $src2_modifiers$op_sel$op_sel_hi$clamp"; 55 let AsmVOP3DPPBase = Asm64; 56} 57 58multiclass VOP3PInst<string OpName, VOPProfile P, 59 SDPatternOperator node = null_frag, bit IsDOT = 0> { 60 def NAME : VOP3P_Pseudo<OpName, P, 61 !if (P.HasModifiers, 62 getVOP3PModPat<P, node, IsDOT, IsDOT>.ret, 63 getVOP3Pat<P, node>.ret)>; 64 let SubtargetPredicate = isGFX11Plus in { 65 if P.HasExtVOP3DPP then 66 def _dpp : VOP3_DPP_Pseudo<OpName, P> { 67 let VOP3P = 1; 68 let PseudoInstr = OpName #"_dpp"; 69 } 70 } // end SubtargetPredicate = isGFX11Plus 71} 72 73// Non-packed instructions that use the VOP3P encoding. 74// VOP3 neg/abs and VOP3P opsel/opsel_hi modifiers are allowed. 75multiclass VOP3_VOP3PInst<string OpName, VOP3P_Mix_Profile P> { 76 def NAME : VOP3P_Pseudo<OpName, P> { 77 let Constraints = !if(P.UseTiedOutput, "$vdst = $vdst_in", ""); 78 let DisableEncoding = !if(P.UseTiedOutput, "$vdst_in", ""); 79 } 80 let SubtargetPredicate = isGFX11Plus in { 81 if P.HasExtVOP3DPP then 82 def _dpp : VOP3_DPP_Pseudo<OpName, P> { 83 let VOP3P = 1; 84 let PseudoInstr = OpName#"_dpp"; 85 let Constraints = !if(P.UseTiedOutput, "$vdst = $vdst_in", ""); 86 let DisableEncoding = !if(P.UseTiedOutput, "$vdst_in", ""); 87 } 88 } // end SubtargetPredicate = isGFX11Plus 89} 90 91let isReMaterializable = 1 in { 92let isCommutable = 1 in { 93defm V_PK_MAD_I16 : VOP3PInst<"v_pk_mad_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>; 94defm V_PK_MAD_U16 : VOP3PInst<"v_pk_mad_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>; 95 96let FPDPRounding = 1 in { 97defm V_PK_FMA_F16 : VOP3PInst<"v_pk_fma_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16_V2F16>, any_fma>; 98defm V_PK_ADD_F16 : VOP3PInst<"v_pk_add_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, any_fadd>; 99defm V_PK_MUL_F16 : VOP3PInst<"v_pk_mul_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, any_fmul>; 100} // End FPDPRounding = 1 101defm V_PK_MAX_F16 : VOP3PInst<"v_pk_max_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fmaxnum_like>; 102defm V_PK_MIN_F16 : VOP3PInst<"v_pk_min_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fminnum_like>; 103 104defm V_PK_ADD_U16 : VOP3PInst<"v_pk_add_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, add>; 105defm V_PK_ADD_I16 : VOP3PInst<"v_pk_add_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>>; 106defm V_PK_MUL_LO_U16 : VOP3PInst<"v_pk_mul_lo_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, mul>; 107 108defm V_PK_MIN_I16 : VOP3PInst<"v_pk_min_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, smin>; 109defm V_PK_MIN_U16 : VOP3PInst<"v_pk_min_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umin>; 110defm V_PK_MAX_I16 : VOP3PInst<"v_pk_max_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, smax>; 111defm V_PK_MAX_U16 : VOP3PInst<"v_pk_max_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umax>; 112} 113 114defm V_PK_SUB_U16 : VOP3PInst<"v_pk_sub_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>>; 115defm V_PK_SUB_I16 : VOP3PInst<"v_pk_sub_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, sub>; 116 117defm V_PK_LSHLREV_B16 : VOP3PInst<"v_pk_lshlrev_b16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, clshl_rev_16>; 118defm V_PK_ASHRREV_I16 : VOP3PInst<"v_pk_ashrrev_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, cashr_rev_16>; 119defm V_PK_LSHRREV_B16 : VOP3PInst<"v_pk_lshrrev_b16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, clshr_rev_16>; 120} // End isReMaterializable = 1 121 122let SubtargetPredicate = HasVOP3PInsts in { 123 124// Undo sub x, c -> add x, -c canonicalization since c is more likely 125// an inline immediate than -c. 126// The constant will be emitted as a mov, and folded later. 127// TODO: We could directly encode the immediate now 128def : GCNPat< 129 (add (v2i16 (VOP3PMods v2i16:$src0, i32:$src0_modifiers)), NegSubInlineConstV216:$src1), 130 (V_PK_SUB_U16 $src0_modifiers, $src0, SRCMODS.OP_SEL_1, NegSubInlineConstV216:$src1) 131>; 132 133// Integer operations with clamp bit set. 134class VOP3PSatPat<SDPatternOperator pat, Instruction inst> : GCNPat< 135 (pat (v2i16 (VOP3PMods v2i16:$src0, i32:$src0_modifiers)), 136 (v2i16 (VOP3PMods v2i16:$src1, i32:$src1_modifiers))), 137 (inst $src0_modifiers, $src0, $src1_modifiers, $src1, DSTCLAMP.ENABLE) 138>; 139 140def : VOP3PSatPat<uaddsat, V_PK_ADD_U16>; 141def : VOP3PSatPat<saddsat, V_PK_ADD_I16>; 142def : VOP3PSatPat<usubsat, V_PK_SUB_U16>; 143def : VOP3PSatPat<ssubsat, V_PK_SUB_I16>; 144} // End SubtargetPredicate = HasVOP3PInsts 145 146multiclass MadFmaMixPats<SDPatternOperator fma_like, 147 Instruction mixlo_inst, 148 Instruction mixhi_inst> { 149 def : GCNPat < 150 (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 151 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 152 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))), 153 (mixlo_inst $src0_modifiers, $src0, 154 $src1_modifiers, $src1, 155 $src2_modifiers, $src2, 156 DSTCLAMP.NONE, 157 (i32 (IMPLICIT_DEF))) 158 >; 159 160 // FIXME: Special case handling for maxhi (especially for clamp) 161 // because dealing with the write to high half of the register is 162 // difficult. 163 def : GCNPat < 164 (build_vector f16:$elt0, (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 165 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 166 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))), 167 (v2f16 (mixhi_inst $src0_modifiers, $src0, 168 $src1_modifiers, $src1, 169 $src2_modifiers, $src2, 170 DSTCLAMP.NONE, 171 $elt0)) 172 >; 173 174 def : GCNPat < 175 (build_vector 176 f16:$elt0, 177 (AMDGPUclamp (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 178 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 179 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers)))))), 180 (v2f16 (mixhi_inst $src0_modifiers, $src0, 181 $src1_modifiers, $src1, 182 $src2_modifiers, $src2, 183 DSTCLAMP.ENABLE, 184 $elt0)) 185 >; 186 187 def : GCNPat < 188 (AMDGPUclamp (build_vector 189 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$lo_src0, i32:$lo_src0_modifiers)), 190 (f32 (VOP3PMadMixMods f16:$lo_src1, i32:$lo_src1_modifiers)), 191 (f32 (VOP3PMadMixMods f16:$lo_src2, i32:$lo_src2_modifiers)))), 192 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$hi_src0, i32:$hi_src0_modifiers)), 193 (f32 (VOP3PMadMixMods f16:$hi_src1, i32:$hi_src1_modifiers)), 194 (f32 (VOP3PMadMixMods f16:$hi_src2, i32:$hi_src2_modifiers)))))), 195 (v2f16 (mixhi_inst $hi_src0_modifiers, $hi_src0, 196 $hi_src1_modifiers, $hi_src1, 197 $hi_src2_modifiers, $hi_src2, 198 DSTCLAMP.ENABLE, 199 (mixlo_inst $lo_src0_modifiers, $lo_src0, 200 $lo_src1_modifiers, $lo_src1, 201 $lo_src2_modifiers, $lo_src2, 202 DSTCLAMP.ENABLE, 203 (i32 (IMPLICIT_DEF))))) 204 >; 205} 206 207let SubtargetPredicate = HasMadMixInsts in { 208 209// These are VOP3a-like opcodes which accept no omod. 210// Size of src arguments (16/32) is controlled by op_sel. 211// For 16-bit src arguments their location (hi/lo) are controlled by op_sel_hi. 212let isCommutable = 1, mayRaiseFPException = 0 in { 213let isReMaterializable = 1 in 214defm V_MAD_MIX_F32 : VOP3_VOP3PInst<"v_mad_mix_f32", VOP3P_Mix_Profile<VOP_F32_F16_F16_F16, VOP3_OPSEL>>; 215 216let FPDPRounding = 1 in { 217// Clamp modifier is applied after conversion to f16. 218defm V_MAD_MIXLO_F16 : VOP3_VOP3PInst<"v_mad_mixlo_f16", VOP3P_Mix_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL, 1>>; 219 220let ClampLo = 0, ClampHi = 1 in { 221defm V_MAD_MIXHI_F16 : VOP3_VOP3PInst<"v_mad_mixhi_f16", VOP3P_Mix_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL, 1>>; 222} 223} // End FPDPRounding = 1 224} 225 226defm : MadFmaMixPats<fmad, V_MAD_MIXLO_F16, V_MAD_MIXHI_F16>; 227} // End SubtargetPredicate = HasMadMixInsts 228 229 230// Essentially the same as the mad_mix versions 231let SubtargetPredicate = HasFmaMixInsts in { 232let isCommutable = 1 in { 233 234let isReMaterializable = 1 in 235defm V_FMA_MIX_F32 : VOP3_VOP3PInst<"v_fma_mix_f32", VOP3P_Mix_Profile<VOP_F32_F16_F16_F16, VOP3_OPSEL>>; 236 237let FPDPRounding = 1 in { 238// Clamp modifier is applied after conversion to f16. 239defm V_FMA_MIXLO_F16 : VOP3_VOP3PInst<"v_fma_mixlo_f16", VOP3P_Mix_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL, 1>>; 240 241let ClampLo = 0, ClampHi = 1 in { 242defm V_FMA_MIXHI_F16 : VOP3_VOP3PInst<"v_fma_mixhi_f16", VOP3P_Mix_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL, 1>>; 243} 244} // End FPDPRounding = 1 245} 246 247defm : MadFmaMixPats<fma, V_FMA_MIXLO_F16, V_FMA_MIXHI_F16>; 248} 249 250// Defines patterns that extract signed 4bit from each Idx[0]. 251foreach Idx = [[0,28],[4,24],[8,20],[12,16],[16,12],[20,8],[24,4]] in 252 def ExtractSigned4bit_#Idx[0] : PatFrag<(ops node:$src), 253 (sra (shl node:$src, (i32 Idx[1])), (i32 28))>; 254 255// Defines code pattern that extracts U(unsigned/signed) 4/8bit from FromBitIndex. 256class Extract<int FromBitIndex, int BitMask, bit U>: PatFrag< 257 (ops node:$src), 258 !if (!or (!and (!eq (BitMask, 255), !eq (FromBitIndex, 24)), !eq (FromBitIndex, 28)), // last element 259 !if (U, (srl node:$src, (i32 FromBitIndex)), (sra node:$src, (i32 FromBitIndex))), 260 !if (!eq (FromBitIndex, 0), // first element 261 !if (U, (and node:$src, (i32 BitMask)), 262 !if (!eq (BitMask, 15), (!cast<PatFrag>("ExtractSigned4bit_"#FromBitIndex) node:$src), 263 (sext_inreg node:$src, i8))), 264 !if (U, (and (srl node:$src, (i32 FromBitIndex)), (i32 BitMask)), 265 !if (!eq (BitMask, 15), (!cast<PatFrag>("ExtractSigned4bit_"#FromBitIndex) node:$src), 266 (sext_inreg (srl node:$src, (i32 FromBitIndex)), i8)))))>; 267 268 269foreach Type = ["I", "U"] in 270 foreach Index = 0-3 in { 271 // Defines patterns that extract each Index'ed 8bit from an unsigned 272 // 32bit scalar value; 273 def Type#Index#"_8bit" : Extract<!shl(Index, 3), 255, !eq (Type, "U")>; 274 275 // Defines multiplication patterns where the multiplication is happening on each 276 // Index'ed 8bit of a 32bit scalar value. 277 278 def Mul#Type#_Elt#Index : PatFrag< 279 (ops node:$src0, node:$src1), 280 (!cast<HasOneUseBinOp>(!if (!eq (Type, "I"), AMDGPUmul_i24_oneuse, AMDGPUmul_u24_oneuse)) 281 (!cast<Extract>(Type#Index#"_8bit") node:$src0), 282 (!cast<Extract>(Type#Index#"_8bit") node:$src1))>; 283 } 284 285// Different variants of dot8 patterns cause a huge increase in the compile time. 286// Define non-associative/commutative add/mul to prevent permutation in the dot8 287// pattern. 288def NonACAdd : SDNode<"ISD::ADD" , SDTIntBinOp>; 289def NonACAdd_oneuse : HasOneUseBinOp<NonACAdd>; 290 291def NonACAMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24" , SDTIntBinOp>; 292def NonACAMDGPUmul_u24_oneuse : HasOneUseBinOp<NonACAMDGPUmul_u24>; 293 294def NonACAMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24" , SDTIntBinOp>; 295def NonACAMDGPUmul_i24_oneuse : HasOneUseBinOp<NonACAMDGPUmul_i24>; 296 297foreach Type = ["I", "U"] in 298 foreach Index = 0-7 in { 299 // Defines patterns that extract each Index'ed 4bit from an unsigned 300 // 32bit scalar value; 301 def Type#Index#"_4bit" : Extract<!shl(Index, 2), 15, !eq (Type, "U")>; 302 303 // Defines multiplication patterns where the multiplication is happening on each 304 // Index'ed 8bit of a 32bit scalar value. 305 def Mul#Type#Index#"_4bit" : PatFrag< 306 (ops node:$src0, node:$src1), 307 (!cast<HasOneUseBinOp>(!if (!eq (Type, "I"), NonACAMDGPUmul_i24_oneuse, NonACAMDGPUmul_u24_oneuse)) 308 (!cast<Extract>(Type#Index#"_4bit") node:$src0), 309 (!cast<Extract>(Type#Index#"_4bit") node:$src1))>; 310 } 311 312class UDot2Pat<Instruction Inst> : GCNPat < 313 (add (add_oneuse (AMDGPUmul_u24_oneuse (srl i32:$src0, (i32 16)), 314 (srl i32:$src1, (i32 16))), i32:$src2), 315 (AMDGPUmul_u24_oneuse (and i32:$src0, (i32 65535)), 316 (and i32:$src1, (i32 65535))) 317 ), 318 (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))> { 319 let SubtargetPredicate = !cast<VOP_Pseudo>(Inst).SubtargetPredicate; 320} 321 322class SDot2Pat<Instruction Inst> : GCNPat < 323 (add (add_oneuse (AMDGPUmul_i24_oneuse (sra i32:$src0, (i32 16)), 324 (sra i32:$src1, (i32 16))), i32:$src2), 325 (AMDGPUmul_i24_oneuse (sext_inreg i32:$src0, i16), 326 (sext_inreg i32:$src1, i16))), 327 (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))> { 328 let SubtargetPredicate = !cast<VOP_Pseudo>(Inst).SubtargetPredicate; 329} 330 331let IsDOT = 1 in { 332let SubtargetPredicate = HasDot2Insts in { 333 334defm V_DOT2_I32_I16 : VOP3PInst<"v_dot2_i32_i16", 335 VOP3P_Profile<VOP_I32_V2I16_V2I16_I32>, int_amdgcn_sdot2, 1>; 336defm V_DOT2_U32_U16 : VOP3PInst<"v_dot2_u32_u16", 337 VOP3P_Profile<VOP_I32_V2I16_V2I16_I32>, int_amdgcn_udot2, 1>; 338 339} // End SubtargetPredicate = HasDot2Insts 340 341let SubtargetPredicate = HasDot7Insts in { 342 343defm V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16", 344 VOP3P_Profile<VOP_F32_V2F16_V2F16_F32, VOP3_REGULAR, /*HasDPP*/ 1>, 345 AMDGPUfdot2, 1/*ExplicitClamp*/>; 346defm V_DOT4_U32_U8 : VOP3PInst<"v_dot4_u32_u8", 347 VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_udot4, 1>; 348defm V_DOT8_U32_U4 : VOP3PInst<"v_dot8_u32_u4", 349 VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_udot8, 1>; 350 351} // End SubtargetPredicate = HasDot7Insts 352 353let SubtargetPredicate = HasDot1Insts in { 354 355defm V_DOT4_I32_I8 : VOP3PInst<"v_dot4_i32_i8", 356 VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_sdot4, 1>; 357defm V_DOT8_I32_I4 : VOP3PInst<"v_dot8_i32_i4", 358 VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_sdot8, 1>; 359 360} // End SubtargetPredicate = HasDot1Insts 361 362let SubtargetPredicate = HasDot8Insts in { 363 364defm V_DOT2_F32_BF16 : VOP3PInst<"v_dot2_f32_bf16", 365 VOP3P_Profile<VOP_F32_V2I16_V2I16_F32, VOP3_REGULAR, /*HasDPP*/ 1>, 366 int_amdgcn_fdot2_f32_bf16, 1>; 367 368} // End SubtargetPredicate = HasDot8Insts 369 370} // End let IsDOT = 1 371 372multiclass VOP3PDOTIUInst <string OpName, SDPatternOperator intrinsic_node> { 373 let IsDOT = 1 in 374 defm NAME : VOP3PInst<OpName, VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, 375 null_frag, 1>; 376 // Dot-iu instructions consider input as signed if imod neg bits are set. Thus 377 // Dot-iu Intrinsics have extra operands and require separate codegen pattern. 378 def : GCNPat < (intrinsic_node (DotIUVOP3PMods i32:$src0_mods), i32:$src0, 379 (DotIUVOP3PMods i32:$src1_mods), i32:$src1, 380 i32:$src2, (i1 timm:$clamp)), 381 (!cast<Instruction>(NAME) $src0_mods, i32:$src0, 382 $src1_mods, i32:$src1, 383 (i32 8), i32:$src2, i1:$clamp) 384 >; 385} 386 387let SubtargetPredicate = HasDot8Insts in { 388defm V_DOT4_I32_IU8 : VOP3PDOTIUInst<"v_dot4_i32_iu8", int_amdgcn_sudot4>; 389defm V_DOT8_I32_IU4 : VOP3PDOTIUInst<"v_dot8_i32_iu4", int_amdgcn_sudot8>; 390} // End SubtargetPredicate = HasDot8Insts 391 392def : UDot2Pat<V_DOT2_U32_U16>; 393def : SDot2Pat<V_DOT2_I32_I16>; 394 395foreach Type = ["U", "I"] in 396 let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT4_"#Type#"32_"#Type#8).SubtargetPredicate in 397 def : GCNPat < 398 !cast<dag>(!foldl((i32 i32:$src2), [0, 1, 2, 3], lhs, y, 399 (add_oneuse lhs, (!cast<PatFrag>("Mul"#Type#"_Elt"#y) i32:$src0, i32:$src1)))), 400 (!cast<VOP3P_Pseudo>("V_DOT4_"#Type#"32_"#Type#8) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>; 401 402foreach Type = ["U", "I"] in 403 let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT8_"#Type#"32_"#Type#4).SubtargetPredicate in 404 def : GCNPat < 405 !cast<dag>(!foldl((add_oneuse i32:$src2, (!cast<PatFrag>("Mul"#Type#"0_4bit") i32:$src0, i32:$src1)), 406 [1, 2, 3, 4, 5, 6, 7], lhs, y, 407 (NonACAdd_oneuse lhs, (!cast<PatFrag>("Mul"#Type#y#"_4bit") i32:$src0, i32:$src1)))), 408 (!cast<VOP3P_Pseudo>("V_DOT8_"#Type#"32_"#Type#4) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>; 409 410// Different variants of dot8 code-gen dag patterns are not generated through table-gen due to a huge increase 411// in the compile time. Directly handle the pattern generated by the FE here. 412foreach Type = ["U", "I"] in 413 let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT8_"#Type#"32_"#Type#4).SubtargetPredicate in 414 def : GCNPat < 415 !cast<dag>(!foldl((add_oneuse i32:$src2, (!cast<PatFrag>("Mul"#Type#"0_4bit") i32:$src0, i32:$src1)), 416 [7, 1, 2, 3, 4, 5, 6], lhs, y, 417 (NonACAdd_oneuse lhs, (!cast<PatFrag>("Mul"#Type#y#"_4bit") i32:$src0, i32:$src1)))), 418 (!cast<VOP3P_Pseudo>("V_DOT8_"#Type#"32_"#Type#4) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>; 419 420def ADst_32 : VOPDstOperand<AGPR_32>; 421def ADst_64 : VOPDstOperand<AReg_64>; 422def ADst_128 : VOPDstOperand<AReg_128>; 423def ADst_256 : VOPDstOperand<AReg_256>; 424def ADst_512 : VOPDstOperand<AReg_512>; 425def ADst_1024 : VOPDstOperand<AReg_1024>; 426def VDst_64 : VOPDstOperand<VReg_64>; 427def VDst_128 : VOPDstOperand<VReg_128>; 428def VDst_256 : VOPDstOperand<VReg_256>; 429def VDst_512 : VOPDstOperand<VReg_512>; 430def VDst_1024 : VOPDstOperand<VReg_1024>; 431 432def VOPProfileAccRead : VOP3P_Profile<VOP_I32_I32, VOP3_MAI> { 433 let Src0RC64 = ARegSrc_32; 434} 435 436def VOPProfileAccWrite : VOP3P_Profile<VOP_I32_I32, VOP3_MAI> { 437 let DstRC = ADst_32; 438 let Src0RC64 = VCSrc_b32; 439} 440 441class VOPProfileMAI<VOPProfile P, RegisterOperand _SrcRC, RegisterOperand _DstRC, 442 RegisterOperand SrcABRC = AVSrc_32> 443 : VOP3P_Profile<P, VOP3_MAI> { 444 let DstRC = _DstRC; 445 let Src0RC64 = SrcABRC; 446 let Src1RC64 = SrcABRC; 447 let Src2RC64 = _SrcRC; 448 let HasOpSel = 0; 449 let HasClamp = 0; 450 let HasIntClamp = 0; 451 let HasOMod = 0; 452 let HasModifiers = 0; 453 let Asm64 = "$vdst, $src0, $src1, $src2$cbsz$abid$blgp"; 454 let AsmVOP3DPPBase = Asm64; 455 let Ins64 = (ins Src0RC64:$src0, Src1RC64:$src1, Src2RC64:$src2, cbsz:$cbsz, abid:$abid, blgp:$blgp); 456 let InsVOP3Base = Ins64; 457 // Dst and SrcC cannot partially overlap if SrcC/Dst is bigger than 4 VGPRs. 458 // We then create two versions of the instruction: with tied dst and src2 459 // and with the earlyclobber flag on the dst. This is stricter than the 460 // actual HW restriction. In particular earlyclobber also affects src0 and 461 // src1 allocation which is not required. 462 bit NoDstOverlap = !gt(DstVT.Size, 128); 463} 464 465class VOPProfileSMFMAC<VOPProfile P, RegisterOperand _DstRC, 466 RegisterOperand _SrcARC, RegisterOperand _SrcBRC> 467 : VOPProfileMAI<P, _DstRC, _DstRC, _SrcARC> { 468 let Src1RC64 = _SrcBRC; 469 let Src2VT = DstVT; 470 let Asm64 = " $vdst, $src0, $src1, $idx$cbsz$abid"; 471 let Outs64 = (outs DstRC:$vdst); 472 let Ins64 = (ins Src0RC64:$src0, Src1RC64:$src1, VRegSrc_32:$idx, cbsz:$cbsz, abid:$abid, Src2RC64:$src2); 473} 474 475def VOPProfileMAI_F32_F32_X4 : VOPProfileMAI<VOP_V4F32_F32_F32_V4F32, AISrc_128_f32, ADst_128>; 476def VOPProfileMAI_F32_F32_X16 : VOPProfileMAI<VOP_V16F32_F32_F32_V16F32, AISrc_512_f32, ADst_512>; 477def VOPProfileMAI_F32_F32_X32 : VOPProfileMAI<VOP_V32F32_F32_F32_V32F32, AISrc_1024_f32, ADst_1024>; 478def VOPProfileMAI_I32_I32_X4 : VOPProfileMAI<VOP_V4I32_I32_I32_V4I32, AISrc_128_b32, ADst_128>; 479def VOPProfileMAI_I32_I32_X16 : VOPProfileMAI<VOP_V16I32_I32_I32_V16I32, AISrc_512_b32, ADst_512>; 480def VOPProfileMAI_I32_I32_X32 : VOPProfileMAI<VOP_V32I32_I32_I32_V32I32, AISrc_1024_b32, ADst_1024>; 481def VOPProfileMAI_F32_V2I16_X4 : VOPProfileMAI<VOP_V4F32_V2I16_V2I16_V4F32, AISrc_128_b32, ADst_128>; 482def VOPProfileMAI_F32_V2I16_X16 : VOPProfileMAI<VOP_V16F32_V2I16_V2I16_V16F32, AISrc_512_b32, ADst_512>; 483def VOPProfileMAI_F32_V2I16_X32 : VOPProfileMAI<VOP_V32F32_V2I16_V2I16_V32F32, AISrc_1024_b32, ADst_1024>; 484def VOPProfileMAI_F32_V4F16_X4 : VOPProfileMAI<VOP_V4F32_V4F16_V4F16_V4F32, AISrc_128_b32, ADst_128, AVSrc_64>; 485def VOPProfileMAI_F32_V4F16_X16 : VOPProfileMAI<VOP_V16F32_V4F16_V4F16_V16F32, AISrc_512_b32, ADst_512, AVSrc_64>; 486def VOPProfileMAI_F32_V4F16_X32 : VOPProfileMAI<VOP_V32F32_V4F16_V4F16_V32F32, AISrc_1024_b32, ADst_1024, AVSrc_64>; 487def VOPProfileMAI_F32_V4I16_X4 : VOPProfileMAI<VOP_V4F32_V4I16_V4I16_V4F32, AISrc_128_b32, ADst_128, AVSrc_64>; 488def VOPProfileMAI_F32_V4I16_X16 : VOPProfileMAI<VOP_V16F32_V4I16_V4I16_V16F32, AISrc_512_b32, ADst_512, AVSrc_64>; 489def VOPProfileMAI_F32_V4I16_X32 : VOPProfileMAI<VOP_V32F32_V4I16_V4I16_V32F32, AISrc_1024_b32, ADst_1024, AVSrc_64>; 490def VOPProfileMAI_F64_16X16X4F64 : VOPProfileMAI<VOP_V4F64_F64_F64_V4F64, AISrc_256_f64, ADst_256, AVSrc_64>; 491def VOPProfileMAI_F64_4X4X4F64 : VOPProfileMAI<VOP_F64_F64_F64_F64, AISrc_64_f64, ADst_64, AVSrc_64>; 492def VOPProfileMAI_I32_I64_X16 : VOPProfileMAI<VOP_V4I32_I64_I64_V4I32, AISrc_128_b32, ADst_128, AVSrc_64>; 493def VOPProfileMAI_I32_I64_X32 : VOPProfileMAI<VOP_V16I32_I64_I64_V16I32, AISrc_512_b32, ADst_512, AVSrc_64>; 494def VOPProfileMAI_F32_V2F32_X16 : VOPProfileMAI<VOP_V4F32_V2F32_V2F32_V4F32, AISrc_128_b32, ADst_128, AVSrc_64>; 495def VOPProfileMAI_F32_V2F32_X32 : VOPProfileMAI<VOP_V16F32_V2F32_V2F32_V16F32, AISrc_512_b32, ADst_512, AVSrc_64>; 496 497def VOPProfileMAI_F32_F32_X4_VCD : VOPProfileMAI<VOP_V4F32_F32_F32_V4F32, VISrc_128_f32, VDst_128>; 498def VOPProfileMAI_F32_F32_X16_VCD : VOPProfileMAI<VOP_V16F32_F32_F32_V16F32, VISrc_512_f32, VDst_512>; 499def VOPProfileMAI_F32_F32_X32_VCD : VOPProfileMAI<VOP_V32F32_F32_F32_V32F32, VISrc_1024_f32, VDst_1024>; 500def VOPProfileMAI_I32_I32_X4_VCD : VOPProfileMAI<VOP_V4I32_I32_I32_V4I32, VISrc_128_b32, VDst_128>; 501def VOPProfileMAI_I32_I32_X16_VCD : VOPProfileMAI<VOP_V16I32_I32_I32_V16I32, VISrc_512_b32, VDst_512>; 502def VOPProfileMAI_I32_I32_X32_VCD : VOPProfileMAI<VOP_V32I32_I32_I32_V32I32, VISrc_1024_b32, VDst_1024>; 503def VOPProfileMAI_F32_V2I16_X4_VCD : VOPProfileMAI<VOP_V4F32_V2I16_V2I16_V4F32, VISrc_128_b32, VDst_128>; 504def VOPProfileMAI_F32_V2I16_X16_VCD : VOPProfileMAI<VOP_V16F32_V2I16_V2I16_V16F32, VISrc_512_b32, VDst_512>; 505def VOPProfileMAI_F32_V2I16_X32_VCD : VOPProfileMAI<VOP_V32F32_V2I16_V2I16_V32F32, VISrc_1024_b32, VDst_1024>; 506def VOPProfileMAI_F32_V4F16_X4_VCD : VOPProfileMAI<VOP_V4F32_V4F16_V4F16_V4F32, VISrc_128_b32, VDst_128, AVSrc_64>; 507def VOPProfileMAI_F32_V4F16_X16_VCD : VOPProfileMAI<VOP_V16F32_V4F16_V4F16_V16F32, VISrc_512_b32, VDst_512, AVSrc_64>; 508def VOPProfileMAI_F32_V4F16_X32_VCD : VOPProfileMAI<VOP_V32F32_V4F16_V4F16_V32F32, VISrc_1024_b32, VDst_1024, AVSrc_64>; 509def VOPProfileMAI_F32_V4I16_X4_VCD : VOPProfileMAI<VOP_V4F32_V4I16_V4I16_V4F32, VISrc_128_b32, VDst_128, AVSrc_64>; 510def VOPProfileMAI_F32_V4I16_X16_VCD : VOPProfileMAI<VOP_V16F32_V4I16_V4I16_V16F32, VISrc_512_b32, VDst_512, AVSrc_64>; 511def VOPProfileMAI_F32_V4I16_X32_VCD : VOPProfileMAI<VOP_V32F32_V4I16_V4I16_V32F32, VISrc_1024_b32, VDst_1024, AVSrc_64>; 512def VOPProfileMAI_F64_16X16X4F64_VCD : VOPProfileMAI<VOP_V4F64_F64_F64_V4F64, VISrc_256_f64, VDst_256, AVSrc_64>; 513def VOPProfileMAI_F64_4X4X4F64_VCD : VOPProfileMAI<VOP_F64_F64_F64_F64, VISrc_64_f64, VDst_64, AVSrc_64>; 514def VOPProfileMAI_I32_I64_X16_VCD : VOPProfileMAI<VOP_V4I32_I64_I64_V4I32, VISrc_128_b32, VDst_128, AVSrc_64>; 515def VOPProfileMAI_I32_I64_X32_VCD : VOPProfileMAI<VOP_V16I32_I64_I64_V16I32, VISrc_512_b32, VDst_512, AVSrc_64>; 516def VOPProfileMAI_F32_V2F32_X16_VCD : VOPProfileMAI<VOP_V4F32_V2F32_V2F32_V4F32, VISrc_128_b32, VDst_128, AVSrc_64>; 517def VOPProfileMAI_F32_V2F32_X32_VCD : VOPProfileMAI<VOP_V16F32_V2F32_V2F32_V16F32, VISrc_512_b32, VDst_512, AVSrc_64>; 518 519def VOPProfileSMFMAC_F32_16X16X32_F16 : VOPProfileSMFMAC<VOP_V4F32_V4F16_V8F16_I32, AVDst_128, AVSrc_64, AVSrc_128>; 520def VOPProfileSMFMAC_F32_32X32X16_F16 : VOPProfileSMFMAC<VOP_V16F32_V4F16_V8F16_I32, AVDst_512, AVSrc_64, AVSrc_128>; 521def VOPProfileSMFMAC_F32_16X16X32_I16 : VOPProfileSMFMAC<VOP_V4F32_V4I16_V8I16_I32, AVDst_128, AVSrc_64, AVSrc_128>; 522def VOPProfileSMFMAC_F32_32X32X16_I16 : VOPProfileSMFMAC<VOP_V16F32_V4I16_V8I16_I32, AVDst_512, AVSrc_64, AVSrc_128>; 523def VOPProfileSMFMAC_I32_16X16X64_I8 : VOPProfileSMFMAC<VOP_V4I32_V2I32_V4I32_I32, AVDst_128, AVSrc_64, AVSrc_128>; 524def VOPProfileSMFMAC_I32_32X32X32_I8 : VOPProfileSMFMAC<VOP_V16I32_V2I32_V4I32_I32, AVDst_512, AVSrc_64, AVSrc_128>; 525 526class MFMATable <bit is_mac, string Name> { 527 bit IsMac = is_mac; 528 string FMAOp = Name; 529} 530 531class MAIFrag<SDPatternOperator Op, code pred> : PatFrag < 532 (ops node:$src0, node:$src1, node:$src2, node:$cbsz, node:$abid, node:$blgp), 533 (Op $src0, $src1, $src2, $cbsz, $abid, $blgp), 534 pred 535>; 536 537let GISelPredicateCode = [{ return MF.getInfo<SIMachineFunctionInfo>()->mayNeedAGPRs(); }] in 538class AgprMAIFrag<SDPatternOperator Op> : 539 MAIFrag<Op, [{ return MF->getInfo<SIMachineFunctionInfo>()->mayNeedAGPRs(); }]>; 540 541let GISelPredicateCode = [{ return !MF.getInfo<SIMachineFunctionInfo>()->mayNeedAGPRs(); }] in 542class VgprMAIFrag<SDPatternOperator Op> : 543 MAIFrag<Op, [{ return !MF->getInfo<SIMachineFunctionInfo>()->mayNeedAGPRs(); }]>; 544 545let Predicates = [HasMAIInsts] in { 546 547let isAsCheapAsAMove = 1, isReMaterializable = 1 in { 548 defm V_ACCVGPR_READ_B32 : VOP3Inst<"v_accvgpr_read_b32", VOPProfileAccRead>; 549 let isMoveImm = 1 in { 550 defm V_ACCVGPR_WRITE_B32 : VOP3Inst<"v_accvgpr_write_b32", VOPProfileAccWrite>; 551 } // End isMoveImm = 1 552} // End isAsCheapAsAMove = 1, isReMaterializable = 1 553 554class MAIInst<string OpName, VOPProfile P, SDPatternOperator node> 555 : VOP3InstBase<OpName, P, node> { 556 Instruction Opcode = !cast<Instruction>(NAME); 557 bit is_dgemm = 0; 558 bit is_gfx940_xdl = 0; 559} 560 561multiclass MAIInst<string OpName, string P, SDPatternOperator node, 562 bit NoDstOverlap = !cast<VOPProfileMAI>("VOPProfileMAI_" # P).NoDstOverlap> { 563 let isConvergent = 1, mayRaiseFPException = 0, ReadsModeReg = 1 in { 564 // FP32 denorm mode is respected, rounding mode is not. Exceptions are not supported. 565 let Constraints = !if(NoDstOverlap, "@earlyclobber $vdst", "") in { 566 def _e64 : MAIInst<OpName, !cast<VOPProfileMAI>("VOPProfileMAI_" # P), 567 !if(NoDstOverlap, null_frag, AgprMAIFrag<node>)>, 568 MFMATable<0, NAME # "_e64">; 569 570 let SubtargetPredicate = isGFX90APlus, Mnemonic = OpName in 571 def _vgprcd_e64 : MAIInst<OpName # "_vgprcd", !cast<VOPProfileMAI>("VOPProfileMAI_" # P # "_VCD"), 572 !if(NoDstOverlap, null_frag, VgprMAIFrag<node>)>, 573 MFMATable<0, NAME # "_vgprcd_e64">; 574 } 575 576 foreach _ = BoolToList<NoDstOverlap>.ret in { 577 let Constraints = !if(NoDstOverlap, "$vdst = $src2", ""), 578 isConvertibleToThreeAddress = NoDstOverlap, 579 Mnemonic = OpName in { 580 def "_mac_e64" : MAIInst<OpName # "_mac", !cast<VOPProfileMAI>("VOPProfileMAI_" # P), AgprMAIFrag<node>>, 581 MFMATable<1, NAME # "_e64">; 582 583 let SubtargetPredicate = isGFX90APlus in 584 def _mac_vgprcd_e64 : MAIInst<OpName # "_mac_vgprcd", !cast<VOPProfileMAI>("VOPProfileMAI_" # P # "_VCD"), 585 VgprMAIFrag<node>>, 586 MFMATable<1, NAME # "_vgprcd_e64">; 587 } 588 } 589 } // End isConvergent = 1, mayRaiseFPException = 0, ReadsModeReg = 1 590} 591 592defm V_MFMA_F32_4X4X1F32 : MAIInst<"v_mfma_f32_4x4x1f32", "F32_F32_X4", int_amdgcn_mfma_f32_4x4x1f32>; 593defm V_MFMA_F32_16X16X1F32 : MAIInst<"v_mfma_f32_16x16x1f32", "F32_F32_X16", int_amdgcn_mfma_f32_16x16x1f32>; 594defm V_MFMA_F32_16X16X4F32 : MAIInst<"v_mfma_f32_16x16x4f32", "F32_F32_X4", int_amdgcn_mfma_f32_16x16x4f32>; 595defm V_MFMA_F32_32X32X1F32 : MAIInst<"v_mfma_f32_32x32x1f32", "F32_F32_X32", int_amdgcn_mfma_f32_32x32x1f32>; 596defm V_MFMA_F32_32X32X2F32 : MAIInst<"v_mfma_f32_32x32x2f32", "F32_F32_X16", int_amdgcn_mfma_f32_32x32x2f32>; 597 598let is_gfx940_xdl = 1 in { 599defm V_MFMA_F32_4X4X4F16 : MAIInst<"v_mfma_f32_4x4x4f16", "F32_V4F16_X4", int_amdgcn_mfma_f32_4x4x4f16>; 600defm V_MFMA_I32_4X4X4I8 : MAIInst<"v_mfma_i32_4x4x4i8", "I32_I32_X4", int_amdgcn_mfma_i32_4x4x4i8>; 601defm V_MFMA_F32_16X16X4F16 : MAIInst<"v_mfma_f32_16x16x4f16", "F32_V4F16_X16", int_amdgcn_mfma_f32_16x16x4f16>; 602defm V_MFMA_F32_16X16X16F16 : MAIInst<"v_mfma_f32_16x16x16f16", "F32_V4F16_X4", int_amdgcn_mfma_f32_16x16x16f16>; 603defm V_MFMA_I32_16X16X4I8 : MAIInst<"v_mfma_i32_16x16x4i8", "I32_I32_X16", int_amdgcn_mfma_i32_16x16x4i8>; 604defm V_MFMA_F32_32X32X4F16 : MAIInst<"v_mfma_f32_32x32x4f16", "F32_V4F16_X32", int_amdgcn_mfma_f32_32x32x4f16>; 605defm V_MFMA_F32_32X32X8F16 : MAIInst<"v_mfma_f32_32x32x8f16", "F32_V4F16_X16", int_amdgcn_mfma_f32_32x32x8f16>; 606defm V_MFMA_I32_32X32X4I8 : MAIInst<"v_mfma_i32_32x32x4i8", "I32_I32_X32", int_amdgcn_mfma_i32_32x32x4i8>; 607} 608 609let Predicates = [isGFX908orGFX90A] in { 610defm V_MFMA_I32_16X16X16I8 : MAIInst<"v_mfma_i32_16x16x16i8", "I32_I32_X4", int_amdgcn_mfma_i32_16x16x16i8>; 611defm V_MFMA_I32_32X32X8I8 : MAIInst<"v_mfma_i32_32x32x8i8", "I32_I32_X16", int_amdgcn_mfma_i32_32x32x8i8>; 612defm V_MFMA_F32_4X4X2BF16 : MAIInst<"v_mfma_f32_4x4x2bf16", "F32_V2I16_X4", int_amdgcn_mfma_f32_4x4x2bf16>; 613defm V_MFMA_F32_16X16X2BF16 : MAIInst<"v_mfma_f32_16x16x2bf16", "F32_V2I16_X16", int_amdgcn_mfma_f32_16x16x2bf16>; 614defm V_MFMA_F32_16X16X8BF16 : MAIInst<"v_mfma_f32_16x16x8bf16", "F32_V2I16_X4", int_amdgcn_mfma_f32_16x16x8bf16>; 615defm V_MFMA_F32_32X32X2BF16 : MAIInst<"v_mfma_f32_32x32x2bf16", "F32_V2I16_X32", int_amdgcn_mfma_f32_32x32x2bf16>; 616defm V_MFMA_F32_32X32X4BF16 : MAIInst<"v_mfma_f32_32x32x4bf16", "F32_V2I16_X16", int_amdgcn_mfma_f32_32x32x4bf16>; 617} 618 619} // End SubtargetPredicate = HasMAIInsts 620 621let Predicates = [isGFX90APlus] in { 622 let is_gfx940_xdl = 1 in { 623 defm V_MFMA_F32_32X32X4BF16_1K : MAIInst<"v_mfma_f32_32x32x4bf16_1k", "F32_V4I16_X32", int_amdgcn_mfma_f32_32x32x4bf16_1k>; 624 defm V_MFMA_F32_16X16X4BF16_1K : MAIInst<"v_mfma_f32_16x16x4bf16_1k", "F32_V4I16_X16", int_amdgcn_mfma_f32_16x16x4bf16_1k>; 625 defm V_MFMA_F32_4X4X4BF16_1K : MAIInst<"v_mfma_f32_4x4x4bf16_1k", "F32_V4I16_X4", int_amdgcn_mfma_f32_4x4x4bf16_1k>; 626 defm V_MFMA_F32_32X32X8BF16_1K : MAIInst<"v_mfma_f32_32x32x8bf16_1k", "F32_V4I16_X16", int_amdgcn_mfma_f32_32x32x8bf16_1k>; 627 defm V_MFMA_F32_16X16X16BF16_1K : MAIInst<"v_mfma_f32_16x16x16bf16_1k", "F32_V4I16_X4", int_amdgcn_mfma_f32_16x16x16bf16_1k>; 628 } 629 630 let is_dgemm = 1 in { 631 defm V_MFMA_F64_16X16X4F64 : MAIInst<"v_mfma_f64_16x16x4f64", "F64_16X16X4F64", int_amdgcn_mfma_f64_16x16x4f64>; 632 defm V_MFMA_F64_4X4X4F64 : MAIInst<"v_mfma_f64_4x4x4f64", "F64_4X4X4F64", int_amdgcn_mfma_f64_4x4x4f64>; 633 } 634} // End Predicates = [isGFX90APlus] 635 636let Predicates = [isGFX940Plus], is_gfx940_xdl = 1 in { 637 defm V_MFMA_I32_32X32X16I8 : MAIInst<"v_mfma_i32_32x32x16i8", "I32_I64_X32", int_amdgcn_mfma_i32_32x32x16_i8>; 638 defm V_MFMA_I32_16X16X32I8 : MAIInst<"v_mfma_i32_16x16x32i8", "I32_I64_X16", int_amdgcn_mfma_i32_16x16x32_i8>; 639 defm V_MFMA_F32_16X16X8XF32 : MAIInst<"v_mfma_f32_16x16x8xf32", "F32_V2F32_X16", int_amdgcn_mfma_f32_16x16x8_xf32>; 640 defm V_MFMA_F32_32X32X4XF32 : MAIInst<"v_mfma_f32_32x32x4xf32", "F32_V2F32_X32", int_amdgcn_mfma_f32_32x32x4_xf32>; 641} // End Predicates = [isGFX940Plus], is_gfx940_xdl = 1 642 643multiclass SMFMACInst<string OpName, string P, SDPatternOperator node> { 644 let Constraints = "$vdst = $src2", DisableEncoding = "$src2", 645 isConvergent = 1, mayRaiseFPException = 0, ReadsModeReg = 1, is_gfx940_xdl = 1 in { 646 def _e64 : MAIInst<OpName, !cast<VOPProfileSMFMAC>("VOPProfileSMFMAC_" # P), node>; 647 } 648} 649 650let SubtargetPredicate = isGFX940Plus in { 651defm V_SMFMAC_F32_16X16X32_F16 : SMFMACInst<"v_smfmac_f32_16x16x32_f16", "F32_16X16X32_F16", int_amdgcn_smfmac_f32_16x16x32_f16>; 652defm V_SMFMAC_F32_32X32X16_F16 : SMFMACInst<"v_smfmac_f32_32x32x16_f16", "F32_32X32X16_F16", int_amdgcn_smfmac_f32_32x32x16_f16>; 653defm V_SMFMAC_F32_16X16X32_BF16 : SMFMACInst<"v_smfmac_f32_16x16x32_bf16", "F32_16X16X32_I16", int_amdgcn_smfmac_f32_16x16x32_bf16>; 654defm V_SMFMAC_F32_32X32X16_BF16 : SMFMACInst<"v_smfmac_f32_32x32x16_bf16", "F32_32X32X16_I16", int_amdgcn_smfmac_f32_32x32x16_bf16>; 655defm V_SMFMAC_I32_16X16X64_I8 : SMFMACInst<"v_smfmac_i32_16x16x64_i8", "I32_16X16X64_I8", int_amdgcn_smfmac_i32_16x16x64_i8>; 656defm V_SMFMAC_I32_32X32X32_I8 : SMFMACInst<"v_smfmac_i32_32x32x32_i8", "I32_32X32X32_I8", int_amdgcn_smfmac_i32_32x32x32_i8>; 657} 658 659def MAIInstInfoTable : GenericTable { 660 let FilterClass = "MAIInst"; 661 let CppTypeName = "MAIInstInfo"; 662 let Fields = [ 663 "Opcode", "is_dgemm", "is_gfx940_xdl" 664 ]; 665 666 let PrimaryKey = ["Opcode"]; 667 let PrimaryKeyName = "getMAIInstInfoHelper"; 668} 669 670let SubtargetPredicate = HasPackedFP32Ops, isCommutable = 1, isReMaterializable = 1 in { 671 defm V_PK_FMA_F32 : VOP3PInst<"v_pk_fma_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fma>; 672 defm V_PK_MUL_F32 : VOP3PInst<"v_pk_mul_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fmul>; 673 defm V_PK_ADD_F32 : VOP3PInst<"v_pk_add_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fadd>; 674 defm V_PK_MOV_B32 : VOP3PInst<"v_pk_mov_b32", VOP3P_Profile<VOP_V2I32_V2I32_V2I32, VOP3_PACKED>>; 675} // End SubtargetPredicate = HasPackedFP32Ops, isCommutable = 1 676 677def : MnemonicAlias<"v_accvgpr_read", "v_accvgpr_read_b32">; 678def : MnemonicAlias<"v_accvgpr_write", "v_accvgpr_write_b32">; 679 680class VOPProfileWMMA<VOPProfile P, string Suffix, RegisterOperand _Src01RC64, bit _HasClamp, bit _HasOpSel> : VOP3P_Profile<P> { 681 let DstRC = !if(!eq(Suffix, "_w32"), VDst_256, VDst_128); 682 let Src0RC64 = _Src01RC64; 683 let Src1RC64 = _Src01RC64; 684 let Src2RC64 = !if(!eq(Suffix, "_w32"), VISrc_256_f64, VISrc_128_f32); 685 let HasClamp = _HasClamp; 686 let HasOpSel = _HasOpSel; 687 let IsPacked = 1; 688 let IsWMMA = 1; 689} 690 691def VOP_V8F32_V16F16_V16F16_V8F32 : VOPProfile <[v8f32, v16f16, v16f16, v8f32]>; 692def VOP_V8F32_V16I16_V16I16_V8F32 : VOPProfile <[v8f32, v16i16, v16i16, v8f32]>; 693def VOP_V16F16_V16F16_V16F16_V16F16 : VOPProfile <[v16f16, v16f16, v16f16, v16f16]>; 694def VOP_V16I16_V16I16_V16I16_V16I16 : VOPProfile <[v16i16, v16i16, v16i16, v16i16]>; 695def VOP_V8I32_V4I32_V4I32_V8I32 : VOPProfile <[v8i32, v4i32, v4i32, v8i32]>; 696def VOP_V8I32_V2I32_V2I32_V8I32 : VOPProfile <[v8i32, v2i32, v2i32, v8i32]>; 697 698def VOP_V4F32_V16F16_V16F16_V4F32 : VOPProfile <[v4f32, v16f16, v16f16, v4f32]>; 699def VOP_V4F32_V16I16_V16I16_V4F32 : VOPProfile <[v4f32, v16i16, v16i16, v4f32]>; 700def VOP_V8F16_V16F16_V16F16_V8F16 : VOPProfile <[v8f16, v16f16, v16f16, v8f16]>; 701def VOP_V8I16_V16I16_V16I16_V8I16 : VOPProfile <[v8i16, v16i16, v16i16, v8i16]>; 702def VOP_V4I32_V4I32_V4I32_V4I32 : VOPProfile <[v4i32, v4i32, v4i32, v4i32]>; 703def VOP_V4I32_V2I32_V2I32_V4I32 : VOPProfile <[v4i32, v2i32, v2i32, v4i32]>; 704 705 706class WMMAType <bits<2> val> { 707 bit hasClamp = val{0}; 708 bit hasOpsel = val{1}; 709} 710 711def WMMARegular : WMMAType<0b00>; 712def WMMAUIClamp : WMMAType<0b01>; 713def WMMAOpSel : WMMAType<0b10>; 714 715class WMMARegularPat<Instruction Inst, SDPatternOperator node, VOPProfile P> : 716 GCNPat < (P.DstVT (node 717 (P.Src0VT (VOP3PMods P.Src0VT:$src0, i32:$src0_modifiers)), 718 (P.Src1VT (VOP3PMods P.Src1VT:$src1, i32:$src1_modifiers)), 719 (P.Src2VT (VOP3PMods P.Src2VT:$src2, i32:$src2_modifiers)) 720 )), 721 (P.DstVT (Inst i32:$src0_modifiers, P.Src0VT:$src0, i32:$src1_modifiers, P.Src1VT:$src1, $src2_modifiers, P.Src2VT:$src2)) 722>; 723 724class WMMAOpSelPat<Instruction Inst, SDPatternOperator node, VOPProfile P> : 725 GCNPat < (P.DstVT (node 726 (P.Src0VT P.Src0VT:$src0), 727 (P.Src1VT P.Src1VT:$src1), 728 (P.Src2VT P.Src2VT:$src2), (WMMAOpSelVOP3PMods i32:$src2_modifiers) 729 )), 730 (P.DstVT (Inst (i32 8), P.Src0VT:$src0, (i32 8), P.Src1VT:$src1, i32:$src2_modifiers, P.Src2VT:$src2)) 731>; 732 733class WMMAUIClampPat<Instruction Inst, SDPatternOperator node, VOPProfile P> : 734 GCNPat < (P.DstVT (node 735 (DotIUVOP3PMods i32:$src0_modifiers), (P.Src0VT P.Src0VT:$src0), 736 (DotIUVOP3PMods i32:$src1_modifiers), (P.Src1VT P.Src1VT:$src1), 737 (P.Src2VT P.Src2VT:$src2), (i1 timm:$clamp) 738 )), 739 (P.DstVT (Inst i32:$src0_modifiers, P.Src0VT:$src0, i32:$src1_modifiers, P.Src1VT:$src1, (i32 8), P.Src2VT:$src2, i1:$clamp)) 740>; 741 742class WMMAOpcodeMapping<Instruction TwoAddr, Instruction ThreeAddr> { 743 Instruction Opcode2Addr = TwoAddr; 744 Instruction Opcode3Addr = ThreeAddr; 745 Predicate WaveSizePredicate; 746} 747 748def WMMAOpcode : GenericEnum { 749 let FilterClass = "VOP3P_Pseudo"; 750} 751 752class WMMAMappingTable : GenericTable { 753 let FilterClass = "WMMAOpcodeMapping"; 754 let CppTypeName = "WMMAOpcodeMappingInfo"; 755 let Fields = ["Opcode2Addr", "Opcode3Addr"]; 756 string TypeOf_Opcode2Addr = "WMMAOpcode"; 757 string TypeOf_Opcode3Addr = "WMMAOpcode"; 758} 759 760def WMMAOpcode2AddrMappingTable : WMMAMappingTable { 761 let PrimaryKey = ["Opcode2Addr"]; 762 let PrimaryKeyName = "getWMMAMappingInfoFrom2AddrOpcode"; 763} 764 765def WMMAOpcode3AddrMappingTable : WMMAMappingTable { 766 let PrimaryKey = ["Opcode3Addr"]; 767 let PrimaryKeyName = "getWMMAMappingInfoFrom3AddrOpcode"; 768} 769 770// The WMMA instruction has extra constraints: 771// Matrices A and B cannot overlap with D. C cannot partially overlap with D, 772// but it is OK for them to be the same (which is a typical case). 773// 774// We implement it as follows: 775// 1) Map the intrinsic to the pseudo where D is tied to C ($vdst = $src2). 776// 2) The pass twoaddressinstruction checks if src2 is live and if that is the case 777// it converts the default pseudo to the pseudo where src2 is not the same as vdst. 778// 3) @earlyclobber on the destination satisfies the constraint during RA. 779 780multiclass WMMAInst<string Suffix, string Instr, VOPProfile P, SDPatternOperator node = null_frag, RegisterOperand _Src01RC64 = VRegSrc_256, WMMAType Type> { 781 782 defvar WMMAConstraints2Addr = "@earlyclobber $vdst,$vdst = $src2"; 783 defvar WMMAConstraints3Addr = "@earlyclobber $vdst"; 784 785 defvar WMMAProfile = VOPProfileWMMA<P, Suffix, _Src01RC64, Type.hasClamp, Type.hasOpsel>; 786 if !eq(Suffix, "_w32") then { 787 let Mnemonic = Instr, mayRaiseFPException = 0, ReadsModeReg = 0 in { 788 let Constraints = WMMAConstraints2Addr, isConvertibleToThreeAddress = 1 in { 789 def _twoaddr_w32 : VOP3P_Pseudo<Instr # Suffix, WMMAProfile>; 790 } 791 let Constraints = WMMAConstraints3Addr, SchedRW = [Write32Bit, Write32Bit] in { 792 def _threeaddr_w32 : VOP3P_Pseudo<Instr # Suffix, WMMAProfile>; 793 } 794 } 795 def : WMMAOpcodeMapping<!cast<Instruction>(NAME # _twoaddr_w32), 796 !cast<Instruction>(NAME # _threeaddr_w32)>; 797 } else if !eq(Suffix, "_w64") then { 798 let Mnemonic = Instr, mayRaiseFPException = 0, ReadsModeReg = 0 in { 799 let Constraints = WMMAConstraints2Addr, isConvertibleToThreeAddress = 1 in { 800 def _twoaddr_w64 : VOP3P_Pseudo<Instr # Suffix, WMMAProfile>; 801 } 802 let Constraints = WMMAConstraints3Addr, SchedRW = [Write32Bit, Write32Bit] in { 803 def _threeaddr_w64 : VOP3P_Pseudo<Instr # Suffix, WMMAProfile>; 804 } 805 } 806 def : WMMAOpcodeMapping<!cast<Instruction>(NAME # _twoaddr_w64), 807 !cast<Instruction>(NAME # _threeaddr_w64)>; 808 } 809 810 if !eq(Type, WMMAOpSel) then { 811 def : WMMAOpSelPat<!cast<Instruction>(NAME # _twoaddr # Suffix), node, P>; 812 } else if !eq(Type, WMMAUIClamp) then { 813 def : WMMAUIClampPat<!cast<Instruction>(NAME # _twoaddr # Suffix), node, P>; 814 } else { 815 def : WMMARegularPat<!cast<Instruction>(NAME # _twoaddr # Suffix), node, P>; 816 } 817} 818 819 820let WaveSizePredicate = isWave32 in { 821 defm V_WMMA_F32_16X16X16_F16 : WMMAInst<"_w32", "v_wmma_f32_16x16x16_f16", VOP_V8F32_V16F16_V16F16_V8F32, int_amdgcn_wmma_f32_16x16x16_f16, VRegSrc_256, WMMARegular>; 822 defm V_WMMA_F32_16X16X16_BF16 : WMMAInst<"_w32", "v_wmma_f32_16x16x16_bf16", VOP_V8F32_V16I16_V16I16_V8F32, int_amdgcn_wmma_f32_16x16x16_bf16, VRegSrc_256, WMMARegular>; 823 defm V_WMMA_F16_16X16X16_F16 : WMMAInst<"_w32", "v_wmma_f16_16x16x16_f16", VOP_V16F16_V16F16_V16F16_V16F16, int_amdgcn_wmma_f16_16x16x16_f16, VRegSrc_256, WMMAOpSel>; 824 defm V_WMMA_BF16_16X16X16_BF16 : WMMAInst<"_w32", "v_wmma_bf16_16x16x16_bf16", VOP_V16I16_V16I16_V16I16_V16I16, int_amdgcn_wmma_bf16_16x16x16_bf16, VRegSrc_256, WMMAOpSel>; 825 defm V_WMMA_I32_16X16X16_IU8 : WMMAInst<"_w32", "v_wmma_i32_16x16x16_iu8", VOP_V8I32_V4I32_V4I32_V8I32, int_amdgcn_wmma_i32_16x16x16_iu8, VRegSrc_128, WMMAUIClamp>; 826 defm V_WMMA_I32_16X16X16_IU4 : WMMAInst<"_w32", "v_wmma_i32_16x16x16_iu4", VOP_V8I32_V2I32_V2I32_V8I32, int_amdgcn_wmma_i32_16x16x16_iu4, VRegSrc_64, WMMAUIClamp>; 827} 828 829let WaveSizePredicate = isWave64 in { 830 defm V_WMMA_F32_16X16X16_F16 : WMMAInst<"_w64", "v_wmma_f32_16x16x16_f16", VOP_V4F32_V16F16_V16F16_V4F32, int_amdgcn_wmma_f32_16x16x16_f16, VRegSrc_256, WMMARegular>; 831 defm V_WMMA_F32_16X16X16_BF16 : WMMAInst<"_w64", "v_wmma_f32_16x16x16_bf16", VOP_V4F32_V16I16_V16I16_V4F32, int_amdgcn_wmma_f32_16x16x16_bf16, VRegSrc_256, WMMARegular>; 832 defm V_WMMA_F16_16X16X16_F16 : WMMAInst<"_w64", "v_wmma_f16_16x16x16_f16", VOP_V8F16_V16F16_V16F16_V8F16, int_amdgcn_wmma_f16_16x16x16_f16, VRegSrc_256, WMMAOpSel>; 833 defm V_WMMA_BF16_16X16X16_BF16 : WMMAInst<"_w64", "v_wmma_bf16_16x16x16_bf16", VOP_V8I16_V16I16_V16I16_V8I16, int_amdgcn_wmma_bf16_16x16x16_bf16, VRegSrc_256, WMMAOpSel>; 834 defm V_WMMA_I32_16X16X16_IU8 : WMMAInst<"_w64", "v_wmma_i32_16x16x16_iu8", VOP_V4I32_V4I32_V4I32_V4I32, int_amdgcn_wmma_i32_16x16x16_iu8, VRegSrc_128, WMMAUIClamp>; 835 defm V_WMMA_I32_16X16X16_IU4 : WMMAInst<"_w64", "v_wmma_i32_16x16x16_iu4", VOP_V4I32_V2I32_V2I32_V4I32, int_amdgcn_wmma_i32_16x16x16_iu4, VRegSrc_64, WMMAUIClamp>; 836 837} 838 839//===----------------------------------------------------------------------===// 840// Begin Real Encodings 841//===----------------------------------------------------------------------===// 842 843class VOP3P_DPP16<bits<7> op, VOP_DPP_Pseudo ps, int subtarget, 844 string opName = ps.OpName> 845 : VOP3P_DPP<op, opName, ps.Pfl, 1>, SIMCInstr<ps.PseudoInstr, subtarget> { 846 let hasSideEffects = ps.hasSideEffects; 847 let Defs = ps.Defs; 848 let SchedRW = ps.SchedRW; 849 let Uses = ps.Uses; 850 let AssemblerPredicate = HasDPP16; 851 let SubtargetPredicate = HasDPP16; 852 let OtherPredicates = ps.OtherPredicates; 853} 854 855class VOP3P_DPP8_Base<bits<7> op, VOP_Pseudo ps, string opName = ps.OpName> 856 : VOP3P_DPP8<op, opName, ps.Pfl> { 857 let hasSideEffects = ps.hasSideEffects; 858 let Defs = ps.Defs; 859 let SchedRW = ps.SchedRW; 860 let Uses = ps.Uses; 861 let OtherPredicates = ps.OtherPredicates; 862} 863 864//===----------------------------------------------------------------------===// 865// GFX11. 866//===----------------------------------------------------------------------===// 867 868let AssemblerPredicate = isGFX11Plus, 869 DecoderNamespace = "GFX11" in { 870 871 multiclass VOP3P_Real_gfx11<bits<7> op, string backing_ps_name = NAME, 872 string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> { 873 def _gfx11 : VOP3P_Real<!cast<VOP3P_Pseudo>(backing_ps_name), 874 SIEncodingFamily.GFX11, asmName>, 875 VOP3Pe_gfx11<op, !cast<VOP3P_Pseudo>(backing_ps_name).Pfl>; 876 } 877 878 multiclass VOP3P_Real_dpp_gfx11<bits<7> op, string backing_ps_name = NAME, 879 string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> { 880 defvar ps = !cast<VOP3P_Pseudo>(backing_ps_name); 881 def _dpp_gfx11 882 : VOP3P_DPP16<op, !cast<VOP_DPP_Pseudo>(backing_ps_name #"_dpp"), 883 SIEncodingFamily.GFX11> { 884 let AsmString = asmName #ps.Pfl.AsmVOP3DPP16; 885 let DecoderNamespace = "DPPGFX11"; 886 } 887 } 888 889 multiclass VOP3P_Real_dpp8_gfx11<bits<7> op, string backing_ps_name = NAME, 890 string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> { 891 defvar ps = !cast<VOP3P_Pseudo>(backing_ps_name); 892 def _dpp8_gfx11 : VOP3P_DPP8_Base<op, ps> { 893 let AsmString = asmName #ps.Pfl.AsmVOP3DPP8; 894 let DecoderNamespace = "DPP8GFX11"; 895 } 896 } 897 898 multiclass VOP3P_Realtriple_gfx11<bits<7> op, string backing_ps_name = NAME, 899 string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> 900 : VOP3P_Real_gfx11<op, backing_ps_name, asmName>, 901 VOP3P_Real_dpp_gfx11<op, backing_ps_name, asmName>, 902 VOP3P_Real_dpp8_gfx11<op, backing_ps_name, asmName>; 903} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" 904 905defm V_DOT4_I32_IU8 : VOP3P_Real_gfx11 <0x16>; 906defm V_DOT8_I32_IU4 : VOP3P_Real_gfx11 <0x18>; 907defm V_DOT2_F32_BF16 : VOP3P_Real_gfx11 <0x1a>; 908 909multiclass VOP3P_Real_WMMA <bits<7> op> { 910 let WaveSizePredicate = isWave32, DecoderNamespace = "GFX11" in { 911 defm _twoaddr_w32 : VOP3P_Real_gfx11 <op>; 912 } 913 let WaveSizePredicate = isWave64, DecoderNamespace = "WMMAGFX11" in { 914 defm _twoaddr_w64 : VOP3P_Real_gfx11 <op>; 915 } 916} 917 918defm V_WMMA_F32_16X16X16_F16 : VOP3P_Real_WMMA <0x040>; 919defm V_WMMA_F32_16X16X16_BF16 : VOP3P_Real_WMMA <0x041>; 920defm V_WMMA_F16_16X16X16_F16 : VOP3P_Real_WMMA <0x042>; 921defm V_WMMA_BF16_16X16X16_BF16 : VOP3P_Real_WMMA <0x043>; 922defm V_WMMA_I32_16X16X16_IU8 : VOP3P_Real_WMMA <0x044>; 923defm V_WMMA_I32_16X16X16_IU4 : VOP3P_Real_WMMA <0x045>; 924 925//===----------------------------------------------------------------------===// 926// GFX8 (VI) 927//===----------------------------------------------------------------------===// 928 929multiclass VOP3P_Real_vi<bits<7> op> { 930 def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>, 931 VOP3Pe <op, !cast<VOP3_Pseudo>(NAME).Pfl> { 932 let AssemblerPredicate = HasVOP3PInsts; 933 let DecoderNamespace = "GFX8"; 934 let VOP3P = 1; 935 } 936} 937 938multiclass VOP3P_Real_MAI<bits<7> op> { 939 def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>, 940 VOP3Pe_MAI <op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl, ?> { 941 let AssemblerPredicate = HasMAIInsts; 942 let DecoderNamespace = "GFX8"; 943 let Inst{14} = ?; // op_sel_hi(2) 944 let Inst{59} = ?; // op_sel_hi(0) 945 let Inst{60} = ?; // op_sel_hi(1) 946 } 947} 948 949let Constraints = "" in { 950multiclass VOP3P_Real_MFMA_gfx90a<bits<7> op> { 951 let SubtargetPredicate = isGFX90AOnly, 952 AssemblerPredicate = isGFX90AOnly, DecoderNamespace = "GFX90A" in { 953 def _gfx90a_acd : VOP3P_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX90A>, 954 VOP3Pe_MAI <op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl, 1>; 955 956 def _gfx90a_vcd : VOP3P_Real<!cast<VOP3_Pseudo>(NAME # "_vgprcd" # "_e64"), SIEncodingFamily.GFX90A>, 957 VOP3Pe_MAI <op, !cast<VOP3_Pseudo>(NAME # "_vgprcd" # "_e64").Pfl, 0>; 958 } // End AssemblerPredicate = isGFX90AOnly, DecoderNamespace = "GFX90A" 959} 960} 961 962multiclass VOP3P_Real_MFMA_gfx940_aliases<string NameFrom, string NameTo, string Op, 963 VOP3_Pseudo PS_ACD = !cast<VOP3_Pseudo>(Op # "_e64"), 964 VOP3_Pseudo PS_VCD = !cast<VOP3_Pseudo>(Op # "_vgprcd" # "_e64"), 965 VOPProfile Pfl_ACD = PS_ACD.Pfl, 966 VOPProfile Pfl_VCD = PS_VCD.Pfl> { 967 let Predicates = [isGFX940Plus] in { 968 foreach _ = BoolToList<!ne(NameFrom, NameTo)>.ret in { 969 def : InstAlias <NameTo # " " # PS_ACD.AsmOperands, 970 (!cast<VOP3P_Real>(Op # "_gfx940_acd") Pfl_ACD.DstRC:$vdst, 971 Pfl_ACD.Src0RC64:$src0, Pfl_ACD.Src1RC64:$src1, Pfl_ACD.Src2RC64:$src2, 972 cbsz:$cbsz, abid:$abid, blgp:$blgp)>, PredicateControl; 973 def : InstAlias <NameTo # " " # PS_VCD.AsmOperands, 974 (!cast<VOP3P_Real>(Op # "_gfx940_vcd") Pfl_VCD.DstRC:$vdst, 975 Pfl_VCD.Src0RC64:$src0, Pfl_VCD.Src1RC64:$src1, Pfl_VCD.Src2RC64:$src2, 976 cbsz:$cbsz, abid:$abid, blgp:$blgp)>, PredicateControl; 977 } 978 } // End Predicates = [isGFX940Plus] 979} 980 981multiclass VOP3P_Real_MFMA_gfx940<bits<7> op, string Name = !cast<VOP3_Pseudo>(NAME#"_e64").Mnemonic, 982 VOP3_Pseudo PS_ACD = !cast<VOP3_Pseudo>(NAME # "_e64"), 983 VOP3_Pseudo PS_VCD = !cast<VOP3_Pseudo>(NAME # "_vgprcd" # "_e64")> { 984 let SubtargetPredicate = isGFX940Plus, 985 AssemblerPredicate = isGFX940Plus, DecoderNamespace = "GFX9", 986 AsmString = Name # PS_ACD.AsmOperands, Constraints = "" in { 987 def _gfx940_acd : VOP3P_Real<PS_ACD, SIEncodingFamily.GFX940>, 988 VOP3Pe_MAI <op, PS_ACD.Pfl, 1>; 989 990 def _gfx940_vcd : VOP3P_Real<PS_VCD, SIEncodingFamily.GFX940>, 991 VOP3Pe_MAI <op, PS_VCD.Pfl, 0>; 992 } // End AssemblerPredicate = isGFX940Plus, DecoderNamespace = "GFX9" 993 994 defm : VOP3P_Real_MFMA_gfx940_aliases<Name, PS_ACD.Mnemonic, NAME>; 995 996 foreach _ = BoolToList<!ne(!subst("_1k", "", PS_ACD.Mnemonic), PS_ACD.Mnemonic)>.ret in 997 defm : VOP3P_Real_MFMA_gfx940_aliases<Name, !subst("_1k", "", PS_ACD.Mnemonic), NAME>; 998} 999 1000multiclass VOP3P_Real_MFMA<bits<7> op, string GFX940Name = !cast<VOP3_Pseudo>(NAME#"_e64").Mnemonic> : 1001 VOP3P_Real_MFMA_gfx90a <op>, 1002 VOP3P_Real_MFMA_gfx940 <op, GFX940Name> { 1003 def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>, 1004 VOP3Pe_MAI <op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl, ?> { 1005 let AssemblerPredicate = HasMAIInsts; 1006 let DecoderNamespace = "GFX8"; 1007 let Constraints = ""; 1008 } 1009} 1010 1011multiclass VOP3P_Real_SMFMAC<bits<7> op, string alias> { 1012 def _gfx940 : VOP3P_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>, 1013 VOP3Pe_SMFMAC <op> { 1014 let AssemblerPredicate = isGFX940Plus; 1015 let DecoderNamespace = "GFX8"; 1016 } 1017 def : MnemonicAlias<alias, !cast<VOP3_Pseudo>(NAME#"_e64").Mnemonic>; 1018} 1019 1020defm V_PK_MAD_I16 : VOP3P_Real_vi <0x00>; 1021defm V_PK_MUL_LO_U16 : VOP3P_Real_vi <0x01>; 1022defm V_PK_ADD_I16 : VOP3P_Real_vi <0x02>; 1023defm V_PK_SUB_I16 : VOP3P_Real_vi <0x03>; 1024defm V_PK_LSHLREV_B16 : VOP3P_Real_vi <0x04>; 1025defm V_PK_LSHRREV_B16 : VOP3P_Real_vi <0x05>; 1026defm V_PK_ASHRREV_I16 : VOP3P_Real_vi <0x06>; 1027defm V_PK_MAX_I16 : VOP3P_Real_vi <0x07>; 1028defm V_PK_MIN_I16 : VOP3P_Real_vi <0x08>; 1029defm V_PK_MAD_U16 : VOP3P_Real_vi <0x09>; 1030 1031defm V_PK_ADD_U16 : VOP3P_Real_vi <0x0a>; 1032defm V_PK_SUB_U16 : VOP3P_Real_vi <0x0b>; 1033defm V_PK_MAX_U16 : VOP3P_Real_vi <0x0c>; 1034defm V_PK_MIN_U16 : VOP3P_Real_vi <0x0d>; 1035defm V_PK_FMA_F16 : VOP3P_Real_vi <0x0e>; 1036defm V_PK_ADD_F16 : VOP3P_Real_vi <0x0f>; 1037defm V_PK_MUL_F16 : VOP3P_Real_vi <0x10>; 1038defm V_PK_MIN_F16 : VOP3P_Real_vi <0x11>; 1039defm V_PK_MAX_F16 : VOP3P_Real_vi <0x12>; 1040 1041 1042let SubtargetPredicate = HasMadMixInsts in { 1043defm V_MAD_MIX_F32 : VOP3P_Real_vi <0x20>; 1044defm V_MAD_MIXLO_F16 : VOP3P_Real_vi <0x21>; 1045defm V_MAD_MIXHI_F16 : VOP3P_Real_vi <0x22>; 1046} 1047 1048let SubtargetPredicate = HasFmaMixInsts in { 1049let DecoderNamespace = "GFX9_DL" in { 1050// The mad_mix instructions were renamed and their behaviors changed, 1051// but the opcode stayed the same so we need to put these in a 1052// different DecoderNamespace to avoid the ambiguity. 1053defm V_FMA_MIX_F32 : VOP3P_Real_vi <0x20>; 1054defm V_FMA_MIXLO_F16 : VOP3P_Real_vi <0x21>; 1055defm V_FMA_MIXHI_F16 : VOP3P_Real_vi <0x22>; 1056} 1057} 1058 1059 1060let SubtargetPredicate = HasDot2Insts in { 1061 1062defm V_DOT2_I32_I16 : VOP3P_Real_vi <0x26>; 1063defm V_DOT2_U32_U16 : VOP3P_Real_vi <0x27>; 1064 1065} // End SubtargetPredicate = HasDot2Insts 1066 1067let SubtargetPredicate = HasDot7Insts in { 1068 1069defm V_DOT2_F32_F16 : VOP3P_Real_vi <0x23>; 1070defm V_DOT4_U32_U8 : VOP3P_Real_vi <0x29>; 1071defm V_DOT8_U32_U4 : VOP3P_Real_vi <0x2b>; 1072 1073} // End SubtargetPredicate = HasDot7Insts 1074 1075let SubtargetPredicate = HasDot1Insts in { 1076 1077defm V_DOT4_I32_I8 : VOP3P_Real_vi <0x28>; 1078defm V_DOT8_I32_I4 : VOP3P_Real_vi <0x2a>; 1079 1080} // End SubtargetPredicate = HasDot1Insts 1081 1082let SubtargetPredicate = HasMAIInsts in { 1083 1084defm V_ACCVGPR_READ_B32 : VOP3P_Real_MAI <0x58>; 1085defm V_ACCVGPR_WRITE_B32 : VOP3P_Real_MAI <0x59>; 1086defm V_MFMA_F32_32X32X1F32 : VOP3P_Real_MFMA <0x40, "v_mfma_f32_32x32x1_2b_f32">; 1087defm V_MFMA_F32_16X16X1F32 : VOP3P_Real_MFMA <0x41, "v_mfma_f32_16x16x1_4b_f32">; 1088defm V_MFMA_F32_4X4X1F32 : VOP3P_Real_MFMA <0x42, "v_mfma_f32_4x4x1_16b_f32">; 1089defm V_MFMA_F32_32X32X2F32 : VOP3P_Real_MFMA <0x44, "v_mfma_f32_32x32x2_f32">; 1090defm V_MFMA_F32_16X16X4F32 : VOP3P_Real_MFMA <0x45, "v_mfma_f32_16x16x4_f32">; 1091defm V_MFMA_F32_32X32X4F16 : VOP3P_Real_MFMA <0x48, "v_mfma_f32_32x32x4_2b_f16">; 1092defm V_MFMA_F32_16X16X4F16 : VOP3P_Real_MFMA <0x49, "v_mfma_f32_16x16x4_4b_f16">; 1093defm V_MFMA_F32_4X4X4F16 : VOP3P_Real_MFMA <0x4a, "v_mfma_f32_4x4x4_16b_f16">; 1094defm V_MFMA_F32_32X32X8F16 : VOP3P_Real_MFMA <0x4c, "v_mfma_f32_32x32x8_f16">; 1095defm V_MFMA_F32_16X16X16F16 : VOP3P_Real_MFMA <0x4d, "v_mfma_f32_16x16x16_f16">; 1096defm V_MFMA_I32_32X32X4I8 : VOP3P_Real_MFMA <0x50, "v_mfma_i32_32x32x4_2b_i8">; 1097defm V_MFMA_I32_16X16X4I8 : VOP3P_Real_MFMA <0x51, "v_mfma_i32_16x16x4_4b_i8">; 1098defm V_MFMA_I32_4X4X4I8 : VOP3P_Real_MFMA <0x52, "v_mfma_i32_4x4x4_16b_i8">; 1099 1100let SubtargetPredicate = isGFX908orGFX90A in { 1101defm V_MFMA_I32_16X16X16I8 : VOP3P_Real_MFMA <0x55>; 1102defm V_MFMA_I32_32X32X8I8 : VOP3P_Real_MFMA <0x54>; 1103defm V_MFMA_F32_32X32X2BF16 : VOP3P_Real_MFMA <0x68>; 1104defm V_MFMA_F32_16X16X2BF16 : VOP3P_Real_MFMA <0x69>; 1105defm V_MFMA_F32_4X4X2BF16 : VOP3P_Real_MFMA <0x6b>; 1106defm V_MFMA_F32_32X32X4BF16 : VOP3P_Real_MFMA <0x6c>; 1107defm V_MFMA_F32_16X16X8BF16 : VOP3P_Real_MFMA <0x6d>; 1108} 1109 1110} // End SubtargetPredicate = HasMAIInsts 1111 1112defm V_MFMA_F32_32X32X4BF16_1K : VOP3P_Real_MFMA_gfx90a <0x63>; 1113defm V_MFMA_F32_16X16X4BF16_1K : VOP3P_Real_MFMA_gfx90a <0x64>; 1114defm V_MFMA_F32_4X4X4BF16_1K : VOP3P_Real_MFMA_gfx90a <0x65>; 1115defm V_MFMA_F32_32X32X8BF16_1K : VOP3P_Real_MFMA_gfx90a <0x66>; 1116defm V_MFMA_F32_16X16X16BF16_1K : VOP3P_Real_MFMA_gfx90a <0x67>; 1117defm V_MFMA_F64_16X16X4F64 : VOP3P_Real_MFMA_gfx90a <0x6e>; 1118defm V_MFMA_F64_4X4X4F64 : VOP3P_Real_MFMA_gfx90a <0x6f>; 1119 1120defm V_MFMA_I32_32X32X16I8 : VOP3P_Real_MFMA_gfx940 <0x56, "v_mfma_i32_32x32x16_i8">; 1121defm V_MFMA_I32_16X16X32I8 : VOP3P_Real_MFMA_gfx940 <0x57, "v_mfma_i32_16x16x32_i8">; 1122defm V_MFMA_F32_16X16X8XF32 : VOP3P_Real_MFMA_gfx940 <0x3e, "v_mfma_f32_16x16x8_xf32">; 1123defm V_MFMA_F32_32X32X4XF32 : VOP3P_Real_MFMA_gfx940 <0x3f, "v_mfma_f32_32x32x4_xf32">; 1124 1125defm V_MFMA_F32_32X32X4BF16_1K : VOP3P_Real_MFMA_gfx940 <0x5d, "v_mfma_f32_32x32x4_2b_bf16">; 1126defm V_MFMA_F32_16X16X4BF16_1K : VOP3P_Real_MFMA_gfx940 <0x5e, "v_mfma_f32_16x16x4_4b_bf16">; 1127defm V_MFMA_F32_4X4X4BF16_1K : VOP3P_Real_MFMA_gfx940 <0x5f, "v_mfma_f32_4x4x4_16b_bf16">; 1128defm V_MFMA_F32_32X32X8BF16_1K : VOP3P_Real_MFMA_gfx940 <0x60, "v_mfma_f32_32x32x8_bf16">; 1129defm V_MFMA_F32_16X16X16BF16_1K : VOP3P_Real_MFMA_gfx940 <0x61, "v_mfma_f32_16x16x16_bf16">; 1130 1131defm V_MFMA_F64_16X16X4F64 : VOP3P_Real_MFMA_gfx940 <0x6e, "v_mfma_f64_16x16x4_f64">; 1132defm V_MFMA_F64_4X4X4F64 : VOP3P_Real_MFMA_gfx940 <0x6f, "v_mfma_f64_4x4x4_4b_f64">; 1133 1134defm V_SMFMAC_F32_16X16X32_F16 : VOP3P_Real_SMFMAC <0x62, "v_smfmac_f32_16x16x32f16">; 1135defm V_SMFMAC_F32_32X32X16_F16 : VOP3P_Real_SMFMAC <0x64, "v_smfmac_f32_32x32x16f16">; 1136defm V_SMFMAC_F32_16X16X32_BF16 : VOP3P_Real_SMFMAC <0x66, "v_smfmac_f32_16x16x32bf16">; 1137defm V_SMFMAC_F32_32X32X16_BF16 : VOP3P_Real_SMFMAC <0x68, "v_smfmac_f32_32x32x16bf16">; 1138defm V_SMFMAC_I32_16X16X64_I8 : VOP3P_Real_SMFMAC <0x6a, "v_smfmac_i32_16x16x64i8">; 1139defm V_SMFMAC_I32_32X32X32_I8 : VOP3P_Real_SMFMAC <0x6c, "v_smfmac_i32_32x32x32i8">; 1140 1141let SubtargetPredicate = HasPackedFP32Ops in { 1142 defm V_PK_FMA_F32 : VOP3P_Real_vi <0x30>; 1143 defm V_PK_MUL_F32 : VOP3P_Real_vi <0x31>; 1144 defm V_PK_ADD_F32 : VOP3P_Real_vi <0x32>; 1145 defm V_PK_MOV_B32 : VOP3P_Real_vi <0x33>; 1146} // End SubtargetPredicate = HasPackedFP32Ops 1147 1148//===----------------------------------------------------------------------===// 1149// GFX10. 1150//===----------------------------------------------------------------------===// 1151 1152let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10", VOP3P = 1 in { 1153 multiclass VOP3P_Real_gfx10<bits<7> op> { 1154 def _gfx10 : VOP3P_Real<!cast<VOP3P_Pseudo>(NAME), SIEncodingFamily.GFX10>, 1155 VOP3Pe_gfx10 <op, !cast<VOP3P_Pseudo>(NAME).Pfl>; 1156 } 1157} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10", VOP3P = 1 1158 1159multiclass VOP3P_Real_gfx10_gfx11<bits<7> op> 1160 : VOP3P_Real_gfx10<op>, VOP3P_Real_gfx11<op>; 1161 1162multiclass VOP3P_Real_gfx10_gfx11_Triple<bits<7> op> 1163 : VOP3P_Real_gfx10<op>, VOP3P_Realtriple_gfx11<op>; 1164 1165defm V_PK_MAD_I16 : VOP3P_Real_gfx10_gfx11<0x00>; 1166defm V_PK_MUL_LO_U16 : VOP3P_Real_gfx10_gfx11<0x01>; 1167defm V_PK_ADD_I16 : VOP3P_Real_gfx10_gfx11<0x02>; 1168defm V_PK_SUB_I16 : VOP3P_Real_gfx10_gfx11<0x03>; 1169defm V_PK_LSHLREV_B16 : VOP3P_Real_gfx10_gfx11<0x04>; 1170defm V_PK_LSHRREV_B16 : VOP3P_Real_gfx10_gfx11<0x05>; 1171defm V_PK_ASHRREV_I16 : VOP3P_Real_gfx10_gfx11<0x06>; 1172defm V_PK_MAX_I16 : VOP3P_Real_gfx10_gfx11<0x07>; 1173defm V_PK_MIN_I16 : VOP3P_Real_gfx10_gfx11<0x08>; 1174defm V_PK_MAD_U16 : VOP3P_Real_gfx10_gfx11<0x09>; 1175defm V_PK_ADD_U16 : VOP3P_Real_gfx10_gfx11<0x0a>; 1176defm V_PK_SUB_U16 : VOP3P_Real_gfx10_gfx11<0x0b>; 1177defm V_PK_MAX_U16 : VOP3P_Real_gfx10_gfx11<0x0c>; 1178defm V_PK_MIN_U16 : VOP3P_Real_gfx10_gfx11<0x0d>; 1179defm V_PK_FMA_F16 : VOP3P_Real_gfx10_gfx11<0x0e>; 1180defm V_PK_ADD_F16 : VOP3P_Real_gfx10_gfx11<0x0f>; 1181defm V_PK_MUL_F16 : VOP3P_Real_gfx10_gfx11<0x10>; 1182defm V_PK_MIN_F16 : VOP3P_Real_gfx10_gfx11<0x11>; 1183defm V_PK_MAX_F16 : VOP3P_Real_gfx10_gfx11<0x12>; 1184defm V_FMA_MIX_F32 : VOP3P_Real_gfx10_gfx11_Triple <0x20>; 1185defm V_FMA_MIXLO_F16 : VOP3P_Real_gfx10_gfx11_Triple <0x21>; 1186defm V_FMA_MIXHI_F16 : VOP3P_Real_gfx10_gfx11_Triple <0x22>; 1187 1188let SubtargetPredicate = HasDot2Insts in { 1189 1190defm V_DOT2_I32_I16 : VOP3P_Real_gfx10 <0x14>; 1191defm V_DOT2_U32_U16 : VOP3P_Real_gfx10 <0x15>; 1192 1193} // End SubtargetPredicate = HasDot2Insts 1194 1195let SubtargetPredicate = HasDot7Insts in { 1196 1197defm V_DOT2_F32_F16 : VOP3P_Real_gfx10_gfx11_Triple <0x13>; 1198defm V_DOT4_U32_U8 : VOP3P_Real_gfx10_gfx11 <0x17>; 1199defm V_DOT8_U32_U4 : VOP3P_Real_gfx10_gfx11 <0x19>; 1200 1201} // End SubtargetPredicate = HasDot7Insts 1202 1203let SubtargetPredicate = HasDot1Insts in { 1204 1205defm V_DOT4_I32_I8 : VOP3P_Real_gfx10 <0x16>; 1206defm V_DOT8_I32_I4 : VOP3P_Real_gfx10 <0x18>; 1207 1208} // End SubtargetPredicate = HasDot1Insts 1209