1//===-- VOP3PInstructions.td - Vector Instruction Defintions --------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10//===----------------------------------------------------------------------===// 11// VOP3P Classes 12//===----------------------------------------------------------------------===// 13 14class VOP3PInst<string OpName, VOPProfile P, SDPatternOperator node = null_frag> : 15 VOP3P_Pseudo<OpName, P, 16 !if(P.HasModifiers, getVOP3PModPat<P, node>.ret, getVOP3Pat<P, node>.ret) 17>; 18 19// Non-packed instructions that use the VOP3P encoding. 20// VOP3 neg/abs and VOP3P opsel/opsel_hi modifiers are allowed. 21class VOP3_VOP3PInst<string OpName, VOPProfile P, bit UseTiedOutput = 0, 22 SDPatternOperator node = null_frag> : 23 VOP3P_Pseudo<OpName, P> { 24 // These operands are only sort of f16 operands. Depending on 25 // op_sel_hi, these may be interpreted as f32. The inline immediate 26 // values are really f16 converted to f32, so we treat these as f16 27 // operands. 28 let InOperandList = 29 !con( 30 !con( 31 (ins FP16InputMods:$src0_modifiers, VCSrc_f16:$src0, 32 FP16InputMods:$src1_modifiers, VCSrc_f16:$src1, 33 FP16InputMods:$src2_modifiers, VCSrc_f16:$src2, 34 clampmod:$clamp), 35 !if(UseTiedOutput, (ins VGPR_32:$vdst_in), (ins))), 36 (ins op_sel:$op_sel, op_sel_hi:$op_sel_hi)); 37 38 let Constraints = !if(UseTiedOutput, "$vdst = $vdst_in", ""); 39 let DisableEncoding = !if(UseTiedOutput, "$vdst_in", ""); 40 let AsmOperands = 41 " $vdst, $src0_modifiers, $src1_modifiers, $src2_modifiers$op_sel$op_sel_hi$clamp"; 42} 43 44let isCommutable = 1 in { 45def V_PK_FMA_F16 : VOP3PInst<"v_pk_fma_f16", VOP3_Profile<VOP_V2F16_V2F16_V2F16_V2F16>, fma>; 46def V_PK_MAD_I16 : VOP3PInst<"v_pk_mad_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>; 47def V_PK_MAD_U16 : VOP3PInst<"v_pk_mad_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>; 48 49def V_PK_ADD_F16 : VOP3PInst<"v_pk_add_f16", VOP3_Profile<VOP_V2F16_V2F16_V2F16>, fadd>; 50def V_PK_MUL_F16 : VOP3PInst<"v_pk_mul_f16", VOP3_Profile<VOP_V2F16_V2F16_V2F16>, fmul>; 51def V_PK_MAX_F16 : VOP3PInst<"v_pk_max_f16", VOP3_Profile<VOP_V2F16_V2F16_V2F16>, fmaxnum_like>; 52def V_PK_MIN_F16 : VOP3PInst<"v_pk_min_f16", VOP3_Profile<VOP_V2F16_V2F16_V2F16>, fminnum_like>; 53 54def V_PK_ADD_U16 : VOP3PInst<"v_pk_add_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, add>; 55def V_PK_ADD_I16 : VOP3PInst<"v_pk_add_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>>; 56def V_PK_MUL_LO_U16 : VOP3PInst<"v_pk_mul_lo_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, mul>; 57 58def V_PK_MIN_I16 : VOP3PInst<"v_pk_min_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, smin>; 59def V_PK_MIN_U16 : VOP3PInst<"v_pk_min_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, umin>; 60def V_PK_MAX_I16 : VOP3PInst<"v_pk_max_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, smax>; 61def V_PK_MAX_U16 : VOP3PInst<"v_pk_max_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, umax>; 62} 63 64def V_PK_SUB_U16 : VOP3PInst<"v_pk_sub_u16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>>; 65def V_PK_SUB_I16 : VOP3PInst<"v_pk_sub_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, sub>; 66 67def V_PK_LSHLREV_B16 : VOP3PInst<"v_pk_lshlrev_b16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, lshl_rev>; 68def V_PK_ASHRREV_I16 : VOP3PInst<"v_pk_ashrrev_i16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, ashr_rev>; 69def V_PK_LSHRREV_B16 : VOP3PInst<"v_pk_lshrrev_b16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, lshr_rev>; 70 71multiclass MadFmaMixPats<SDPatternOperator fma_like, 72 Instruction mix_inst, 73 Instruction mixlo_inst, 74 Instruction mixhi_inst> { 75 def : GCNPat < 76 (f16 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 77 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 78 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))), 79 (mixlo_inst $src0_modifiers, $src0, 80 $src1_modifiers, $src1, 81 $src2_modifiers, $src2, 82 DSTCLAMP.NONE, 83 (i32 (IMPLICIT_DEF))) 84 >; 85 86 // FIXME: Special case handling for maxhi (especially for clamp) 87 // because dealing with the write to high half of the register is 88 // difficult. 89 def : GCNPat < 90 (build_vector f16:$elt0, (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 91 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 92 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers))))), 93 (v2f16 (mixhi_inst $src0_modifiers, $src0, 94 $src1_modifiers, $src1, 95 $src2_modifiers, $src2, 96 DSTCLAMP.NONE, 97 $elt0)) 98 >; 99 100 def : GCNPat < 101 (build_vector 102 f16:$elt0, 103 (AMDGPUclamp (fpround (fma_like (f32 (VOP3PMadMixMods f16:$src0, i32:$src0_modifiers)), 104 (f32 (VOP3PMadMixMods f16:$src1, i32:$src1_modifiers)), 105 (f32 (VOP3PMadMixMods f16:$src2, i32:$src2_modifiers)))))), 106 (v2f16 (mixhi_inst $src0_modifiers, $src0, 107 $src1_modifiers, $src1, 108 $src2_modifiers, $src2, 109 DSTCLAMP.ENABLE, 110 $elt0)) 111 >; 112 113 def : GCNPat < 114 (AMDGPUclamp (build_vector 115 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$lo_src0, i32:$lo_src0_modifiers)), 116 (f32 (VOP3PMadMixMods f16:$lo_src1, i32:$lo_src1_modifiers)), 117 (f32 (VOP3PMadMixMods f16:$lo_src2, i32:$lo_src2_modifiers)))), 118 (fpround (fma_like (f32 (VOP3PMadMixMods f16:$hi_src0, i32:$hi_src0_modifiers)), 119 (f32 (VOP3PMadMixMods f16:$hi_src1, i32:$hi_src1_modifiers)), 120 (f32 (VOP3PMadMixMods f16:$hi_src2, i32:$hi_src2_modifiers)))))), 121 (v2f16 (mixhi_inst $hi_src0_modifiers, $hi_src0, 122 $hi_src1_modifiers, $hi_src1, 123 $hi_src2_modifiers, $hi_src2, 124 DSTCLAMP.ENABLE, 125 (mixlo_inst $lo_src0_modifiers, $lo_src0, 126 $lo_src1_modifiers, $lo_src1, 127 $lo_src2_modifiers, $lo_src2, 128 DSTCLAMP.ENABLE, 129 (i32 (IMPLICIT_DEF))))) 130 >; 131} 132 133let SubtargetPredicate = HasMadMixInsts in { 134// These are VOP3a-like opcodes which accept no omod. 135// Size of src arguments (16/32) is controlled by op_sel. 136// For 16-bit src arguments their location (hi/lo) are controlled by op_sel_hi. 137let isCommutable = 1 in { 138def V_MAD_MIX_F32 : VOP3_VOP3PInst<"v_mad_mix_f32", VOP3_Profile<VOP_F32_F16_F16_F16, VOP3_OPSEL>>; 139 140// Clamp modifier is applied after conversion to f16. 141def V_MAD_MIXLO_F16 : VOP3_VOP3PInst<"v_mad_mixlo_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, 1>; 142 143let ClampLo = 0, ClampHi = 1 in { 144def V_MAD_MIXHI_F16 : VOP3_VOP3PInst<"v_mad_mixhi_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, 1>; 145} 146} 147 148defm : MadFmaMixPats<fmad, V_MAD_MIX_F32, V_MAD_MIXLO_F16, V_MAD_MIXHI_F16>; 149} // End SubtargetPredicate = HasMadMixInsts 150 151 152// Essentially the same as the mad_mix versions 153let SubtargetPredicate = HasFmaMixInsts in { 154let isCommutable = 1 in { 155def V_FMA_MIX_F32 : VOP3_VOP3PInst<"v_fma_mix_f32", VOP3_Profile<VOP_F32_F16_F16_F16, VOP3_OPSEL>>; 156 157// Clamp modifier is applied after conversion to f16. 158def V_FMA_MIXLO_F16 : VOP3_VOP3PInst<"v_fma_mixlo_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, 1>; 159 160let ClampLo = 0, ClampHi = 1 in { 161def V_FMA_MIXHI_F16 : VOP3_VOP3PInst<"v_fma_mixhi_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, 1>; 162} 163} 164 165defm : MadFmaMixPats<fma, V_FMA_MIX_F32, V_FMA_MIXLO_F16, V_FMA_MIXHI_F16>; 166} 167 168// Defines patterns that extract signed 4bit from each Idx[0]. 169foreach Idx = [[0,28],[4,24],[8,20],[12,16],[16,12],[20,8],[24,4]] in 170 def ExtractSigned4bit_#Idx[0] : PatFrag<(ops node:$src), 171 (sra (shl node:$src, (i32 Idx[1])), (i32 28))>; 172 173// Defines code pattern that extracts U(unsigned/signed) 4/8bit from FromBitIndex. 174class Extract<int FromBitIndex, int BitMask, bit U>: PatFrag< 175 (ops node:$src), 176 !if (!or (!and (!eq (BitMask, 255), !eq (FromBitIndex, 24)), !eq (FromBitIndex, 28)), // last element 177 !if (U, (srl node:$src, (i32 FromBitIndex)), (sra node:$src, (i32 FromBitIndex))), 178 !if (!eq (FromBitIndex, 0), // first element 179 !if (U, (and node:$src, (i32 BitMask)), 180 !if (!eq (BitMask, 15), (!cast<PatFrag>("ExtractSigned4bit_"#FromBitIndex) node:$src), 181 (sext_inreg node:$src, i8))), 182 !if (U, (and (srl node:$src, (i32 FromBitIndex)), (i32 BitMask)), 183 !if (!eq (BitMask, 15), (!cast<PatFrag>("ExtractSigned4bit_"#FromBitIndex) node:$src), 184 (sext_inreg (srl node:$src, (i32 FromBitIndex)), i8)))))>; 185 186 187foreach Type = ["I", "U"] in 188 foreach Index = 0-3 in { 189 // Defines patterns that extract each Index'ed 8bit from an unsigned 190 // 32bit scalar value; 191 def #Type#Index#"_8bit" : Extract<!shl(Index, 3), 255, !if (!eq (Type, "U"), 1, 0)>; 192 193 // Defines multiplication patterns where the multiplication is happening on each 194 // Index'ed 8bit of a 32bit scalar value. 195 196 def Mul#Type#_Elt#Index : PatFrag< 197 (ops node:$src0, node:$src1), 198 (!cast<HasOneUseBinOp>(!if (!eq (Type, "I"), AMDGPUmul_i24_oneuse, AMDGPUmul_u24_oneuse)) 199 (!cast<Extract>(#Type#Index#"_8bit") node:$src0), 200 (!cast<Extract>(#Type#Index#"_8bit") node:$src1))>; 201 } 202 203// Different variants of dot8 patterns cause a huge increase in the compile time. 204// Define non-associative/commutative add/mul to prevent permutation in the dot8 205// pattern. 206def NonACAdd : SDNode<"ISD::ADD" , SDTIntBinOp>; 207def NonACAdd_oneuse : HasOneUseBinOp<NonACAdd>; 208 209def NonACAMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24" , SDTIntBinOp>; 210def NonACAMDGPUmul_u24_oneuse : HasOneUseBinOp<NonACAMDGPUmul_u24>; 211 212def NonACAMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24" , SDTIntBinOp>; 213def NonACAMDGPUmul_i24_oneuse : HasOneUseBinOp<NonACAMDGPUmul_i24>; 214 215foreach Type = ["I", "U"] in 216 foreach Index = 0-7 in { 217 // Defines patterns that extract each Index'ed 4bit from an unsigned 218 // 32bit scalar value; 219 def #Type#Index#"_4bit" : Extract<!shl(Index, 2), 15, !if (!eq (Type, "U"), 1, 0)>; 220 221 // Defines multiplication patterns where the multiplication is happening on each 222 // Index'ed 8bit of a 32bit scalar value. 223 def Mul#Type#Index#"_4bit" : PatFrag< 224 (ops node:$src0, node:$src1), 225 (!cast<HasOneUseBinOp>(!if (!eq (Type, "I"), NonACAMDGPUmul_i24_oneuse, NonACAMDGPUmul_u24_oneuse)) 226 (!cast<Extract>(#Type#Index#"_4bit") node:$src0), 227 (!cast<Extract>(#Type#Index#"_4bit") node:$src1))>; 228 } 229 230class UDot2Pat<Instruction Inst> : GCNPat < 231 (add (add_oneuse (AMDGPUmul_u24_oneuse (srl i32:$src0, (i32 16)), 232 (srl i32:$src1, (i32 16))), i32:$src2), 233 (AMDGPUmul_u24_oneuse (and i32:$src0, (i32 65535)), 234 (and i32:$src1, (i32 65535))) 235 ), 236 (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0)) 237>; 238 239class SDot2Pat<Instruction Inst> : GCNPat < 240 (add (add_oneuse (AMDGPUmul_i24_oneuse (sra i32:$src0, (i32 16)), 241 (sra i32:$src1, (i32 16))), i32:$src2), 242 (AMDGPUmul_i24_oneuse (sext_inreg i32:$src0, i16), 243 (sext_inreg i32:$src1, i16))), 244 (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0)) 245>; 246 247let SubtargetPredicate = HasDLInsts in { 248 249def V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16", VOP3_Profile<VOP_F32_V2F16_V2F16_F32>>; 250def V_DOT2_I32_I16 : VOP3PInst<"v_dot2_i32_i16", VOP3_Profile<VOP_I32_V2I16_V2I16_I32>>; 251def V_DOT2_U32_U16 : VOP3PInst<"v_dot2_u32_u16", VOP3_Profile<VOP_I32_V2I16_V2I16_I32>>; 252def V_DOT4_I32_I8 : VOP3PInst<"v_dot4_i32_i8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>; 253def V_DOT4_U32_U8 : VOP3PInst<"v_dot4_u32_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>; 254def V_DOT8_I32_I4 : VOP3PInst<"v_dot8_i32_i4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>; 255def V_DOT8_U32_U4 : VOP3PInst<"v_dot8_u32_u4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>; 256 257multiclass DotPats<SDPatternOperator dot_op, 258 VOP3PInst dot_inst> { 259 def : GCNPat < 260 (dot_op (dot_inst.Pfl.Src0VT (VOP3PMods0 dot_inst.Pfl.Src0VT:$src0, i32:$src0_modifiers)), 261 (dot_inst.Pfl.Src1VT (VOP3PMods dot_inst.Pfl.Src1VT:$src1, i32:$src1_modifiers)), 262 (dot_inst.Pfl.Src2VT (VOP3PMods dot_inst.Pfl.Src2VT:$src2, i32:$src2_modifiers)), i1:$clamp), 263 (dot_inst $src0_modifiers, $src0, $src1_modifiers, $src1, $src2_modifiers, $src2, (as_i1imm $clamp))>; 264} 265 266defm : DotPats<AMDGPUfdot2, V_DOT2_F32_F16>; 267defm : DotPats<int_amdgcn_sdot2, V_DOT2_I32_I16>; 268defm : DotPats<int_amdgcn_udot2, V_DOT2_U32_U16>; 269defm : DotPats<int_amdgcn_sdot4, V_DOT4_I32_I8>; 270defm : DotPats<int_amdgcn_udot4, V_DOT4_U32_U8>; 271defm : DotPats<int_amdgcn_sdot8, V_DOT8_I32_I4>; 272defm : DotPats<int_amdgcn_udot8, V_DOT8_U32_U4>; 273 274def : UDot2Pat<V_DOT2_U32_U16>; 275def : SDot2Pat<V_DOT2_I32_I16>; 276 277foreach Type = ["U", "I"] in 278 def : GCNPat < 279 !cast<dag>(!foldl((i32 i32:$src2), [0, 1, 2, 3], lhs, y, 280 (add_oneuse lhs, (!cast<PatFrag>("Mul"#Type#"_Elt"#y) i32:$src0, i32:$src1)))), 281 (!cast<VOP3PInst>("V_DOT4_"#Type#"32_"#Type#8) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>; 282 283foreach Type = ["U", "I"] in 284 def : GCNPat < 285 !cast<dag>(!foldl((add_oneuse i32:$src2, (!cast<PatFrag>("Mul"#Type#"0_4bit") i32:$src0, i32:$src1)), 286 [1, 2, 3, 4, 5, 6, 7], lhs, y, 287 (NonACAdd_oneuse lhs, (!cast<PatFrag>("Mul"#Type#y#"_4bit") i32:$src0, i32:$src1)))), 288 (!cast<VOP3PInst>("V_DOT8_"#Type#"32_"#Type#4) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>; 289 290} // End SubtargetPredicate = HasDLInsts 291 292multiclass VOP3P_Real_vi<bits<10> op> { 293 def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>, 294 VOP3Pe <op, !cast<VOP3_Pseudo>(NAME).Pfl> { 295 let AssemblerPredicates = [HasVOP3PInsts]; 296 let DecoderNamespace = "VI"; 297 } 298} 299 300defm V_PK_MAD_I16 : VOP3P_Real_vi <0x380>; 301defm V_PK_MUL_LO_U16 : VOP3P_Real_vi <0x381>; 302defm V_PK_ADD_I16 : VOP3P_Real_vi <0x382>; 303defm V_PK_SUB_I16 : VOP3P_Real_vi <0x383>; 304defm V_PK_LSHLREV_B16 : VOP3P_Real_vi <0x384>; 305defm V_PK_LSHRREV_B16 : VOP3P_Real_vi <0x385>; 306defm V_PK_ASHRREV_I16 : VOP3P_Real_vi <0x386>; 307defm V_PK_MAX_I16 : VOP3P_Real_vi <0x387>; 308defm V_PK_MIN_I16 : VOP3P_Real_vi <0x388>; 309defm V_PK_MAD_U16 : VOP3P_Real_vi <0x389>; 310 311defm V_PK_ADD_U16 : VOP3P_Real_vi <0x38a>; 312defm V_PK_SUB_U16 : VOP3P_Real_vi <0x38b>; 313defm V_PK_MAX_U16 : VOP3P_Real_vi <0x38c>; 314defm V_PK_MIN_U16 : VOP3P_Real_vi <0x38d>; 315defm V_PK_FMA_F16 : VOP3P_Real_vi <0x38e>; 316defm V_PK_ADD_F16 : VOP3P_Real_vi <0x38f>; 317defm V_PK_MUL_F16 : VOP3P_Real_vi <0x390>; 318defm V_PK_MIN_F16 : VOP3P_Real_vi <0x391>; 319defm V_PK_MAX_F16 : VOP3P_Real_vi <0x392>; 320 321 322let SubtargetPredicate = HasMadMixInsts in { 323defm V_MAD_MIX_F32 : VOP3P_Real_vi <0x3a0>; 324defm V_MAD_MIXLO_F16 : VOP3P_Real_vi <0x3a1>; 325defm V_MAD_MIXHI_F16 : VOP3P_Real_vi <0x3a2>; 326} 327 328let SubtargetPredicate = HasFmaMixInsts in { 329let DecoderNamespace = "GFX9_DL" in { 330// The mad_mix instructions were renamed and their behaviors changed, 331// but the opcode stayed the same so we need to put these in a 332// different DecoderNamespace to avoid the ambiguity. 333defm V_FMA_MIX_F32 : VOP3P_Real_vi <0x3a0>; 334defm V_FMA_MIXLO_F16 : VOP3P_Real_vi <0x3a1>; 335defm V_FMA_MIXHI_F16 : VOP3P_Real_vi <0x3a2>; 336} 337} 338 339 340let SubtargetPredicate = HasDLInsts in { 341 342defm V_DOT2_F32_F16 : VOP3P_Real_vi <0x3a3>; 343defm V_DOT2_I32_I16 : VOP3P_Real_vi <0x3a6>; 344defm V_DOT2_U32_U16 : VOP3P_Real_vi <0x3a7>; 345defm V_DOT4_I32_I8 : VOP3P_Real_vi <0x3a8>; 346defm V_DOT4_U32_U8 : VOP3P_Real_vi <0x3a9>; 347defm V_DOT8_I32_I4 : VOP3P_Real_vi <0x3aa>; 348defm V_DOT8_U32_U4 : VOP3P_Real_vi <0x3ab>; 349 350} // End SubtargetPredicate = HasDLInsts 351