1 //===-- X86InstrFMA3Info.cpp - X86 FMA3 Instruction Information -----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the implementation of the classes providing information 11 // about existing X86 FMA3 opcodes, classifying and grouping them. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86InstrFMA3Info.h" 16 #include "X86InstrInfo.h" 17 #include "llvm/Support/ManagedStatic.h" 18 #include "llvm/Support/Threading.h" 19 #include <cassert> 20 #include <cstdint> 21 22 using namespace llvm; 23 24 /// This flag is used in the method llvm::call_once() used below to make the 25 /// initialization of the map 'OpcodeToGroup' thread safe. 26 static llvm::once_flag InitGroupsOnceFlag; 27 28 static ManagedStatic<X86InstrFMA3Info> X86InstrFMA3InfoObj; 29 X86InstrFMA3Info *X86InstrFMA3Info::getX86InstrFMA3Info() { 30 return &*X86InstrFMA3InfoObj; 31 } 32 33 void X86InstrFMA3Info::initRMGroup(const uint16_t *RegOpcodes, 34 const uint16_t *MemOpcodes, unsigned Attr) { 35 // Create a new instance of this class that would hold a group of FMA opcodes. 36 X86InstrFMA3Group *G = new X86InstrFMA3Group(RegOpcodes, MemOpcodes, Attr); 37 38 // Add the references from indvidual opcodes to the group holding them. 39 assert((!OpcodeToGroup[RegOpcodes[0]] && !OpcodeToGroup[RegOpcodes[1]] && 40 !OpcodeToGroup[RegOpcodes[2]] && !OpcodeToGroup[MemOpcodes[0]] && 41 !OpcodeToGroup[MemOpcodes[1]] && !OpcodeToGroup[MemOpcodes[2]]) && 42 "Duplication or rewrite of elements in OpcodeToGroup."); 43 OpcodeToGroup[RegOpcodes[0]] = G; 44 OpcodeToGroup[RegOpcodes[1]] = G; 45 OpcodeToGroup[RegOpcodes[2]] = G; 46 OpcodeToGroup[MemOpcodes[0]] = G; 47 OpcodeToGroup[MemOpcodes[1]] = G; 48 OpcodeToGroup[MemOpcodes[2]] = G; 49 } 50 51 void X86InstrFMA3Info::initRGroup(const uint16_t *RegOpcodes, unsigned Attr) { 52 // Create a new instance of this class that would hold a group of FMA opcodes. 53 X86InstrFMA3Group *G = new X86InstrFMA3Group(RegOpcodes, nullptr, Attr); 54 55 // Add the references from indvidual opcodes to the group holding them. 56 assert((!OpcodeToGroup[RegOpcodes[0]] && !OpcodeToGroup[RegOpcodes[1]] && 57 !OpcodeToGroup[RegOpcodes[2]]) && 58 "Duplication or rewrite of elements in OpcodeToGroup."); 59 OpcodeToGroup[RegOpcodes[0]] = G; 60 OpcodeToGroup[RegOpcodes[1]] = G; 61 OpcodeToGroup[RegOpcodes[2]] = G; 62 } 63 64 void X86InstrFMA3Info::initMGroup(const uint16_t *MemOpcodes, unsigned Attr) { 65 // Create a new instance of this class that would hold a group of FMA opcodes. 66 X86InstrFMA3Group *G = new X86InstrFMA3Group(nullptr, MemOpcodes, Attr); 67 68 // Add the references from indvidual opcodes to the group holding them. 69 assert((!OpcodeToGroup[MemOpcodes[0]] && !OpcodeToGroup[MemOpcodes[1]] && 70 !OpcodeToGroup[MemOpcodes[2]]) && 71 "Duplication or rewrite of elements in OpcodeToGroup."); 72 OpcodeToGroup[MemOpcodes[0]] = G; 73 OpcodeToGroup[MemOpcodes[1]] = G; 74 OpcodeToGroup[MemOpcodes[2]] = G; 75 } 76 77 #define FMA3RM(R132, R213, R231, M132, M213, M231) \ 78 static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231}; \ 79 static const uint16_t Mem##R132[3] = {X86::M132, X86::M213, X86::M231}; \ 80 initRMGroup(Reg##R132, Mem##R132); 81 82 #define FMA3RMA(R132, R213, R231, M132, M213, M231, Attrs) \ 83 static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231}; \ 84 static const uint16_t Mem##R132[3] = {X86::M132, X86::M213, X86::M231}; \ 85 initRMGroup(Reg##R132, Mem##R132, (Attrs)); 86 87 #define FMA3R(R132, R213, R231) \ 88 static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231}; \ 89 initRGroup(Reg##R132); 90 91 #define FMA3RA(R132, R213, R231, Attrs) \ 92 static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231}; \ 93 initRGroup(Reg##R132, (Attrs)); 94 95 #define FMA3M(M132, M213, M231) \ 96 static const uint16_t Mem##M132[3] = {X86::M132, X86::M213, X86::M231}; \ 97 initMGroup(Mem##M132); 98 99 #define FMA3MA(M132, M213, M231, Attrs) \ 100 static const uint16_t Mem##M132[3] = {X86::M132, X86::M213, X86::M231}; \ 101 initMGroup(Mem##M132, (Attrs)); 102 103 #define FMA3_AVX2_VECTOR_GROUP(Name) \ 104 FMA3RM(Name##132PSr, Name##213PSr, Name##231PSr, \ 105 Name##132PSm, Name##213PSm, Name##231PSm); \ 106 FMA3RM(Name##132PDr, Name##213PDr, Name##231PDr, \ 107 Name##132PDm, Name##213PDm, Name##231PDm); \ 108 FMA3RM(Name##132PSYr, Name##213PSYr, Name##231PSYr, \ 109 Name##132PSYm, Name##213PSYm, Name##231PSYm); \ 110 FMA3RM(Name##132PDYr, Name##213PDYr, Name##231PDYr, \ 111 Name##132PDYm, Name##213PDYm, Name##231PDYm); 112 113 #define FMA3_AVX2_SCALAR_GROUP(Name) \ 114 FMA3RM(Name##132SSr, Name##213SSr, Name##231SSr, \ 115 Name##132SSm, Name##213SSm, Name##231SSm); \ 116 FMA3RM(Name##132SDr, Name##213SDr, Name##231SDr, \ 117 Name##132SDm, Name##213SDm, Name##231SDm); \ 118 FMA3RMA(Name##132SSr_Int, Name##213SSr_Int, Name##231SSr_Int, \ 119 Name##132SSm_Int, Name##213SSm_Int, Name##231SSm_Int, \ 120 X86InstrFMA3Group::X86FMA3Intrinsic); \ 121 FMA3RMA(Name##132SDr_Int, Name##213SDr_Int, Name##231SDr_Int, \ 122 Name##132SDm_Int, Name##213SDm_Int, Name##231SDm_Int, \ 123 X86InstrFMA3Group::X86FMA3Intrinsic); 124 125 #define FMA3_AVX2_FULL_GROUP(Name) \ 126 FMA3_AVX2_VECTOR_GROUP(Name); \ 127 FMA3_AVX2_SCALAR_GROUP(Name); 128 129 #define FMA3_AVX512_VECTOR_GROUP(Name) \ 130 FMA3RM(Name##132PSZ128r, Name##213PSZ128r, Name##231PSZ128r, \ 131 Name##132PSZ128m, Name##213PSZ128m, Name##231PSZ128m); \ 132 FMA3RM(Name##132PDZ128r, Name##213PDZ128r, Name##231PDZ128r, \ 133 Name##132PDZ128m, Name##213PDZ128m, Name##231PDZ128m); \ 134 FMA3RM(Name##132PSZ256r, Name##213PSZ256r, Name##231PSZ256r, \ 135 Name##132PSZ256m, Name##213PSZ256m, Name##231PSZ256m); \ 136 FMA3RM(Name##132PDZ256r, Name##213PDZ256r, Name##231PDZ256r, \ 137 Name##132PDZ256m, Name##213PDZ256m, Name##231PDZ256m); \ 138 FMA3RM(Name##132PSZr, Name##213PSZr, Name##231PSZr, \ 139 Name##132PSZm, Name##213PSZm, Name##231PSZm); \ 140 FMA3RM(Name##132PDZr, Name##213PDZr, Name##231PDZr, \ 141 Name##132PDZm, Name##213PDZm, Name##231PDZm); \ 142 FMA3RMA(Name##132PSZ128rk, Name##213PSZ128rk, Name##231PSZ128rk, \ 143 Name##132PSZ128mk, Name##213PSZ128mk, Name##231PSZ128mk, \ 144 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 145 FMA3RMA(Name##132PDZ128rk, Name##213PDZ128rk, Name##231PDZ128rk, \ 146 Name##132PDZ128mk, Name##213PDZ128mk, Name##231PDZ128mk, \ 147 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 148 FMA3RMA(Name##132PSZ256rk, Name##213PSZ256rk, Name##231PSZ256rk, \ 149 Name##132PSZ256mk, Name##213PSZ256mk, Name##231PSZ256mk, \ 150 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 151 FMA3RMA(Name##132PDZ256rk, Name##213PDZ256rk, Name##231PDZ256rk, \ 152 Name##132PDZ256mk, Name##213PDZ256mk, Name##231PDZ256mk, \ 153 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 154 FMA3RMA(Name##132PSZrk, Name##213PSZrk, Name##231PSZrk, \ 155 Name##132PSZmk, Name##213PSZmk, Name##231PSZmk, \ 156 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 157 FMA3RMA(Name##132PDZrk, Name##213PDZrk, Name##231PDZrk, \ 158 Name##132PDZmk, Name##213PDZmk, Name##231PDZmk, \ 159 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 160 FMA3RMA(Name##132PSZ128rkz, Name##213PSZ128rkz, Name##231PSZ128rkz, \ 161 Name##132PSZ128mkz, Name##213PSZ128mkz, Name##231PSZ128mkz, \ 162 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 163 FMA3RMA(Name##132PDZ128rkz, Name##213PDZ128rkz, Name##231PDZ128rkz, \ 164 Name##132PDZ128mkz, Name##213PDZ128mkz, Name##231PDZ128mkz, \ 165 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 166 FMA3RMA(Name##132PSZ256rkz, Name##213PSZ256rkz, Name##231PSZ256rkz, \ 167 Name##132PSZ256mkz, Name##213PSZ256mkz, Name##231PSZ256mkz, \ 168 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 169 FMA3RMA(Name##132PDZ256rkz, Name##213PDZ256rkz, Name##231PDZ256rkz, \ 170 Name##132PDZ256mkz, Name##213PDZ256mkz, Name##231PDZ256mkz, \ 171 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 172 FMA3RMA(Name##132PSZrkz, Name##213PSZrkz, Name##231PSZrkz, \ 173 Name##132PSZmkz, Name##213PSZmkz, Name##231PSZmkz, \ 174 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 175 FMA3RMA(Name##132PDZrkz, Name##213PDZrkz, Name##231PDZrkz, \ 176 Name##132PDZmkz, Name##213PDZmkz, Name##231PDZmkz, \ 177 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 178 FMA3R(Name##132PSZrb, Name##213PSZrb, Name##231PSZrb); \ 179 FMA3R(Name##132PDZrb, Name##213PDZrb, Name##231PDZrb); \ 180 FMA3RA(Name##132PSZrbk, Name##213PSZrbk, Name##231PSZrbk, \ 181 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 182 FMA3RA(Name##132PDZrbk, Name##213PDZrbk, Name##231PDZrbk, \ 183 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 184 FMA3RA(Name##132PSZrbkz, Name##213PSZrbkz, Name##231PSZrbkz, \ 185 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 186 FMA3RA(Name##132PDZrbkz, Name##213PDZrbkz, Name##231PDZrbkz, \ 187 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 188 FMA3M(Name##132PSZ128mb, Name##213PSZ128mb, Name##231PSZ128mb); \ 189 FMA3M(Name##132PDZ128mb, Name##213PDZ128mb, Name##231PDZ128mb); \ 190 FMA3M(Name##132PSZ256mb, Name##213PSZ256mb, Name##231PSZ256mb); \ 191 FMA3M(Name##132PDZ256mb, Name##213PDZ256mb, Name##231PDZ256mb); \ 192 FMA3M(Name##132PSZmb, Name##213PSZmb, Name##231PSZmb); \ 193 FMA3M(Name##132PDZmb, Name##213PDZmb, Name##231PDZmb); \ 194 FMA3MA(Name##132PSZ128mbk, Name##213PSZ128mbk, Name##231PSZ128mbk, \ 195 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 196 FMA3MA(Name##132PDZ128mbk, Name##213PDZ128mbk, Name##231PDZ128mbk, \ 197 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 198 FMA3MA(Name##132PSZ256mbk, Name##213PSZ256mbk, Name##231PSZ256mbk, \ 199 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 200 FMA3MA(Name##132PDZ256mbk, Name##213PDZ256mbk, Name##231PDZ256mbk, \ 201 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 202 FMA3MA(Name##132PSZmbk, Name##213PSZmbk, Name##231PSZmbk, \ 203 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 204 FMA3MA(Name##132PDZmbk, Name##213PDZmbk, Name##231PDZmbk, \ 205 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 206 FMA3MA(Name##132PSZ128mbkz, Name##213PSZ128mbkz, Name##231PSZ128mbkz, \ 207 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 208 FMA3MA(Name##132PDZ128mbkz, Name##213PDZ128mbkz, Name##231PDZ128mbkz, \ 209 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 210 FMA3MA(Name##132PSZ256mbkz, Name##213PSZ256mbkz, Name##231PSZ256mbkz, \ 211 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 212 FMA3MA(Name##132PDZ256mbkz, Name##213PDZ256mbkz, Name##231PDZ256mbkz, \ 213 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 214 FMA3MA(Name##132PSZmbkz, Name##213PSZmbkz, Name##231PSZmbkz, \ 215 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 216 FMA3MA(Name##132PDZmbkz, Name##213PDZmbkz, Name##231PDZmbkz, \ 217 X86InstrFMA3Group::X86FMA3KZeroMasked); 218 219 #define FMA3_AVX512_SCALAR_GROUP(Name) \ 220 FMA3RM(Name##132SSZr, Name##213SSZr, Name##231SSZr, \ 221 Name##132SSZm, Name##213SSZm, Name##231SSZm); \ 222 FMA3RM(Name##132SDZr, Name##213SDZr, Name##231SDZr, \ 223 Name##132SDZm, Name##213SDZm, Name##231SDZm); \ 224 FMA3RMA(Name##132SSZr_Int, Name##213SSZr_Int, Name##231SSZr_Int, \ 225 Name##132SSZm_Int, Name##213SSZm_Int, Name##231SSZm_Int, \ 226 X86InstrFMA3Group::X86FMA3Intrinsic); \ 227 FMA3RMA(Name##132SDZr_Int, Name##213SDZr_Int, Name##231SDZr_Int, \ 228 Name##132SDZm_Int, Name##213SDZm_Int, Name##231SDZm_Int, \ 229 X86InstrFMA3Group::X86FMA3Intrinsic); \ 230 FMA3RMA(Name##132SSZr_Intk, Name##213SSZr_Intk, Name##231SSZr_Intk, \ 231 Name##132SSZm_Intk, Name##213SSZm_Intk, Name##231SSZm_Intk, \ 232 X86InstrFMA3Group::X86FMA3Intrinsic | \ 233 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 234 FMA3RMA(Name##132SDZr_Intk, Name##213SDZr_Intk, Name##231SDZr_Intk, \ 235 Name##132SDZm_Intk, Name##213SDZm_Intk, Name##231SDZm_Intk, \ 236 X86InstrFMA3Group::X86FMA3Intrinsic | \ 237 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 238 FMA3RMA(Name##132SSZr_Intkz, Name##213SSZr_Intkz, Name##231SSZr_Intkz, \ 239 Name##132SSZm_Intkz, Name##213SSZm_Intkz, Name##231SSZm_Intkz, \ 240 X86InstrFMA3Group::X86FMA3Intrinsic | \ 241 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 242 FMA3RMA(Name##132SDZr_Intkz, Name##213SDZr_Intkz, Name##231SDZr_Intkz, \ 243 Name##132SDZm_Intkz, Name##213SDZm_Intkz, Name##231SDZm_Intkz, \ 244 X86InstrFMA3Group::X86FMA3Intrinsic | \ 245 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 246 FMA3RA(Name##132SSZrb_Int, Name##213SSZrb_Int, Name##231SSZrb_Int, \ 247 X86InstrFMA3Group::X86FMA3Intrinsic); \ 248 FMA3RA(Name##132SDZrb_Int, Name##213SDZrb_Int, Name##231SDZrb_Int, \ 249 X86InstrFMA3Group::X86FMA3Intrinsic); \ 250 FMA3RA(Name##132SSZrb_Intk, Name##213SSZrb_Intk, Name##231SSZrb_Intk, \ 251 X86InstrFMA3Group::X86FMA3Intrinsic | \ 252 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 253 FMA3RA(Name##132SDZrb_Intk, Name##213SDZrb_Intk, Name##231SDZrb_Intk, \ 254 X86InstrFMA3Group::X86FMA3Intrinsic | \ 255 X86InstrFMA3Group::X86FMA3KMergeMasked); \ 256 FMA3RA(Name##132SSZrb_Intkz, Name##213SSZrb_Intkz, Name##231SSZrb_Intkz, \ 257 X86InstrFMA3Group::X86FMA3Intrinsic | \ 258 X86InstrFMA3Group::X86FMA3KZeroMasked); \ 259 FMA3RA(Name##132SDZrb_Intkz, Name##213SDZrb_Intkz, Name##231SDZrb_Intkz, \ 260 X86InstrFMA3Group::X86FMA3Intrinsic | \ 261 X86InstrFMA3Group::X86FMA3KZeroMasked); 262 263 #define FMA3_AVX512_FULL_GROUP(Name) \ 264 FMA3_AVX512_VECTOR_GROUP(Name); \ 265 FMA3_AVX512_SCALAR_GROUP(Name); 266 267 void X86InstrFMA3Info::initGroupsOnceImpl() { 268 FMA3_AVX2_FULL_GROUP(VFMADD); 269 FMA3_AVX2_FULL_GROUP(VFMSUB); 270 FMA3_AVX2_FULL_GROUP(VFNMADD); 271 FMA3_AVX2_FULL_GROUP(VFNMSUB); 272 273 FMA3_AVX2_VECTOR_GROUP(VFMADDSUB); 274 FMA3_AVX2_VECTOR_GROUP(VFMSUBADD); 275 276 FMA3_AVX512_FULL_GROUP(VFMADD); 277 FMA3_AVX512_FULL_GROUP(VFMSUB); 278 FMA3_AVX512_FULL_GROUP(VFNMADD); 279 FMA3_AVX512_FULL_GROUP(VFNMSUB); 280 281 FMA3_AVX512_VECTOR_GROUP(VFMADDSUB); 282 FMA3_AVX512_VECTOR_GROUP(VFMSUBADD); 283 } 284 285 void X86InstrFMA3Info::initGroupsOnce() { 286 llvm::call_once(InitGroupsOnceFlag, 287 []() { getX86InstrFMA3Info()->initGroupsOnceImpl(); }); 288 } 289