1 //===- RISCVVIntrinsicUtils.cpp - RISC-V Vector Intrinsic Utils -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Support/RISCVVIntrinsicUtils.h" 10 #include "llvm/ADT/ArrayRef.h" 11 #include "llvm/ADT/Optional.h" 12 #include "llvm/ADT/SmallSet.h" 13 #include "llvm/ADT/StringExtras.h" 14 #include "llvm/ADT/StringMap.h" 15 #include "llvm/ADT/StringSet.h" 16 #include "llvm/ADT/Twine.h" 17 #include "llvm/Support/raw_ostream.h" 18 #include <numeric> 19 20 using namespace llvm; 21 22 namespace clang { 23 namespace RISCV { 24 25 //===----------------------------------------------------------------------===// 26 // Type implementation 27 //===----------------------------------------------------------------------===// 28 29 LMULType::LMULType(int NewLog2LMUL) { 30 // Check Log2LMUL is -3, -2, -1, 0, 1, 2, 3 31 assert(NewLog2LMUL <= 3 && NewLog2LMUL >= -3 && "Bad LMUL number!"); 32 Log2LMUL = NewLog2LMUL; 33 } 34 35 std::string LMULType::str() const { 36 if (Log2LMUL < 0) 37 return "mf" + utostr(1ULL << (-Log2LMUL)); 38 return "m" + utostr(1ULL << Log2LMUL); 39 } 40 41 VScaleVal LMULType::getScale(unsigned ElementBitwidth) const { 42 int Log2ScaleResult = 0; 43 switch (ElementBitwidth) { 44 default: 45 break; 46 case 8: 47 Log2ScaleResult = Log2LMUL + 3; 48 break; 49 case 16: 50 Log2ScaleResult = Log2LMUL + 2; 51 break; 52 case 32: 53 Log2ScaleResult = Log2LMUL + 1; 54 break; 55 case 64: 56 Log2ScaleResult = Log2LMUL; 57 break; 58 } 59 // Illegal vscale result would be less than 1 60 if (Log2ScaleResult < 0) 61 return llvm::None; 62 return 1 << Log2ScaleResult; 63 } 64 65 void LMULType::MulLog2LMUL(int log2LMUL) { Log2LMUL += log2LMUL; } 66 67 LMULType &LMULType::operator*=(uint32_t RHS) { 68 assert(isPowerOf2_32(RHS)); 69 this->Log2LMUL = this->Log2LMUL + Log2_32(RHS); 70 return *this; 71 } 72 73 RVVType::RVVType(BasicType BT, int Log2LMUL, StringRef prototype) 74 : BT(BT), LMUL(LMULType(Log2LMUL)) { 75 applyBasicType(); 76 applyModifier(prototype); 77 Valid = verifyType(); 78 if (Valid) { 79 initBuiltinStr(); 80 initTypeStr(); 81 if (isVector()) { 82 initClangBuiltinStr(); 83 } 84 } 85 } 86 87 // clang-format off 88 // boolean type are encoded the ratio of n (SEW/LMUL) 89 // SEW/LMUL | 1 | 2 | 4 | 8 | 16 | 32 | 64 90 // c type | vbool64_t | vbool32_t | vbool16_t | vbool8_t | vbool4_t | vbool2_t | vbool1_t 91 // IR type | nxv1i1 | nxv2i1 | nxv4i1 | nxv8i1 | nxv16i1 | nxv32i1 | nxv64i1 92 93 // type\lmul | 1/8 | 1/4 | 1/2 | 1 | 2 | 4 | 8 94 // -------- |------ | -------- | ------- | ------- | -------- | -------- | -------- 95 // i64 | N/A | N/A | N/A | nxv1i64 | nxv2i64 | nxv4i64 | nxv8i64 96 // i32 | N/A | N/A | nxv1i32 | nxv2i32 | nxv4i32 | nxv8i32 | nxv16i32 97 // i16 | N/A | nxv1i16 | nxv2i16 | nxv4i16 | nxv8i16 | nxv16i16 | nxv32i16 98 // i8 | nxv1i8 | nxv2i8 | nxv4i8 | nxv8i8 | nxv16i8 | nxv32i8 | nxv64i8 99 // double | N/A | N/A | N/A | nxv1f64 | nxv2f64 | nxv4f64 | nxv8f64 100 // float | N/A | N/A | nxv1f32 | nxv2f32 | nxv4f32 | nxv8f32 | nxv16f32 101 // half | N/A | nxv1f16 | nxv2f16 | nxv4f16 | nxv8f16 | nxv16f16 | nxv32f16 102 // clang-format on 103 104 bool RVVType::verifyType() const { 105 if (ScalarType == Invalid) 106 return false; 107 if (isScalar()) 108 return true; 109 if (!Scale.hasValue()) 110 return false; 111 if (isFloat() && ElementBitwidth == 8) 112 return false; 113 unsigned V = Scale.getValue(); 114 switch (ElementBitwidth) { 115 case 1: 116 case 8: 117 // Check Scale is 1,2,4,8,16,32,64 118 return (V <= 64 && isPowerOf2_32(V)); 119 case 16: 120 // Check Scale is 1,2,4,8,16,32 121 return (V <= 32 && isPowerOf2_32(V)); 122 case 32: 123 // Check Scale is 1,2,4,8,16 124 return (V <= 16 && isPowerOf2_32(V)); 125 case 64: 126 // Check Scale is 1,2,4,8 127 return (V <= 8 && isPowerOf2_32(V)); 128 } 129 return false; 130 } 131 132 void RVVType::initBuiltinStr() { 133 assert(isValid() && "RVVType is invalid"); 134 switch (ScalarType) { 135 case ScalarTypeKind::Void: 136 BuiltinStr = "v"; 137 return; 138 case ScalarTypeKind::Size_t: 139 BuiltinStr = "z"; 140 if (IsImmediate) 141 BuiltinStr = "I" + BuiltinStr; 142 if (IsPointer) 143 BuiltinStr += "*"; 144 return; 145 case ScalarTypeKind::Ptrdiff_t: 146 BuiltinStr = "Y"; 147 return; 148 case ScalarTypeKind::UnsignedLong: 149 BuiltinStr = "ULi"; 150 return; 151 case ScalarTypeKind::SignedLong: 152 BuiltinStr = "Li"; 153 return; 154 case ScalarTypeKind::Boolean: 155 assert(ElementBitwidth == 1); 156 BuiltinStr += "b"; 157 break; 158 case ScalarTypeKind::SignedInteger: 159 case ScalarTypeKind::UnsignedInteger: 160 switch (ElementBitwidth) { 161 case 8: 162 BuiltinStr += "c"; 163 break; 164 case 16: 165 BuiltinStr += "s"; 166 break; 167 case 32: 168 BuiltinStr += "i"; 169 break; 170 case 64: 171 BuiltinStr += "Wi"; 172 break; 173 default: 174 llvm_unreachable("Unhandled ElementBitwidth!"); 175 } 176 if (isSignedInteger()) 177 BuiltinStr = "S" + BuiltinStr; 178 else 179 BuiltinStr = "U" + BuiltinStr; 180 break; 181 case ScalarTypeKind::Float: 182 switch (ElementBitwidth) { 183 case 16: 184 BuiltinStr += "x"; 185 break; 186 case 32: 187 BuiltinStr += "f"; 188 break; 189 case 64: 190 BuiltinStr += "d"; 191 break; 192 default: 193 llvm_unreachable("Unhandled ElementBitwidth!"); 194 } 195 break; 196 default: 197 llvm_unreachable("ScalarType is invalid!"); 198 } 199 if (IsImmediate) 200 BuiltinStr = "I" + BuiltinStr; 201 if (isScalar()) { 202 if (IsConstant) 203 BuiltinStr += "C"; 204 if (IsPointer) 205 BuiltinStr += "*"; 206 return; 207 } 208 BuiltinStr = "q" + utostr(Scale.getValue()) + BuiltinStr; 209 // Pointer to vector types. Defined for segment load intrinsics. 210 // segment load intrinsics have pointer type arguments to store the loaded 211 // vector values. 212 if (IsPointer) 213 BuiltinStr += "*"; 214 } 215 216 void RVVType::initClangBuiltinStr() { 217 assert(isValid() && "RVVType is invalid"); 218 assert(isVector() && "Handle Vector type only"); 219 220 ClangBuiltinStr = "__rvv_"; 221 switch (ScalarType) { 222 case ScalarTypeKind::Boolean: 223 ClangBuiltinStr += "bool" + utostr(64 / Scale.getValue()) + "_t"; 224 return; 225 case ScalarTypeKind::Float: 226 ClangBuiltinStr += "float"; 227 break; 228 case ScalarTypeKind::SignedInteger: 229 ClangBuiltinStr += "int"; 230 break; 231 case ScalarTypeKind::UnsignedInteger: 232 ClangBuiltinStr += "uint"; 233 break; 234 default: 235 llvm_unreachable("ScalarTypeKind is invalid"); 236 } 237 ClangBuiltinStr += utostr(ElementBitwidth) + LMUL.str() + "_t"; 238 } 239 240 void RVVType::initTypeStr() { 241 assert(isValid() && "RVVType is invalid"); 242 243 if (IsConstant) 244 Str += "const "; 245 246 auto getTypeString = [&](StringRef TypeStr) { 247 if (isScalar()) 248 return Twine(TypeStr + Twine(ElementBitwidth) + "_t").str(); 249 return Twine("v" + TypeStr + Twine(ElementBitwidth) + LMUL.str() + "_t") 250 .str(); 251 }; 252 253 switch (ScalarType) { 254 case ScalarTypeKind::Void: 255 Str = "void"; 256 return; 257 case ScalarTypeKind::Size_t: 258 Str = "size_t"; 259 if (IsPointer) 260 Str += " *"; 261 return; 262 case ScalarTypeKind::Ptrdiff_t: 263 Str = "ptrdiff_t"; 264 return; 265 case ScalarTypeKind::UnsignedLong: 266 Str = "unsigned long"; 267 return; 268 case ScalarTypeKind::SignedLong: 269 Str = "long"; 270 return; 271 case ScalarTypeKind::Boolean: 272 if (isScalar()) 273 Str += "bool"; 274 else 275 // Vector bool is special case, the formulate is 276 // `vbool<N>_t = MVT::nxv<64/N>i1` ex. vbool16_t = MVT::4i1 277 Str += "vbool" + utostr(64 / Scale.getValue()) + "_t"; 278 break; 279 case ScalarTypeKind::Float: 280 if (isScalar()) { 281 if (ElementBitwidth == 64) 282 Str += "double"; 283 else if (ElementBitwidth == 32) 284 Str += "float"; 285 else if (ElementBitwidth == 16) 286 Str += "_Float16"; 287 else 288 llvm_unreachable("Unhandled floating type."); 289 } else 290 Str += getTypeString("float"); 291 break; 292 case ScalarTypeKind::SignedInteger: 293 Str += getTypeString("int"); 294 break; 295 case ScalarTypeKind::UnsignedInteger: 296 Str += getTypeString("uint"); 297 break; 298 default: 299 llvm_unreachable("ScalarType is invalid!"); 300 } 301 if (IsPointer) 302 Str += " *"; 303 } 304 305 void RVVType::initShortStr() { 306 switch (ScalarType) { 307 case ScalarTypeKind::Boolean: 308 assert(isVector()); 309 ShortStr = "b" + utostr(64 / Scale.getValue()); 310 return; 311 case ScalarTypeKind::Float: 312 ShortStr = "f" + utostr(ElementBitwidth); 313 break; 314 case ScalarTypeKind::SignedInteger: 315 ShortStr = "i" + utostr(ElementBitwidth); 316 break; 317 case ScalarTypeKind::UnsignedInteger: 318 ShortStr = "u" + utostr(ElementBitwidth); 319 break; 320 default: 321 llvm_unreachable("Unhandled case!"); 322 } 323 if (isVector()) 324 ShortStr += LMUL.str(); 325 } 326 327 void RVVType::applyBasicType() { 328 switch (BT) { 329 case 'c': 330 ElementBitwidth = 8; 331 ScalarType = ScalarTypeKind::SignedInteger; 332 break; 333 case 's': 334 ElementBitwidth = 16; 335 ScalarType = ScalarTypeKind::SignedInteger; 336 break; 337 case 'i': 338 ElementBitwidth = 32; 339 ScalarType = ScalarTypeKind::SignedInteger; 340 break; 341 case 'l': 342 ElementBitwidth = 64; 343 ScalarType = ScalarTypeKind::SignedInteger; 344 break; 345 case 'x': 346 ElementBitwidth = 16; 347 ScalarType = ScalarTypeKind::Float; 348 break; 349 case 'f': 350 ElementBitwidth = 32; 351 ScalarType = ScalarTypeKind::Float; 352 break; 353 case 'd': 354 ElementBitwidth = 64; 355 ScalarType = ScalarTypeKind::Float; 356 break; 357 default: 358 llvm_unreachable("Unhandled type code!"); 359 } 360 assert(ElementBitwidth != 0 && "Bad element bitwidth!"); 361 } 362 363 void RVVType::applyModifier(StringRef Transformer) { 364 if (Transformer.empty()) 365 return; 366 // Handle primitive type transformer 367 auto PType = Transformer.back(); 368 switch (PType) { 369 case 'e': 370 Scale = 0; 371 break; 372 case 'v': 373 Scale = LMUL.getScale(ElementBitwidth); 374 break; 375 case 'w': 376 ElementBitwidth *= 2; 377 LMUL *= 2; 378 Scale = LMUL.getScale(ElementBitwidth); 379 break; 380 case 'q': 381 ElementBitwidth *= 4; 382 LMUL *= 4; 383 Scale = LMUL.getScale(ElementBitwidth); 384 break; 385 case 'o': 386 ElementBitwidth *= 8; 387 LMUL *= 8; 388 Scale = LMUL.getScale(ElementBitwidth); 389 break; 390 case 'm': 391 ScalarType = ScalarTypeKind::Boolean; 392 Scale = LMUL.getScale(ElementBitwidth); 393 ElementBitwidth = 1; 394 break; 395 case '0': 396 ScalarType = ScalarTypeKind::Void; 397 break; 398 case 'z': 399 ScalarType = ScalarTypeKind::Size_t; 400 break; 401 case 't': 402 ScalarType = ScalarTypeKind::Ptrdiff_t; 403 break; 404 case 'u': 405 ScalarType = ScalarTypeKind::UnsignedLong; 406 break; 407 case 'l': 408 ScalarType = ScalarTypeKind::SignedLong; 409 break; 410 default: 411 llvm_unreachable("Illegal primitive type transformers!"); 412 } 413 Transformer = Transformer.drop_back(); 414 415 // Extract and compute complex type transformer. It can only appear one time. 416 if (Transformer.startswith("(")) { 417 size_t Idx = Transformer.find(')'); 418 assert(Idx != StringRef::npos); 419 StringRef ComplexType = Transformer.slice(1, Idx); 420 Transformer = Transformer.drop_front(Idx + 1); 421 assert(!Transformer.contains('(') && 422 "Only allow one complex type transformer"); 423 424 auto UpdateAndCheckComplexProto = [&]() { 425 Scale = LMUL.getScale(ElementBitwidth); 426 const StringRef VectorPrototypes("vwqom"); 427 if (!VectorPrototypes.contains(PType)) 428 llvm_unreachable("Complex type transformer only supports vector type!"); 429 if (Transformer.find_first_of("PCKWS") != StringRef::npos) 430 llvm_unreachable( 431 "Illegal type transformer for Complex type transformer"); 432 }; 433 auto ComputeFixedLog2LMUL = 434 [&](StringRef Value, 435 std::function<bool(const int32_t &, const int32_t &)> Compare) { 436 int32_t Log2LMUL; 437 Value.getAsInteger(10, Log2LMUL); 438 if (!Compare(Log2LMUL, LMUL.Log2LMUL)) { 439 ScalarType = Invalid; 440 return false; 441 } 442 // Update new LMUL 443 LMUL = LMULType(Log2LMUL); 444 UpdateAndCheckComplexProto(); 445 return true; 446 }; 447 auto ComplexTT = ComplexType.split(":"); 448 if (ComplexTT.first == "Log2EEW") { 449 uint32_t Log2EEW; 450 ComplexTT.second.getAsInteger(10, Log2EEW); 451 // update new elmul = (eew/sew) * lmul 452 LMUL.MulLog2LMUL(Log2EEW - Log2_32(ElementBitwidth)); 453 // update new eew 454 ElementBitwidth = 1 << Log2EEW; 455 ScalarType = ScalarTypeKind::SignedInteger; 456 UpdateAndCheckComplexProto(); 457 } else if (ComplexTT.first == "FixedSEW") { 458 uint32_t NewSEW; 459 ComplexTT.second.getAsInteger(10, NewSEW); 460 // Set invalid type if src and dst SEW are same. 461 if (ElementBitwidth == NewSEW) { 462 ScalarType = Invalid; 463 return; 464 } 465 // Update new SEW 466 ElementBitwidth = NewSEW; 467 UpdateAndCheckComplexProto(); 468 } else if (ComplexTT.first == "LFixedLog2LMUL") { 469 // New LMUL should be larger than old 470 if (!ComputeFixedLog2LMUL(ComplexTT.second, std::greater<int32_t>())) 471 return; 472 } else if (ComplexTT.first == "SFixedLog2LMUL") { 473 // New LMUL should be smaller than old 474 if (!ComputeFixedLog2LMUL(ComplexTT.second, std::less<int32_t>())) 475 return; 476 } else { 477 llvm_unreachable("Illegal complex type transformers!"); 478 } 479 } 480 481 // Compute the remain type transformers 482 for (char I : Transformer) { 483 switch (I) { 484 case 'P': 485 if (IsConstant) 486 llvm_unreachable("'P' transformer cannot be used after 'C'"); 487 if (IsPointer) 488 llvm_unreachable("'P' transformer cannot be used twice"); 489 IsPointer = true; 490 break; 491 case 'C': 492 if (IsConstant) 493 llvm_unreachable("'C' transformer cannot be used twice"); 494 IsConstant = true; 495 break; 496 case 'K': 497 IsImmediate = true; 498 break; 499 case 'U': 500 ScalarType = ScalarTypeKind::UnsignedInteger; 501 break; 502 case 'I': 503 ScalarType = ScalarTypeKind::SignedInteger; 504 break; 505 case 'F': 506 ScalarType = ScalarTypeKind::Float; 507 break; 508 case 'S': 509 LMUL = LMULType(0); 510 // Update ElementBitwidth need to update Scale too. 511 Scale = LMUL.getScale(ElementBitwidth); 512 break; 513 default: 514 llvm_unreachable("Illegal non-primitive type transformer!"); 515 } 516 } 517 } 518 519 //===----------------------------------------------------------------------===// 520 // RVVIntrinsic implementation 521 //===----------------------------------------------------------------------===// 522 RVVIntrinsic::RVVIntrinsic( 523 StringRef NewName, StringRef Suffix, StringRef NewMangledName, 524 StringRef MangledSuffix, StringRef IRName, bool IsMasked, 525 bool HasMaskedOffOperand, bool HasVL, PolicyScheme Scheme, 526 bool HasUnMaskedOverloaded, bool HasBuiltinAlias, StringRef ManualCodegen, 527 const RVVTypes &OutInTypes, const std::vector<int64_t> &NewIntrinsicTypes, 528 const std::vector<StringRef> &RequiredFeatures, unsigned NF) 529 : IRName(IRName), IsMasked(IsMasked), HasVL(HasVL), Scheme(Scheme), 530 HasUnMaskedOverloaded(HasUnMaskedOverloaded), 531 HasBuiltinAlias(HasBuiltinAlias), ManualCodegen(ManualCodegen.str()), 532 NF(NF) { 533 534 // Init BuiltinName, Name and MangledName 535 BuiltinName = NewName.str(); 536 Name = BuiltinName; 537 if (NewMangledName.empty()) 538 MangledName = NewName.split("_").first.str(); 539 else 540 MangledName = NewMangledName.str(); 541 if (!Suffix.empty()) 542 Name += "_" + Suffix.str(); 543 if (!MangledSuffix.empty()) 544 MangledName += "_" + MangledSuffix.str(); 545 if (IsMasked) { 546 BuiltinName += "_m"; 547 Name += "_m"; 548 } 549 550 // Init RISC-V extensions 551 for (const auto &T : OutInTypes) { 552 if (T->isFloatVector(16) || T->isFloat(16)) 553 RISCVPredefinedMacros |= RISCVPredefinedMacro::Zvfh; 554 if (T->isFloatVector(32)) 555 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELenFp32; 556 if (T->isFloatVector(64)) 557 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELenFp64; 558 if (T->isVector(64)) 559 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELen64; 560 } 561 for (auto Feature : RequiredFeatures) { 562 if (Feature == "RV64") 563 RISCVPredefinedMacros |= RISCVPredefinedMacro::RV64; 564 // Note: Full multiply instruction (mulh, mulhu, mulhsu, smul) for EEW=64 565 // require V. 566 if (Feature == "FullMultiply" && 567 (RISCVPredefinedMacros & RISCVPredefinedMacro::VectorMaxELen64)) 568 RISCVPredefinedMacros |= RISCVPredefinedMacro::V; 569 } 570 571 // Init OutputType and InputTypes 572 OutputType = OutInTypes[0]; 573 InputTypes.assign(OutInTypes.begin() + 1, OutInTypes.end()); 574 575 // IntrinsicTypes is unmasked TA version index. Need to update it 576 // if there is merge operand (It is always in first operand). 577 IntrinsicTypes = NewIntrinsicTypes; 578 if ((IsMasked && HasMaskedOffOperand) || 579 (!IsMasked && hasPassthruOperand())) { 580 for (auto &I : IntrinsicTypes) { 581 if (I >= 0) 582 I += NF; 583 } 584 } 585 } 586 587 std::string RVVIntrinsic::getBuiltinTypeStr() const { 588 std::string S; 589 S += OutputType->getBuiltinStr(); 590 for (const auto &T : InputTypes) { 591 S += T->getBuiltinStr(); 592 } 593 return S; 594 } 595 596 } // end namespace RISCV 597 } // end namespace clang 598