1 //===- RISCVVIntrinsicUtils.cpp - RISC-V Vector Intrinsic Utils -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Support/RISCVVIntrinsicUtils.h" 10 #include "llvm/ADT/ArrayRef.h" 11 #include "llvm/ADT/Optional.h" 12 #include "llvm/ADT/SmallSet.h" 13 #include "llvm/ADT/StringExtras.h" 14 #include "llvm/ADT/StringMap.h" 15 #include "llvm/ADT/StringSet.h" 16 #include "llvm/ADT/Twine.h" 17 #include "llvm/Support/raw_ostream.h" 18 #include <numeric> 19 #include <set> 20 #include <unordered_map> 21 22 using namespace llvm; 23 24 namespace clang { 25 namespace RISCV { 26 27 const PrototypeDescriptor PrototypeDescriptor::Mask = PrototypeDescriptor( 28 BaseTypeModifier::Vector, VectorTypeModifier::MaskVector); 29 const PrototypeDescriptor PrototypeDescriptor::VL = 30 PrototypeDescriptor(BaseTypeModifier::SizeT); 31 const PrototypeDescriptor PrototypeDescriptor::Vector = 32 PrototypeDescriptor(BaseTypeModifier::Vector); 33 34 // Concat BasicType, LMUL and Proto as key 35 static std::unordered_map<uint64_t, RVVType> LegalTypes; 36 static std::set<uint64_t> IllegalTypes; 37 38 //===----------------------------------------------------------------------===// 39 // Type implementation 40 //===----------------------------------------------------------------------===// 41 42 LMULType::LMULType(int NewLog2LMUL) { 43 // Check Log2LMUL is -3, -2, -1, 0, 1, 2, 3 44 assert(NewLog2LMUL <= 3 && NewLog2LMUL >= -3 && "Bad LMUL number!"); 45 Log2LMUL = NewLog2LMUL; 46 } 47 48 std::string LMULType::str() const { 49 if (Log2LMUL < 0) 50 return "mf" + utostr(1ULL << (-Log2LMUL)); 51 return "m" + utostr(1ULL << Log2LMUL); 52 } 53 54 VScaleVal LMULType::getScale(unsigned ElementBitwidth) const { 55 int Log2ScaleResult = 0; 56 switch (ElementBitwidth) { 57 default: 58 break; 59 case 8: 60 Log2ScaleResult = Log2LMUL + 3; 61 break; 62 case 16: 63 Log2ScaleResult = Log2LMUL + 2; 64 break; 65 case 32: 66 Log2ScaleResult = Log2LMUL + 1; 67 break; 68 case 64: 69 Log2ScaleResult = Log2LMUL; 70 break; 71 } 72 // Illegal vscale result would be less than 1 73 if (Log2ScaleResult < 0) 74 return llvm::None; 75 return 1 << Log2ScaleResult; 76 } 77 78 void LMULType::MulLog2LMUL(int log2LMUL) { Log2LMUL += log2LMUL; } 79 80 LMULType &LMULType::operator*=(uint32_t RHS) { 81 assert(isPowerOf2_32(RHS)); 82 this->Log2LMUL = this->Log2LMUL + Log2_32(RHS); 83 return *this; 84 } 85 86 RVVType::RVVType(BasicType BT, int Log2LMUL, 87 const PrototypeDescriptor &prototype) 88 : BT(BT), LMUL(LMULType(Log2LMUL)) { 89 applyBasicType(); 90 applyModifier(prototype); 91 Valid = verifyType(); 92 if (Valid) { 93 initBuiltinStr(); 94 initTypeStr(); 95 if (isVector()) { 96 initClangBuiltinStr(); 97 } 98 } 99 } 100 101 // clang-format off 102 // boolean type are encoded the ratio of n (SEW/LMUL) 103 // SEW/LMUL | 1 | 2 | 4 | 8 | 16 | 32 | 64 104 // c type | vbool64_t | vbool32_t | vbool16_t | vbool8_t | vbool4_t | vbool2_t | vbool1_t 105 // IR type | nxv1i1 | nxv2i1 | nxv4i1 | nxv8i1 | nxv16i1 | nxv32i1 | nxv64i1 106 107 // type\lmul | 1/8 | 1/4 | 1/2 | 1 | 2 | 4 | 8 108 // -------- |------ | -------- | ------- | ------- | -------- | -------- | -------- 109 // i64 | N/A | N/A | N/A | nxv1i64 | nxv2i64 | nxv4i64 | nxv8i64 110 // i32 | N/A | N/A | nxv1i32 | nxv2i32 | nxv4i32 | nxv8i32 | nxv16i32 111 // i16 | N/A | nxv1i16 | nxv2i16 | nxv4i16 | nxv8i16 | nxv16i16 | nxv32i16 112 // i8 | nxv1i8 | nxv2i8 | nxv4i8 | nxv8i8 | nxv16i8 | nxv32i8 | nxv64i8 113 // double | N/A | N/A | N/A | nxv1f64 | nxv2f64 | nxv4f64 | nxv8f64 114 // float | N/A | N/A | nxv1f32 | nxv2f32 | nxv4f32 | nxv8f32 | nxv16f32 115 // half | N/A | nxv1f16 | nxv2f16 | nxv4f16 | nxv8f16 | nxv16f16 | nxv32f16 116 // clang-format on 117 118 bool RVVType::verifyType() const { 119 if (ScalarType == Invalid) 120 return false; 121 if (isScalar()) 122 return true; 123 if (!Scale.hasValue()) 124 return false; 125 if (isFloat() && ElementBitwidth == 8) 126 return false; 127 unsigned V = Scale.getValue(); 128 switch (ElementBitwidth) { 129 case 1: 130 case 8: 131 // Check Scale is 1,2,4,8,16,32,64 132 return (V <= 64 && isPowerOf2_32(V)); 133 case 16: 134 // Check Scale is 1,2,4,8,16,32 135 return (V <= 32 && isPowerOf2_32(V)); 136 case 32: 137 // Check Scale is 1,2,4,8,16 138 return (V <= 16 && isPowerOf2_32(V)); 139 case 64: 140 // Check Scale is 1,2,4,8 141 return (V <= 8 && isPowerOf2_32(V)); 142 } 143 return false; 144 } 145 146 void RVVType::initBuiltinStr() { 147 assert(isValid() && "RVVType is invalid"); 148 switch (ScalarType) { 149 case ScalarTypeKind::Void: 150 BuiltinStr = "v"; 151 return; 152 case ScalarTypeKind::Size_t: 153 BuiltinStr = "z"; 154 if (IsImmediate) 155 BuiltinStr = "I" + BuiltinStr; 156 if (IsPointer) 157 BuiltinStr += "*"; 158 return; 159 case ScalarTypeKind::Ptrdiff_t: 160 BuiltinStr = "Y"; 161 return; 162 case ScalarTypeKind::UnsignedLong: 163 BuiltinStr = "ULi"; 164 return; 165 case ScalarTypeKind::SignedLong: 166 BuiltinStr = "Li"; 167 return; 168 case ScalarTypeKind::Boolean: 169 assert(ElementBitwidth == 1); 170 BuiltinStr += "b"; 171 break; 172 case ScalarTypeKind::SignedInteger: 173 case ScalarTypeKind::UnsignedInteger: 174 switch (ElementBitwidth) { 175 case 8: 176 BuiltinStr += "c"; 177 break; 178 case 16: 179 BuiltinStr += "s"; 180 break; 181 case 32: 182 BuiltinStr += "i"; 183 break; 184 case 64: 185 BuiltinStr += "Wi"; 186 break; 187 default: 188 llvm_unreachable("Unhandled ElementBitwidth!"); 189 } 190 if (isSignedInteger()) 191 BuiltinStr = "S" + BuiltinStr; 192 else 193 BuiltinStr = "U" + BuiltinStr; 194 break; 195 case ScalarTypeKind::Float: 196 switch (ElementBitwidth) { 197 case 16: 198 BuiltinStr += "x"; 199 break; 200 case 32: 201 BuiltinStr += "f"; 202 break; 203 case 64: 204 BuiltinStr += "d"; 205 break; 206 default: 207 llvm_unreachable("Unhandled ElementBitwidth!"); 208 } 209 break; 210 default: 211 llvm_unreachable("ScalarType is invalid!"); 212 } 213 if (IsImmediate) 214 BuiltinStr = "I" + BuiltinStr; 215 if (isScalar()) { 216 if (IsConstant) 217 BuiltinStr += "C"; 218 if (IsPointer) 219 BuiltinStr += "*"; 220 return; 221 } 222 BuiltinStr = "q" + utostr(Scale.getValue()) + BuiltinStr; 223 // Pointer to vector types. Defined for segment load intrinsics. 224 // segment load intrinsics have pointer type arguments to store the loaded 225 // vector values. 226 if (IsPointer) 227 BuiltinStr += "*"; 228 } 229 230 void RVVType::initClangBuiltinStr() { 231 assert(isValid() && "RVVType is invalid"); 232 assert(isVector() && "Handle Vector type only"); 233 234 ClangBuiltinStr = "__rvv_"; 235 switch (ScalarType) { 236 case ScalarTypeKind::Boolean: 237 ClangBuiltinStr += "bool" + utostr(64 / Scale.getValue()) + "_t"; 238 return; 239 case ScalarTypeKind::Float: 240 ClangBuiltinStr += "float"; 241 break; 242 case ScalarTypeKind::SignedInteger: 243 ClangBuiltinStr += "int"; 244 break; 245 case ScalarTypeKind::UnsignedInteger: 246 ClangBuiltinStr += "uint"; 247 break; 248 default: 249 llvm_unreachable("ScalarTypeKind is invalid"); 250 } 251 ClangBuiltinStr += utostr(ElementBitwidth) + LMUL.str() + "_t"; 252 } 253 254 void RVVType::initTypeStr() { 255 assert(isValid() && "RVVType is invalid"); 256 257 if (IsConstant) 258 Str += "const "; 259 260 auto getTypeString = [&](StringRef TypeStr) { 261 if (isScalar()) 262 return Twine(TypeStr + Twine(ElementBitwidth) + "_t").str(); 263 return Twine("v" + TypeStr + Twine(ElementBitwidth) + LMUL.str() + "_t") 264 .str(); 265 }; 266 267 switch (ScalarType) { 268 case ScalarTypeKind::Void: 269 Str = "void"; 270 return; 271 case ScalarTypeKind::Size_t: 272 Str = "size_t"; 273 if (IsPointer) 274 Str += " *"; 275 return; 276 case ScalarTypeKind::Ptrdiff_t: 277 Str = "ptrdiff_t"; 278 return; 279 case ScalarTypeKind::UnsignedLong: 280 Str = "unsigned long"; 281 return; 282 case ScalarTypeKind::SignedLong: 283 Str = "long"; 284 return; 285 case ScalarTypeKind::Boolean: 286 if (isScalar()) 287 Str += "bool"; 288 else 289 // Vector bool is special case, the formulate is 290 // `vbool<N>_t = MVT::nxv<64/N>i1` ex. vbool16_t = MVT::4i1 291 Str += "vbool" + utostr(64 / Scale.getValue()) + "_t"; 292 break; 293 case ScalarTypeKind::Float: 294 if (isScalar()) { 295 if (ElementBitwidth == 64) 296 Str += "double"; 297 else if (ElementBitwidth == 32) 298 Str += "float"; 299 else if (ElementBitwidth == 16) 300 Str += "_Float16"; 301 else 302 llvm_unreachable("Unhandled floating type."); 303 } else 304 Str += getTypeString("float"); 305 break; 306 case ScalarTypeKind::SignedInteger: 307 Str += getTypeString("int"); 308 break; 309 case ScalarTypeKind::UnsignedInteger: 310 Str += getTypeString("uint"); 311 break; 312 default: 313 llvm_unreachable("ScalarType is invalid!"); 314 } 315 if (IsPointer) 316 Str += " *"; 317 } 318 319 void RVVType::initShortStr() { 320 switch (ScalarType) { 321 case ScalarTypeKind::Boolean: 322 assert(isVector()); 323 ShortStr = "b" + utostr(64 / Scale.getValue()); 324 return; 325 case ScalarTypeKind::Float: 326 ShortStr = "f" + utostr(ElementBitwidth); 327 break; 328 case ScalarTypeKind::SignedInteger: 329 ShortStr = "i" + utostr(ElementBitwidth); 330 break; 331 case ScalarTypeKind::UnsignedInteger: 332 ShortStr = "u" + utostr(ElementBitwidth); 333 break; 334 default: 335 llvm_unreachable("Unhandled case!"); 336 } 337 if (isVector()) 338 ShortStr += LMUL.str(); 339 } 340 341 void RVVType::applyBasicType() { 342 switch (BT) { 343 case BasicType::Int8: 344 ElementBitwidth = 8; 345 ScalarType = ScalarTypeKind::SignedInteger; 346 break; 347 case BasicType::Int16: 348 ElementBitwidth = 16; 349 ScalarType = ScalarTypeKind::SignedInteger; 350 break; 351 case BasicType::Int32: 352 ElementBitwidth = 32; 353 ScalarType = ScalarTypeKind::SignedInteger; 354 break; 355 case BasicType::Int64: 356 ElementBitwidth = 64; 357 ScalarType = ScalarTypeKind::SignedInteger; 358 break; 359 case BasicType::Float16: 360 ElementBitwidth = 16; 361 ScalarType = ScalarTypeKind::Float; 362 break; 363 case BasicType::Float32: 364 ElementBitwidth = 32; 365 ScalarType = ScalarTypeKind::Float; 366 break; 367 case BasicType::Float64: 368 ElementBitwidth = 64; 369 ScalarType = ScalarTypeKind::Float; 370 break; 371 default: 372 llvm_unreachable("Unhandled type code!"); 373 } 374 assert(ElementBitwidth != 0 && "Bad element bitwidth!"); 375 } 376 377 Optional<PrototypeDescriptor> PrototypeDescriptor::parsePrototypeDescriptor( 378 llvm::StringRef PrototypeDescriptorStr) { 379 PrototypeDescriptor PD; 380 BaseTypeModifier PT = BaseTypeModifier::Invalid; 381 VectorTypeModifier VTM = VectorTypeModifier::NoModifier; 382 383 if (PrototypeDescriptorStr.empty()) 384 return PD; 385 386 // Handle base type modifier 387 auto PType = PrototypeDescriptorStr.back(); 388 switch (PType) { 389 case 'e': 390 PT = BaseTypeModifier::Scalar; 391 break; 392 case 'v': 393 PT = BaseTypeModifier::Vector; 394 break; 395 case 'w': 396 PT = BaseTypeModifier::Vector; 397 VTM = VectorTypeModifier::Widening2XVector; 398 break; 399 case 'q': 400 PT = BaseTypeModifier::Vector; 401 VTM = VectorTypeModifier::Widening4XVector; 402 break; 403 case 'o': 404 PT = BaseTypeModifier::Vector; 405 VTM = VectorTypeModifier::Widening8XVector; 406 break; 407 case 'm': 408 PT = BaseTypeModifier::Vector; 409 VTM = VectorTypeModifier::MaskVector; 410 break; 411 case '0': 412 PT = BaseTypeModifier::Void; 413 break; 414 case 'z': 415 PT = BaseTypeModifier::SizeT; 416 break; 417 case 't': 418 PT = BaseTypeModifier::Ptrdiff; 419 break; 420 case 'u': 421 PT = BaseTypeModifier::UnsignedLong; 422 break; 423 case 'l': 424 PT = BaseTypeModifier::SignedLong; 425 break; 426 default: 427 llvm_unreachable("Illegal primitive type transformers!"); 428 } 429 PD.PT = static_cast<uint8_t>(PT); 430 PrototypeDescriptorStr = PrototypeDescriptorStr.drop_back(); 431 432 // Compute the vector type transformers, it can only appear one time. 433 if (PrototypeDescriptorStr.startswith("(")) { 434 assert(VTM == VectorTypeModifier::NoModifier && 435 "VectorTypeModifier should only have one modifier"); 436 size_t Idx = PrototypeDescriptorStr.find(')'); 437 assert(Idx != StringRef::npos); 438 StringRef ComplexType = PrototypeDescriptorStr.slice(1, Idx); 439 PrototypeDescriptorStr = PrototypeDescriptorStr.drop_front(Idx + 1); 440 assert(!PrototypeDescriptorStr.contains('(') && 441 "Only allow one vector type modifier"); 442 443 auto ComplexTT = ComplexType.split(":"); 444 if (ComplexTT.first == "Log2EEW") { 445 uint32_t Log2EEW; 446 if (ComplexTT.second.getAsInteger(10, Log2EEW)) { 447 llvm_unreachable("Invalid Log2EEW value!"); 448 return None; 449 } 450 switch (Log2EEW) { 451 case 3: 452 VTM = VectorTypeModifier::Log2EEW3; 453 break; 454 case 4: 455 VTM = VectorTypeModifier::Log2EEW4; 456 break; 457 case 5: 458 VTM = VectorTypeModifier::Log2EEW5; 459 break; 460 case 6: 461 VTM = VectorTypeModifier::Log2EEW6; 462 break; 463 default: 464 llvm_unreachable("Invalid Log2EEW value, should be [3-6]"); 465 return None; 466 } 467 } else if (ComplexTT.first == "FixedSEW") { 468 uint32_t NewSEW; 469 if (ComplexTT.second.getAsInteger(10, NewSEW)) { 470 llvm_unreachable("Invalid FixedSEW value!"); 471 return None; 472 } 473 switch (NewSEW) { 474 case 8: 475 VTM = VectorTypeModifier::FixedSEW8; 476 break; 477 case 16: 478 VTM = VectorTypeModifier::FixedSEW16; 479 break; 480 case 32: 481 VTM = VectorTypeModifier::FixedSEW32; 482 break; 483 case 64: 484 VTM = VectorTypeModifier::FixedSEW64; 485 break; 486 default: 487 llvm_unreachable("Invalid FixedSEW value, should be 8, 16, 32 or 64"); 488 return None; 489 } 490 } else if (ComplexTT.first == "LFixedLog2LMUL") { 491 int32_t Log2LMUL; 492 if (ComplexTT.second.getAsInteger(10, Log2LMUL)) { 493 llvm_unreachable("Invalid LFixedLog2LMUL value!"); 494 return None; 495 } 496 switch (Log2LMUL) { 497 case -3: 498 VTM = VectorTypeModifier::LFixedLog2LMULN3; 499 break; 500 case -2: 501 VTM = VectorTypeModifier::LFixedLog2LMULN2; 502 break; 503 case -1: 504 VTM = VectorTypeModifier::LFixedLog2LMULN1; 505 break; 506 case 0: 507 VTM = VectorTypeModifier::LFixedLog2LMUL0; 508 break; 509 case 1: 510 VTM = VectorTypeModifier::LFixedLog2LMUL1; 511 break; 512 case 2: 513 VTM = VectorTypeModifier::LFixedLog2LMUL2; 514 break; 515 case 3: 516 VTM = VectorTypeModifier::LFixedLog2LMUL3; 517 break; 518 default: 519 llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]"); 520 return None; 521 } 522 } else if (ComplexTT.first == "SFixedLog2LMUL") { 523 int32_t Log2LMUL; 524 if (ComplexTT.second.getAsInteger(10, Log2LMUL)) { 525 llvm_unreachable("Invalid SFixedLog2LMUL value!"); 526 return None; 527 } 528 switch (Log2LMUL) { 529 case -3: 530 VTM = VectorTypeModifier::SFixedLog2LMULN3; 531 break; 532 case -2: 533 VTM = VectorTypeModifier::SFixedLog2LMULN2; 534 break; 535 case -1: 536 VTM = VectorTypeModifier::SFixedLog2LMULN1; 537 break; 538 case 0: 539 VTM = VectorTypeModifier::SFixedLog2LMUL0; 540 break; 541 case 1: 542 VTM = VectorTypeModifier::SFixedLog2LMUL1; 543 break; 544 case 2: 545 VTM = VectorTypeModifier::SFixedLog2LMUL2; 546 break; 547 case 3: 548 VTM = VectorTypeModifier::SFixedLog2LMUL3; 549 break; 550 default: 551 llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]"); 552 return None; 553 } 554 555 } else { 556 llvm_unreachable("Illegal complex type transformers!"); 557 } 558 } 559 PD.VTM = static_cast<uint8_t>(VTM); 560 561 // Compute the remain type transformers 562 TypeModifier TM = TypeModifier::NoModifier; 563 for (char I : PrototypeDescriptorStr) { 564 switch (I) { 565 case 'P': 566 if ((TM & TypeModifier::Const) == TypeModifier::Const) 567 llvm_unreachable("'P' transformer cannot be used after 'C'"); 568 if ((TM & TypeModifier::Pointer) == TypeModifier::Pointer) 569 llvm_unreachable("'P' transformer cannot be used twice"); 570 TM |= TypeModifier::Pointer; 571 break; 572 case 'C': 573 TM |= TypeModifier::Const; 574 break; 575 case 'K': 576 TM |= TypeModifier::Immediate; 577 break; 578 case 'U': 579 TM |= TypeModifier::UnsignedInteger; 580 break; 581 case 'I': 582 TM |= TypeModifier::SignedInteger; 583 break; 584 case 'F': 585 TM |= TypeModifier::Float; 586 break; 587 case 'S': 588 TM |= TypeModifier::LMUL1; 589 break; 590 default: 591 llvm_unreachable("Illegal non-primitive type transformer!"); 592 } 593 } 594 PD.TM = static_cast<uint8_t>(TM); 595 596 return PD; 597 } 598 599 void RVVType::applyModifier(const PrototypeDescriptor &Transformer) { 600 // Handle primitive type transformer 601 switch (static_cast<BaseTypeModifier>(Transformer.PT)) { 602 case BaseTypeModifier::Scalar: 603 Scale = 0; 604 break; 605 case BaseTypeModifier::Vector: 606 Scale = LMUL.getScale(ElementBitwidth); 607 break; 608 case BaseTypeModifier::Void: 609 ScalarType = ScalarTypeKind::Void; 610 break; 611 case BaseTypeModifier::SizeT: 612 ScalarType = ScalarTypeKind::Size_t; 613 break; 614 case BaseTypeModifier::Ptrdiff: 615 ScalarType = ScalarTypeKind::Ptrdiff_t; 616 break; 617 case BaseTypeModifier::UnsignedLong: 618 ScalarType = ScalarTypeKind::UnsignedLong; 619 break; 620 case BaseTypeModifier::SignedLong: 621 ScalarType = ScalarTypeKind::SignedLong; 622 break; 623 case BaseTypeModifier::Invalid: 624 ScalarType = ScalarTypeKind::Invalid; 625 return; 626 } 627 628 switch (static_cast<VectorTypeModifier>(Transformer.VTM)) { 629 case VectorTypeModifier::Widening2XVector: 630 ElementBitwidth *= 2; 631 LMUL *= 2; 632 Scale = LMUL.getScale(ElementBitwidth); 633 break; 634 case VectorTypeModifier::Widening4XVector: 635 ElementBitwidth *= 4; 636 LMUL *= 4; 637 Scale = LMUL.getScale(ElementBitwidth); 638 break; 639 case VectorTypeModifier::Widening8XVector: 640 ElementBitwidth *= 8; 641 LMUL *= 8; 642 Scale = LMUL.getScale(ElementBitwidth); 643 break; 644 case VectorTypeModifier::MaskVector: 645 ScalarType = ScalarTypeKind::Boolean; 646 Scale = LMUL.getScale(ElementBitwidth); 647 ElementBitwidth = 1; 648 break; 649 case VectorTypeModifier::Log2EEW3: 650 applyLog2EEW(3); 651 break; 652 case VectorTypeModifier::Log2EEW4: 653 applyLog2EEW(4); 654 break; 655 case VectorTypeModifier::Log2EEW5: 656 applyLog2EEW(5); 657 break; 658 case VectorTypeModifier::Log2EEW6: 659 applyLog2EEW(6); 660 break; 661 case VectorTypeModifier::FixedSEW8: 662 applyFixedSEW(8); 663 break; 664 case VectorTypeModifier::FixedSEW16: 665 applyFixedSEW(16); 666 break; 667 case VectorTypeModifier::FixedSEW32: 668 applyFixedSEW(32); 669 break; 670 case VectorTypeModifier::FixedSEW64: 671 applyFixedSEW(64); 672 break; 673 case VectorTypeModifier::LFixedLog2LMULN3: 674 applyFixedLog2LMUL(-3, FixedLMULType::LargerThan); 675 break; 676 case VectorTypeModifier::LFixedLog2LMULN2: 677 applyFixedLog2LMUL(-2, FixedLMULType::LargerThan); 678 break; 679 case VectorTypeModifier::LFixedLog2LMULN1: 680 applyFixedLog2LMUL(-1, FixedLMULType::LargerThan); 681 break; 682 case VectorTypeModifier::LFixedLog2LMUL0: 683 applyFixedLog2LMUL(0, FixedLMULType::LargerThan); 684 break; 685 case VectorTypeModifier::LFixedLog2LMUL1: 686 applyFixedLog2LMUL(1, FixedLMULType::LargerThan); 687 break; 688 case VectorTypeModifier::LFixedLog2LMUL2: 689 applyFixedLog2LMUL(2, FixedLMULType::LargerThan); 690 break; 691 case VectorTypeModifier::LFixedLog2LMUL3: 692 applyFixedLog2LMUL(3, FixedLMULType::LargerThan); 693 break; 694 case VectorTypeModifier::SFixedLog2LMULN3: 695 applyFixedLog2LMUL(-3, FixedLMULType::SmallerThan); 696 break; 697 case VectorTypeModifier::SFixedLog2LMULN2: 698 applyFixedLog2LMUL(-2, FixedLMULType::SmallerThan); 699 break; 700 case VectorTypeModifier::SFixedLog2LMULN1: 701 applyFixedLog2LMUL(-1, FixedLMULType::SmallerThan); 702 break; 703 case VectorTypeModifier::SFixedLog2LMUL0: 704 applyFixedLog2LMUL(0, FixedLMULType::SmallerThan); 705 break; 706 case VectorTypeModifier::SFixedLog2LMUL1: 707 applyFixedLog2LMUL(1, FixedLMULType::SmallerThan); 708 break; 709 case VectorTypeModifier::SFixedLog2LMUL2: 710 applyFixedLog2LMUL(2, FixedLMULType::SmallerThan); 711 break; 712 case VectorTypeModifier::SFixedLog2LMUL3: 713 applyFixedLog2LMUL(3, FixedLMULType::SmallerThan); 714 break; 715 case VectorTypeModifier::NoModifier: 716 break; 717 } 718 719 for (unsigned TypeModifierMaskShift = 0; 720 TypeModifierMaskShift <= static_cast<unsigned>(TypeModifier::MaxOffset); 721 ++TypeModifierMaskShift) { 722 unsigned TypeModifierMask = 1 << TypeModifierMaskShift; 723 if ((static_cast<unsigned>(Transformer.TM) & TypeModifierMask) != 724 TypeModifierMask) 725 continue; 726 switch (static_cast<TypeModifier>(TypeModifierMask)) { 727 case TypeModifier::Pointer: 728 IsPointer = true; 729 break; 730 case TypeModifier::Const: 731 IsConstant = true; 732 break; 733 case TypeModifier::Immediate: 734 IsImmediate = true; 735 IsConstant = true; 736 break; 737 case TypeModifier::UnsignedInteger: 738 ScalarType = ScalarTypeKind::UnsignedInteger; 739 break; 740 case TypeModifier::SignedInteger: 741 ScalarType = ScalarTypeKind::SignedInteger; 742 break; 743 case TypeModifier::Float: 744 ScalarType = ScalarTypeKind::Float; 745 break; 746 case TypeModifier::LMUL1: 747 LMUL = LMULType(0); 748 // Update ElementBitwidth need to update Scale too. 749 Scale = LMUL.getScale(ElementBitwidth); 750 break; 751 default: 752 llvm_unreachable("Unknown type modifier mask!"); 753 } 754 } 755 } 756 757 void RVVType::applyLog2EEW(unsigned Log2EEW) { 758 // update new elmul = (eew/sew) * lmul 759 LMUL.MulLog2LMUL(Log2EEW - Log2_32(ElementBitwidth)); 760 // update new eew 761 ElementBitwidth = 1 << Log2EEW; 762 ScalarType = ScalarTypeKind::SignedInteger; 763 Scale = LMUL.getScale(ElementBitwidth); 764 } 765 766 void RVVType::applyFixedSEW(unsigned NewSEW) { 767 // Set invalid type if src and dst SEW are same. 768 if (ElementBitwidth == NewSEW) { 769 ScalarType = ScalarTypeKind::Invalid; 770 return; 771 } 772 // Update new SEW 773 ElementBitwidth = NewSEW; 774 Scale = LMUL.getScale(ElementBitwidth); 775 } 776 777 void RVVType::applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type) { 778 switch (Type) { 779 case FixedLMULType::LargerThan: 780 if (Log2LMUL < LMUL.Log2LMUL) { 781 ScalarType = ScalarTypeKind::Invalid; 782 return; 783 } 784 break; 785 case FixedLMULType::SmallerThan: 786 if (Log2LMUL > LMUL.Log2LMUL) { 787 ScalarType = ScalarTypeKind::Invalid; 788 return; 789 } 790 break; 791 } 792 793 // Update new LMUL 794 LMUL = LMULType(Log2LMUL); 795 Scale = LMUL.getScale(ElementBitwidth); 796 } 797 798 Optional<RVVTypes> 799 RVVType::computeTypes(BasicType BT, int Log2LMUL, unsigned NF, 800 ArrayRef<PrototypeDescriptor> PrototypeSeq) { 801 // LMUL x NF must be less than or equal to 8. 802 if ((Log2LMUL >= 1) && (1 << Log2LMUL) * NF > 8) 803 return llvm::None; 804 805 RVVTypes Types; 806 for (const PrototypeDescriptor &Proto : PrototypeSeq) { 807 auto T = computeType(BT, Log2LMUL, Proto); 808 if (!T.hasValue()) 809 return llvm::None; 810 // Record legal type index 811 Types.push_back(T.getValue()); 812 } 813 return Types; 814 } 815 816 // Compute the hash value of RVVType, used for cache the result of computeType. 817 static uint64_t computeRVVTypeHashValue(BasicType BT, int Log2LMUL, 818 PrototypeDescriptor Proto) { 819 // Layout of hash value: 820 // 0 8 16 24 32 40 821 // | Log2LMUL + 3 | BT | Proto.PT | Proto.TM | Proto.VTM | 822 assert(Log2LMUL >= -3 && Log2LMUL <= 3); 823 return (Log2LMUL + 3) | (static_cast<uint64_t>(BT) & 0xff) << 8 | 824 ((uint64_t)(Proto.PT & 0xff) << 16) | 825 ((uint64_t)(Proto.TM & 0xff) << 24) | 826 ((uint64_t)(Proto.VTM & 0xff) << 32); 827 } 828 829 Optional<RVVTypePtr> RVVType::computeType(BasicType BT, int Log2LMUL, 830 PrototypeDescriptor Proto) { 831 uint64_t Idx = computeRVVTypeHashValue(BT, Log2LMUL, Proto); 832 // Search first 833 auto It = LegalTypes.find(Idx); 834 if (It != LegalTypes.end()) 835 return &(It->second); 836 837 if (IllegalTypes.count(Idx)) 838 return llvm::None; 839 840 // Compute type and record the result. 841 RVVType T(BT, Log2LMUL, Proto); 842 if (T.isValid()) { 843 // Record legal type index and value. 844 LegalTypes.insert({Idx, T}); 845 return &(LegalTypes[Idx]); 846 } 847 // Record illegal type index. 848 IllegalTypes.insert(Idx); 849 return llvm::None; 850 } 851 852 //===----------------------------------------------------------------------===// 853 // RVVIntrinsic implementation 854 //===----------------------------------------------------------------------===// 855 RVVIntrinsic::RVVIntrinsic( 856 StringRef NewName, StringRef Suffix, StringRef NewMangledName, 857 StringRef MangledSuffix, StringRef IRName, bool IsMasked, 858 bool HasMaskedOffOperand, bool HasVL, PolicyScheme Scheme, 859 bool HasUnMaskedOverloaded, bool HasBuiltinAlias, StringRef ManualCodegen, 860 const RVVTypes &OutInTypes, const std::vector<int64_t> &NewIntrinsicTypes, 861 const std::vector<StringRef> &RequiredFeatures, unsigned NF) 862 : IRName(IRName), IsMasked(IsMasked), HasVL(HasVL), Scheme(Scheme), 863 HasUnMaskedOverloaded(HasUnMaskedOverloaded), 864 HasBuiltinAlias(HasBuiltinAlias), ManualCodegen(ManualCodegen.str()), 865 NF(NF) { 866 867 // Init BuiltinName, Name and MangledName 868 BuiltinName = NewName.str(); 869 Name = BuiltinName; 870 if (NewMangledName.empty()) 871 MangledName = NewName.split("_").first.str(); 872 else 873 MangledName = NewMangledName.str(); 874 if (!Suffix.empty()) 875 Name += "_" + Suffix.str(); 876 if (!MangledSuffix.empty()) 877 MangledName += "_" + MangledSuffix.str(); 878 if (IsMasked) { 879 BuiltinName += "_m"; 880 Name += "_m"; 881 } 882 883 // Init RISC-V extensions 884 for (const auto &T : OutInTypes) { 885 if (T->isFloatVector(16) || T->isFloat(16)) 886 RISCVPredefinedMacros |= RISCVPredefinedMacro::Zvfh; 887 if (T->isFloatVector(32)) 888 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELenFp32; 889 if (T->isFloatVector(64)) 890 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELenFp64; 891 if (T->isVector(64)) 892 RISCVPredefinedMacros |= RISCVPredefinedMacro::VectorMaxELen64; 893 } 894 for (auto Feature : RequiredFeatures) { 895 if (Feature == "RV64") 896 RISCVPredefinedMacros |= RISCVPredefinedMacro::RV64; 897 // Note: Full multiply instruction (mulh, mulhu, mulhsu, smul) for EEW=64 898 // require V. 899 if (Feature == "FullMultiply" && 900 (RISCVPredefinedMacros & RISCVPredefinedMacro::VectorMaxELen64)) 901 RISCVPredefinedMacros |= RISCVPredefinedMacro::V; 902 } 903 904 // Init OutputType and InputTypes 905 OutputType = OutInTypes[0]; 906 InputTypes.assign(OutInTypes.begin() + 1, OutInTypes.end()); 907 908 // IntrinsicTypes is unmasked TA version index. Need to update it 909 // if there is merge operand (It is always in first operand). 910 IntrinsicTypes = NewIntrinsicTypes; 911 if ((IsMasked && HasMaskedOffOperand) || 912 (!IsMasked && hasPassthruOperand())) { 913 for (auto &I : IntrinsicTypes) { 914 if (I >= 0) 915 I += NF; 916 } 917 } 918 } 919 920 std::string RVVIntrinsic::getBuiltinTypeStr() const { 921 std::string S; 922 S += OutputType->getBuiltinStr(); 923 for (const auto &T : InputTypes) { 924 S += T->getBuiltinStr(); 925 } 926 return S; 927 } 928 929 std::string RVVIntrinsic::getSuffixStr( 930 BasicType Type, int Log2LMUL, 931 const llvm::SmallVector<PrototypeDescriptor> &PrototypeDescriptors) { 932 SmallVector<std::string> SuffixStrs; 933 for (auto PD : PrototypeDescriptors) { 934 auto T = RVVType::computeType(Type, Log2LMUL, PD); 935 SuffixStrs.push_back(T.getValue()->getShortStr()); 936 } 937 return join(SuffixStrs, "_"); 938 } 939 940 SmallVector<PrototypeDescriptor> parsePrototypes(StringRef Prototypes) { 941 SmallVector<PrototypeDescriptor> PrototypeDescriptors; 942 const StringRef Primaries("evwqom0ztul"); 943 while (!Prototypes.empty()) { 944 size_t Idx = 0; 945 // Skip over complex prototype because it could contain primitive type 946 // character. 947 if (Prototypes[0] == '(') 948 Idx = Prototypes.find_first_of(')'); 949 Idx = Prototypes.find_first_of(Primaries, Idx); 950 assert(Idx != StringRef::npos); 951 auto PD = PrototypeDescriptor::parsePrototypeDescriptor( 952 Prototypes.slice(0, Idx + 1)); 953 if (!PD) 954 llvm_unreachable("Error during parsing prototype."); 955 PrototypeDescriptors.push_back(*PD); 956 Prototypes = Prototypes.drop_front(Idx + 1); 957 } 958 return PrototypeDescriptors; 959 } 960 961 } // end namespace RISCV 962 } // end namespace clang 963