1 //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This implements the TargetLoweringBase class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/BitVector.h" 14 #include "llvm/ADT/STLExtras.h" 15 #include "llvm/ADT/SmallVector.h" 16 #include "llvm/ADT/StringExtras.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/ADT/Triple.h" 19 #include "llvm/ADT/Twine.h" 20 #include "llvm/Analysis/Loads.h" 21 #include "llvm/Analysis/TargetTransformInfo.h" 22 #include "llvm/CodeGen/Analysis.h" 23 #include "llvm/CodeGen/ISDOpcodes.h" 24 #include "llvm/CodeGen/MachineBasicBlock.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineInstr.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineMemOperand.h" 30 #include "llvm/CodeGen/MachineOperand.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/RuntimeLibcalls.h" 33 #include "llvm/CodeGen/StackMaps.h" 34 #include "llvm/CodeGen/TargetLowering.h" 35 #include "llvm/CodeGen/TargetOpcodes.h" 36 #include "llvm/CodeGen/TargetRegisterInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/CallingConv.h" 40 #include "llvm/IR/DataLayout.h" 41 #include "llvm/IR/DerivedTypes.h" 42 #include "llvm/IR/Function.h" 43 #include "llvm/IR/GlobalValue.h" 44 #include "llvm/IR/GlobalVariable.h" 45 #include "llvm/IR/IRBuilder.h" 46 #include "llvm/IR/Module.h" 47 #include "llvm/IR/Type.h" 48 #include "llvm/Support/BranchProbability.h" 49 #include "llvm/Support/Casting.h" 50 #include "llvm/Support/CommandLine.h" 51 #include "llvm/Support/Compiler.h" 52 #include "llvm/Support/ErrorHandling.h" 53 #include "llvm/Support/MachineValueType.h" 54 #include "llvm/Support/MathExtras.h" 55 #include "llvm/Target/TargetMachine.h" 56 #include "llvm/Transforms/Utils/SizeOpts.h" 57 #include <algorithm> 58 #include <cassert> 59 #include <cstddef> 60 #include <cstdint> 61 #include <cstring> 62 #include <iterator> 63 #include <string> 64 #include <tuple> 65 #include <utility> 66 67 using namespace llvm; 68 69 static cl::opt<bool> JumpIsExpensiveOverride( 70 "jump-is-expensive", cl::init(false), 71 cl::desc("Do not create extra branches to split comparison logic."), 72 cl::Hidden); 73 74 static cl::opt<unsigned> MinimumJumpTableEntries 75 ("min-jump-table-entries", cl::init(4), cl::Hidden, 76 cl::desc("Set minimum number of entries to use a jump table.")); 77 78 static cl::opt<unsigned> MaximumJumpTableSize 79 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, 80 cl::desc("Set maximum size of jump tables.")); 81 82 /// Minimum jump table density for normal functions. 83 static cl::opt<unsigned> 84 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, 85 cl::desc("Minimum density for building a jump table in " 86 "a normal function")); 87 88 /// Minimum jump table density for -Os or -Oz functions. 89 static cl::opt<unsigned> OptsizeJumpTableDensity( 90 "optsize-jump-table-density", cl::init(40), cl::Hidden, 91 cl::desc("Minimum density for building a jump table in " 92 "an optsize function")); 93 94 // FIXME: This option is only to test if the strict fp operation processed 95 // correctly by preventing mutating strict fp operation to normal fp operation 96 // during development. When the backend supports strict float operation, this 97 // option will be meaningless. 98 static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation", 99 cl::desc("Don't mutate strict-float node to a legalize node"), 100 cl::init(false), cl::Hidden); 101 102 static bool darwinHasSinCos(const Triple &TT) { 103 assert(TT.isOSDarwin() && "should be called with darwin triple"); 104 // Don't bother with 32 bit x86. 105 if (TT.getArch() == Triple::x86) 106 return false; 107 // Macos < 10.9 has no sincos_stret. 108 if (TT.isMacOSX()) 109 return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit(); 110 // iOS < 7.0 has no sincos_stret. 111 if (TT.isiOS()) 112 return !TT.isOSVersionLT(7, 0); 113 // Any other darwin such as WatchOS/TvOS is new enough. 114 return true; 115 } 116 117 // Although this default value is arbitrary, it is not random. It is assumed 118 // that a condition that evaluates the same way by a higher percentage than this 119 // is best represented as control flow. Therefore, the default value N should be 120 // set such that the win from N% correct executions is greater than the loss 121 // from (100 - N)% mispredicted executions for the majority of intended targets. 122 static cl::opt<int> MinPercentageForPredictableBranch( 123 "min-predictable-branch", cl::init(99), 124 cl::desc("Minimum percentage (0-100) that a condition must be either true " 125 "or false to assume that the condition is predictable"), 126 cl::Hidden); 127 128 void TargetLoweringBase::InitLibcalls(const Triple &TT) { 129 #define HANDLE_LIBCALL(code, name) \ 130 setLibcallName(RTLIB::code, name); 131 #include "llvm/IR/RuntimeLibcalls.def" 132 #undef HANDLE_LIBCALL 133 // Initialize calling conventions to their default. 134 for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC) 135 setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C); 136 137 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf". 138 if (TT.getArch() == Triple::ppc || TT.isPPC64()) { 139 setLibcallName(RTLIB::ADD_F128, "__addkf3"); 140 setLibcallName(RTLIB::SUB_F128, "__subkf3"); 141 setLibcallName(RTLIB::MUL_F128, "__mulkf3"); 142 setLibcallName(RTLIB::DIV_F128, "__divkf3"); 143 setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2"); 144 setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2"); 145 setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2"); 146 setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2"); 147 setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi"); 148 setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi"); 149 setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi"); 150 setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi"); 151 setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf"); 152 setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf"); 153 setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf"); 154 setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf"); 155 setLibcallName(RTLIB::OEQ_F128, "__eqkf2"); 156 setLibcallName(RTLIB::UNE_F128, "__nekf2"); 157 setLibcallName(RTLIB::OGE_F128, "__gekf2"); 158 setLibcallName(RTLIB::OLT_F128, "__ltkf2"); 159 setLibcallName(RTLIB::OLE_F128, "__lekf2"); 160 setLibcallName(RTLIB::OGT_F128, "__gtkf2"); 161 setLibcallName(RTLIB::UO_F128, "__unordkf2"); 162 } 163 164 // A few names are different on particular architectures or environments. 165 if (TT.isOSDarwin()) { 166 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead 167 // of the gnueabi-style __gnu_*_ieee. 168 // FIXME: What about other targets? 169 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); 170 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); 171 172 // Some darwins have an optimized __bzero/bzero function. 173 switch (TT.getArch()) { 174 case Triple::x86: 175 case Triple::x86_64: 176 if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6)) 177 setLibcallName(RTLIB::BZERO, "__bzero"); 178 break; 179 case Triple::aarch64: 180 case Triple::aarch64_32: 181 setLibcallName(RTLIB::BZERO, "bzero"); 182 break; 183 default: 184 break; 185 } 186 187 if (darwinHasSinCos(TT)) { 188 setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret"); 189 setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret"); 190 if (TT.isWatchABI()) { 191 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32, 192 CallingConv::ARM_AAPCS_VFP); 193 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64, 194 CallingConv::ARM_AAPCS_VFP); 195 } 196 } 197 } else { 198 setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee"); 199 setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee"); 200 } 201 202 if (TT.isGNUEnvironment() || TT.isOSFuchsia() || 203 (TT.isAndroid() && !TT.isAndroidVersionLT(9))) { 204 setLibcallName(RTLIB::SINCOS_F32, "sincosf"); 205 setLibcallName(RTLIB::SINCOS_F64, "sincos"); 206 setLibcallName(RTLIB::SINCOS_F80, "sincosl"); 207 setLibcallName(RTLIB::SINCOS_F128, "sincosl"); 208 setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl"); 209 } 210 211 if (TT.isPS4CPU()) { 212 setLibcallName(RTLIB::SINCOS_F32, "sincosf"); 213 setLibcallName(RTLIB::SINCOS_F64, "sincos"); 214 } 215 216 if (TT.isOSOpenBSD()) { 217 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr); 218 } 219 } 220 221 /// getFPEXT - Return the FPEXT_*_* value for the given types, or 222 /// UNKNOWN_LIBCALL if there is none. 223 RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { 224 if (OpVT == MVT::f16) { 225 if (RetVT == MVT::f32) 226 return FPEXT_F16_F32; 227 if (RetVT == MVT::f128) 228 return FPEXT_F16_F128; 229 } else if (OpVT == MVT::f32) { 230 if (RetVT == MVT::f64) 231 return FPEXT_F32_F64; 232 if (RetVT == MVT::f128) 233 return FPEXT_F32_F128; 234 if (RetVT == MVT::ppcf128) 235 return FPEXT_F32_PPCF128; 236 } else if (OpVT == MVT::f64) { 237 if (RetVT == MVT::f128) 238 return FPEXT_F64_F128; 239 else if (RetVT == MVT::ppcf128) 240 return FPEXT_F64_PPCF128; 241 } else if (OpVT == MVT::f80) { 242 if (RetVT == MVT::f128) 243 return FPEXT_F80_F128; 244 } 245 246 return UNKNOWN_LIBCALL; 247 } 248 249 /// getFPROUND - Return the FPROUND_*_* value for the given types, or 250 /// UNKNOWN_LIBCALL if there is none. 251 RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) { 252 if (RetVT == MVT::f16) { 253 if (OpVT == MVT::f32) 254 return FPROUND_F32_F16; 255 if (OpVT == MVT::f64) 256 return FPROUND_F64_F16; 257 if (OpVT == MVT::f80) 258 return FPROUND_F80_F16; 259 if (OpVT == MVT::f128) 260 return FPROUND_F128_F16; 261 if (OpVT == MVT::ppcf128) 262 return FPROUND_PPCF128_F16; 263 } else if (RetVT == MVT::f32) { 264 if (OpVT == MVT::f64) 265 return FPROUND_F64_F32; 266 if (OpVT == MVT::f80) 267 return FPROUND_F80_F32; 268 if (OpVT == MVT::f128) 269 return FPROUND_F128_F32; 270 if (OpVT == MVT::ppcf128) 271 return FPROUND_PPCF128_F32; 272 } else if (RetVT == MVT::f64) { 273 if (OpVT == MVT::f80) 274 return FPROUND_F80_F64; 275 if (OpVT == MVT::f128) 276 return FPROUND_F128_F64; 277 if (OpVT == MVT::ppcf128) 278 return FPROUND_PPCF128_F64; 279 } else if (RetVT == MVT::f80) { 280 if (OpVT == MVT::f128) 281 return FPROUND_F128_F80; 282 } 283 284 return UNKNOWN_LIBCALL; 285 } 286 287 /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or 288 /// UNKNOWN_LIBCALL if there is none. 289 RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) { 290 if (OpVT == MVT::f32) { 291 if (RetVT == MVT::i32) 292 return FPTOSINT_F32_I32; 293 if (RetVT == MVT::i64) 294 return FPTOSINT_F32_I64; 295 if (RetVT == MVT::i128) 296 return FPTOSINT_F32_I128; 297 } else if (OpVT == MVT::f64) { 298 if (RetVT == MVT::i32) 299 return FPTOSINT_F64_I32; 300 if (RetVT == MVT::i64) 301 return FPTOSINT_F64_I64; 302 if (RetVT == MVT::i128) 303 return FPTOSINT_F64_I128; 304 } else if (OpVT == MVT::f80) { 305 if (RetVT == MVT::i32) 306 return FPTOSINT_F80_I32; 307 if (RetVT == MVT::i64) 308 return FPTOSINT_F80_I64; 309 if (RetVT == MVT::i128) 310 return FPTOSINT_F80_I128; 311 } else if (OpVT == MVT::f128) { 312 if (RetVT == MVT::i32) 313 return FPTOSINT_F128_I32; 314 if (RetVT == MVT::i64) 315 return FPTOSINT_F128_I64; 316 if (RetVT == MVT::i128) 317 return FPTOSINT_F128_I128; 318 } else if (OpVT == MVT::ppcf128) { 319 if (RetVT == MVT::i32) 320 return FPTOSINT_PPCF128_I32; 321 if (RetVT == MVT::i64) 322 return FPTOSINT_PPCF128_I64; 323 if (RetVT == MVT::i128) 324 return FPTOSINT_PPCF128_I128; 325 } 326 return UNKNOWN_LIBCALL; 327 } 328 329 /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or 330 /// UNKNOWN_LIBCALL if there is none. 331 RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) { 332 if (OpVT == MVT::f32) { 333 if (RetVT == MVT::i32) 334 return FPTOUINT_F32_I32; 335 if (RetVT == MVT::i64) 336 return FPTOUINT_F32_I64; 337 if (RetVT == MVT::i128) 338 return FPTOUINT_F32_I128; 339 } else if (OpVT == MVT::f64) { 340 if (RetVT == MVT::i32) 341 return FPTOUINT_F64_I32; 342 if (RetVT == MVT::i64) 343 return FPTOUINT_F64_I64; 344 if (RetVT == MVT::i128) 345 return FPTOUINT_F64_I128; 346 } else if (OpVT == MVT::f80) { 347 if (RetVT == MVT::i32) 348 return FPTOUINT_F80_I32; 349 if (RetVT == MVT::i64) 350 return FPTOUINT_F80_I64; 351 if (RetVT == MVT::i128) 352 return FPTOUINT_F80_I128; 353 } else if (OpVT == MVT::f128) { 354 if (RetVT == MVT::i32) 355 return FPTOUINT_F128_I32; 356 if (RetVT == MVT::i64) 357 return FPTOUINT_F128_I64; 358 if (RetVT == MVT::i128) 359 return FPTOUINT_F128_I128; 360 } else if (OpVT == MVT::ppcf128) { 361 if (RetVT == MVT::i32) 362 return FPTOUINT_PPCF128_I32; 363 if (RetVT == MVT::i64) 364 return FPTOUINT_PPCF128_I64; 365 if (RetVT == MVT::i128) 366 return FPTOUINT_PPCF128_I128; 367 } 368 return UNKNOWN_LIBCALL; 369 } 370 371 /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or 372 /// UNKNOWN_LIBCALL if there is none. 373 RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) { 374 if (OpVT == MVT::i32) { 375 if (RetVT == MVT::f32) 376 return SINTTOFP_I32_F32; 377 if (RetVT == MVT::f64) 378 return SINTTOFP_I32_F64; 379 if (RetVT == MVT::f80) 380 return SINTTOFP_I32_F80; 381 if (RetVT == MVT::f128) 382 return SINTTOFP_I32_F128; 383 if (RetVT == MVT::ppcf128) 384 return SINTTOFP_I32_PPCF128; 385 } else if (OpVT == MVT::i64) { 386 if (RetVT == MVT::f32) 387 return SINTTOFP_I64_F32; 388 if (RetVT == MVT::f64) 389 return SINTTOFP_I64_F64; 390 if (RetVT == MVT::f80) 391 return SINTTOFP_I64_F80; 392 if (RetVT == MVT::f128) 393 return SINTTOFP_I64_F128; 394 if (RetVT == MVT::ppcf128) 395 return SINTTOFP_I64_PPCF128; 396 } else if (OpVT == MVT::i128) { 397 if (RetVT == MVT::f32) 398 return SINTTOFP_I128_F32; 399 if (RetVT == MVT::f64) 400 return SINTTOFP_I128_F64; 401 if (RetVT == MVT::f80) 402 return SINTTOFP_I128_F80; 403 if (RetVT == MVT::f128) 404 return SINTTOFP_I128_F128; 405 if (RetVT == MVT::ppcf128) 406 return SINTTOFP_I128_PPCF128; 407 } 408 return UNKNOWN_LIBCALL; 409 } 410 411 /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or 412 /// UNKNOWN_LIBCALL if there is none. 413 RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { 414 if (OpVT == MVT::i32) { 415 if (RetVT == MVT::f32) 416 return UINTTOFP_I32_F32; 417 if (RetVT == MVT::f64) 418 return UINTTOFP_I32_F64; 419 if (RetVT == MVT::f80) 420 return UINTTOFP_I32_F80; 421 if (RetVT == MVT::f128) 422 return UINTTOFP_I32_F128; 423 if (RetVT == MVT::ppcf128) 424 return UINTTOFP_I32_PPCF128; 425 } else if (OpVT == MVT::i64) { 426 if (RetVT == MVT::f32) 427 return UINTTOFP_I64_F32; 428 if (RetVT == MVT::f64) 429 return UINTTOFP_I64_F64; 430 if (RetVT == MVT::f80) 431 return UINTTOFP_I64_F80; 432 if (RetVT == MVT::f128) 433 return UINTTOFP_I64_F128; 434 if (RetVT == MVT::ppcf128) 435 return UINTTOFP_I64_PPCF128; 436 } else if (OpVT == MVT::i128) { 437 if (RetVT == MVT::f32) 438 return UINTTOFP_I128_F32; 439 if (RetVT == MVT::f64) 440 return UINTTOFP_I128_F64; 441 if (RetVT == MVT::f80) 442 return UINTTOFP_I128_F80; 443 if (RetVT == MVT::f128) 444 return UINTTOFP_I128_F128; 445 if (RetVT == MVT::ppcf128) 446 return UINTTOFP_I128_PPCF128; 447 } 448 return UNKNOWN_LIBCALL; 449 } 450 451 RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) { 452 #define OP_TO_LIBCALL(Name, Enum) \ 453 case Name: \ 454 switch (VT.SimpleTy) { \ 455 default: \ 456 return UNKNOWN_LIBCALL; \ 457 case MVT::i8: \ 458 return Enum##_1; \ 459 case MVT::i16: \ 460 return Enum##_2; \ 461 case MVT::i32: \ 462 return Enum##_4; \ 463 case MVT::i64: \ 464 return Enum##_8; \ 465 case MVT::i128: \ 466 return Enum##_16; \ 467 } 468 469 switch (Opc) { 470 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET) 471 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP) 472 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD) 473 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB) 474 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND) 475 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR) 476 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR) 477 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND) 478 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX) 479 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX) 480 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN) 481 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN) 482 } 483 484 #undef OP_TO_LIBCALL 485 486 return UNKNOWN_LIBCALL; 487 } 488 489 RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 490 switch (ElementSize) { 491 case 1: 492 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1; 493 case 2: 494 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2; 495 case 4: 496 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4; 497 case 8: 498 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8; 499 case 16: 500 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16; 501 default: 502 return UNKNOWN_LIBCALL; 503 } 504 } 505 506 RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 507 switch (ElementSize) { 508 case 1: 509 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1; 510 case 2: 511 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2; 512 case 4: 513 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4; 514 case 8: 515 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8; 516 case 16: 517 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16; 518 default: 519 return UNKNOWN_LIBCALL; 520 } 521 } 522 523 RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 524 switch (ElementSize) { 525 case 1: 526 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1; 527 case 2: 528 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2; 529 case 4: 530 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4; 531 case 8: 532 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8; 533 case 16: 534 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16; 535 default: 536 return UNKNOWN_LIBCALL; 537 } 538 } 539 540 /// InitCmpLibcallCCs - Set default comparison libcall CC. 541 static void InitCmpLibcallCCs(ISD::CondCode *CCs) { 542 memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL); 543 CCs[RTLIB::OEQ_F32] = ISD::SETEQ; 544 CCs[RTLIB::OEQ_F64] = ISD::SETEQ; 545 CCs[RTLIB::OEQ_F128] = ISD::SETEQ; 546 CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ; 547 CCs[RTLIB::UNE_F32] = ISD::SETNE; 548 CCs[RTLIB::UNE_F64] = ISD::SETNE; 549 CCs[RTLIB::UNE_F128] = ISD::SETNE; 550 CCs[RTLIB::UNE_PPCF128] = ISD::SETNE; 551 CCs[RTLIB::OGE_F32] = ISD::SETGE; 552 CCs[RTLIB::OGE_F64] = ISD::SETGE; 553 CCs[RTLIB::OGE_F128] = ISD::SETGE; 554 CCs[RTLIB::OGE_PPCF128] = ISD::SETGE; 555 CCs[RTLIB::OLT_F32] = ISD::SETLT; 556 CCs[RTLIB::OLT_F64] = ISD::SETLT; 557 CCs[RTLIB::OLT_F128] = ISD::SETLT; 558 CCs[RTLIB::OLT_PPCF128] = ISD::SETLT; 559 CCs[RTLIB::OLE_F32] = ISD::SETLE; 560 CCs[RTLIB::OLE_F64] = ISD::SETLE; 561 CCs[RTLIB::OLE_F128] = ISD::SETLE; 562 CCs[RTLIB::OLE_PPCF128] = ISD::SETLE; 563 CCs[RTLIB::OGT_F32] = ISD::SETGT; 564 CCs[RTLIB::OGT_F64] = ISD::SETGT; 565 CCs[RTLIB::OGT_F128] = ISD::SETGT; 566 CCs[RTLIB::OGT_PPCF128] = ISD::SETGT; 567 CCs[RTLIB::UO_F32] = ISD::SETNE; 568 CCs[RTLIB::UO_F64] = ISD::SETNE; 569 CCs[RTLIB::UO_F128] = ISD::SETNE; 570 CCs[RTLIB::UO_PPCF128] = ISD::SETNE; 571 } 572 573 /// NOTE: The TargetMachine owns TLOF. 574 TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { 575 initActions(); 576 577 // Perform these initializations only once. 578 MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = 579 MaxLoadsPerMemcmp = 8; 580 MaxGluedStoresPerMemcpy = 0; 581 MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize = 582 MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4; 583 HasMultipleConditionRegisters = false; 584 HasExtractBitsInsn = false; 585 JumpIsExpensive = JumpIsExpensiveOverride; 586 PredictableSelectIsExpensive = false; 587 EnableExtLdPromotion = false; 588 StackPointerRegisterToSaveRestore = 0; 589 BooleanContents = UndefinedBooleanContent; 590 BooleanFloatContents = UndefinedBooleanContent; 591 BooleanVectorContents = UndefinedBooleanContent; 592 SchedPreferenceInfo = Sched::ILP; 593 GatherAllAliasesMaxDepth = 18; 594 IsStrictFPEnabled = DisableStrictNodeMutation; 595 // TODO: the default will be switched to 0 in the next commit, along 596 // with the Target-specific changes necessary. 597 MaxAtomicSizeInBitsSupported = 1024; 598 599 MinCmpXchgSizeInBits = 0; 600 SupportsUnalignedAtomics = false; 601 602 std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr); 603 604 InitLibcalls(TM.getTargetTriple()); 605 InitCmpLibcallCCs(CmpLibcallCCs); 606 } 607 608 void TargetLoweringBase::initActions() { 609 // All operations default to being supported. 610 memset(OpActions, 0, sizeof(OpActions)); 611 memset(LoadExtActions, 0, sizeof(LoadExtActions)); 612 memset(TruncStoreActions, 0, sizeof(TruncStoreActions)); 613 memset(IndexedModeActions, 0, sizeof(IndexedModeActions)); 614 memset(CondCodeActions, 0, sizeof(CondCodeActions)); 615 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr); 616 std::fill(std::begin(TargetDAGCombineArray), 617 std::end(TargetDAGCombineArray), 0); 618 619 for (MVT VT : MVT::fp_valuetypes()) { 620 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits()); 621 if (IntVT.isValid()) { 622 setOperationAction(ISD::ATOMIC_SWAP, VT, Promote); 623 AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT); 624 } 625 } 626 627 // Set default actions for various operations. 628 for (MVT VT : MVT::all_valuetypes()) { 629 // Default all indexed load / store to expand. 630 for (unsigned IM = (unsigned)ISD::PRE_INC; 631 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) { 632 setIndexedLoadAction(IM, VT, Expand); 633 setIndexedStoreAction(IM, VT, Expand); 634 setIndexedMaskedLoadAction(IM, VT, Expand); 635 setIndexedMaskedStoreAction(IM, VT, Expand); 636 } 637 638 // Most backends expect to see the node which just returns the value loaded. 639 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand); 640 641 // These operations default to expand. 642 setOperationAction(ISD::FGETSIGN, VT, Expand); 643 setOperationAction(ISD::CONCAT_VECTORS, VT, Expand); 644 setOperationAction(ISD::FMINNUM, VT, Expand); 645 setOperationAction(ISD::FMAXNUM, VT, Expand); 646 setOperationAction(ISD::FMINNUM_IEEE, VT, Expand); 647 setOperationAction(ISD::FMAXNUM_IEEE, VT, Expand); 648 setOperationAction(ISD::FMINIMUM, VT, Expand); 649 setOperationAction(ISD::FMAXIMUM, VT, Expand); 650 setOperationAction(ISD::FMAD, VT, Expand); 651 setOperationAction(ISD::SMIN, VT, Expand); 652 setOperationAction(ISD::SMAX, VT, Expand); 653 setOperationAction(ISD::UMIN, VT, Expand); 654 setOperationAction(ISD::UMAX, VT, Expand); 655 setOperationAction(ISD::ABS, VT, Expand); 656 setOperationAction(ISD::FSHL, VT, Expand); 657 setOperationAction(ISD::FSHR, VT, Expand); 658 setOperationAction(ISD::SADDSAT, VT, Expand); 659 setOperationAction(ISD::UADDSAT, VT, Expand); 660 setOperationAction(ISD::SSUBSAT, VT, Expand); 661 setOperationAction(ISD::USUBSAT, VT, Expand); 662 setOperationAction(ISD::SSHLSAT, VT, Expand); 663 setOperationAction(ISD::USHLSAT, VT, Expand); 664 setOperationAction(ISD::SMULFIX, VT, Expand); 665 setOperationAction(ISD::SMULFIXSAT, VT, Expand); 666 setOperationAction(ISD::UMULFIX, VT, Expand); 667 setOperationAction(ISD::UMULFIXSAT, VT, Expand); 668 setOperationAction(ISD::SDIVFIX, VT, Expand); 669 setOperationAction(ISD::SDIVFIXSAT, VT, Expand); 670 setOperationAction(ISD::UDIVFIX, VT, Expand); 671 setOperationAction(ISD::UDIVFIXSAT, VT, Expand); 672 673 // Overflow operations default to expand 674 setOperationAction(ISD::SADDO, VT, Expand); 675 setOperationAction(ISD::SSUBO, VT, Expand); 676 setOperationAction(ISD::UADDO, VT, Expand); 677 setOperationAction(ISD::USUBO, VT, Expand); 678 setOperationAction(ISD::SMULO, VT, Expand); 679 setOperationAction(ISD::UMULO, VT, Expand); 680 681 // ADDCARRY operations default to expand 682 setOperationAction(ISD::ADDCARRY, VT, Expand); 683 setOperationAction(ISD::SUBCARRY, VT, Expand); 684 setOperationAction(ISD::SETCCCARRY, VT, Expand); 685 setOperationAction(ISD::SADDO_CARRY, VT, Expand); 686 setOperationAction(ISD::SSUBO_CARRY, VT, Expand); 687 688 // ADDC/ADDE/SUBC/SUBE default to expand. 689 setOperationAction(ISD::ADDC, VT, Expand); 690 setOperationAction(ISD::ADDE, VT, Expand); 691 setOperationAction(ISD::SUBC, VT, Expand); 692 setOperationAction(ISD::SUBE, VT, Expand); 693 694 // These default to Expand so they will be expanded to CTLZ/CTTZ by default. 695 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 696 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 697 698 setOperationAction(ISD::BITREVERSE, VT, Expand); 699 setOperationAction(ISD::PARITY, VT, Expand); 700 701 // These library functions default to expand. 702 setOperationAction(ISD::FROUND, VT, Expand); 703 setOperationAction(ISD::FROUNDEVEN, VT, Expand); 704 setOperationAction(ISD::FPOWI, VT, Expand); 705 706 // These operations default to expand for vector types. 707 if (VT.isVector()) { 708 setOperationAction(ISD::FCOPYSIGN, VT, Expand); 709 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 710 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Expand); 711 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Expand); 712 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand); 713 setOperationAction(ISD::SPLAT_VECTOR, VT, Expand); 714 } 715 716 // Constrained floating-point operations default to expand. 717 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 718 setOperationAction(ISD::STRICT_##DAGN, VT, Expand); 719 #include "llvm/IR/ConstrainedOps.def" 720 721 // For most targets @llvm.get.dynamic.area.offset just returns 0. 722 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand); 723 724 // Vector reduction default to expand. 725 setOperationAction(ISD::VECREDUCE_FADD, VT, Expand); 726 setOperationAction(ISD::VECREDUCE_FMUL, VT, Expand); 727 setOperationAction(ISD::VECREDUCE_ADD, VT, Expand); 728 setOperationAction(ISD::VECREDUCE_MUL, VT, Expand); 729 setOperationAction(ISD::VECREDUCE_AND, VT, Expand); 730 setOperationAction(ISD::VECREDUCE_OR, VT, Expand); 731 setOperationAction(ISD::VECREDUCE_XOR, VT, Expand); 732 setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand); 733 setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand); 734 setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand); 735 setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand); 736 setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand); 737 setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand); 738 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Expand); 739 setOperationAction(ISD::VECREDUCE_SEQ_FMUL, VT, Expand); 740 } 741 742 // Most targets ignore the @llvm.prefetch intrinsic. 743 setOperationAction(ISD::PREFETCH, MVT::Other, Expand); 744 745 // Most targets also ignore the @llvm.readcyclecounter intrinsic. 746 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand); 747 748 // ConstantFP nodes default to expand. Targets can either change this to 749 // Legal, in which case all fp constants are legal, or use isFPImmLegal() 750 // to optimize expansions for certain constants. 751 setOperationAction(ISD::ConstantFP, MVT::f16, Expand); 752 setOperationAction(ISD::ConstantFP, MVT::f32, Expand); 753 setOperationAction(ISD::ConstantFP, MVT::f64, Expand); 754 setOperationAction(ISD::ConstantFP, MVT::f80, Expand); 755 setOperationAction(ISD::ConstantFP, MVT::f128, Expand); 756 757 // These library functions default to expand. 758 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) { 759 setOperationAction(ISD::FCBRT, VT, Expand); 760 setOperationAction(ISD::FLOG , VT, Expand); 761 setOperationAction(ISD::FLOG2, VT, Expand); 762 setOperationAction(ISD::FLOG10, VT, Expand); 763 setOperationAction(ISD::FEXP , VT, Expand); 764 setOperationAction(ISD::FEXP2, VT, Expand); 765 setOperationAction(ISD::FFLOOR, VT, Expand); 766 setOperationAction(ISD::FNEARBYINT, VT, Expand); 767 setOperationAction(ISD::FCEIL, VT, Expand); 768 setOperationAction(ISD::FRINT, VT, Expand); 769 setOperationAction(ISD::FTRUNC, VT, Expand); 770 setOperationAction(ISD::FROUND, VT, Expand); 771 setOperationAction(ISD::FROUNDEVEN, VT, Expand); 772 setOperationAction(ISD::LROUND, VT, Expand); 773 setOperationAction(ISD::LLROUND, VT, Expand); 774 setOperationAction(ISD::LRINT, VT, Expand); 775 setOperationAction(ISD::LLRINT, VT, Expand); 776 } 777 778 // Default ISD::TRAP to expand (which turns it into abort). 779 setOperationAction(ISD::TRAP, MVT::Other, Expand); 780 781 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand" 782 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP. 783 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand); 784 } 785 786 MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL, 787 EVT) const { 788 return MVT::getIntegerVT(DL.getPointerSizeInBits(0)); 789 } 790 791 EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL, 792 bool LegalTypes) const { 793 assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); 794 if (LHSTy.isVector()) 795 return LHSTy; 796 return LegalTypes ? getScalarShiftAmountTy(DL, LHSTy) 797 : getPointerTy(DL); 798 } 799 800 bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const { 801 assert(isTypeLegal(VT)); 802 switch (Op) { 803 default: 804 return false; 805 case ISD::SDIV: 806 case ISD::UDIV: 807 case ISD::SREM: 808 case ISD::UREM: 809 return true; 810 } 811 } 812 813 bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS, 814 unsigned DestAS) const { 815 return TM.isNoopAddrSpaceCast(SrcAS, DestAS); 816 } 817 818 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) { 819 // If the command-line option was specified, ignore this request. 820 if (!JumpIsExpensiveOverride.getNumOccurrences()) 821 JumpIsExpensive = isExpensive; 822 } 823 824 TargetLoweringBase::LegalizeKind 825 TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const { 826 // If this is a simple type, use the ComputeRegisterProp mechanism. 827 if (VT.isSimple()) { 828 MVT SVT = VT.getSimpleVT(); 829 assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType)); 830 MVT NVT = TransformToType[SVT.SimpleTy]; 831 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT); 832 833 assert((LA == TypeLegal || LA == TypeSoftenFloat || 834 LA == TypeSoftPromoteHalf || 835 (NVT.isVector() || 836 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) && 837 "Promote may not follow Expand or Promote"); 838 839 if (LA == TypeSplitVector) 840 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context)); 841 if (LA == TypeScalarizeVector) 842 return LegalizeKind(LA, SVT.getVectorElementType()); 843 return LegalizeKind(LA, NVT); 844 } 845 846 // Handle Extended Scalar Types. 847 if (!VT.isVector()) { 848 assert(VT.isInteger() && "Float types must be simple"); 849 unsigned BitSize = VT.getSizeInBits(); 850 // First promote to a power-of-two size, then expand if necessary. 851 if (BitSize < 8 || !isPowerOf2_32(BitSize)) { 852 EVT NVT = VT.getRoundIntegerType(Context); 853 assert(NVT != VT && "Unable to round integer VT"); 854 LegalizeKind NextStep = getTypeConversion(Context, NVT); 855 // Avoid multi-step promotion. 856 if (NextStep.first == TypePromoteInteger) 857 return NextStep; 858 // Return rounded integer type. 859 return LegalizeKind(TypePromoteInteger, NVT); 860 } 861 862 return LegalizeKind(TypeExpandInteger, 863 EVT::getIntegerVT(Context, VT.getSizeInBits() / 2)); 864 } 865 866 // Handle vector types. 867 ElementCount NumElts = VT.getVectorElementCount(); 868 EVT EltVT = VT.getVectorElementType(); 869 870 // Vectors with only one element are always scalarized. 871 if (NumElts.isScalar()) 872 return LegalizeKind(TypeScalarizeVector, EltVT); 873 874 if (VT.getVectorElementCount() == ElementCount::getScalable(1)) 875 report_fatal_error("Cannot legalize this vector"); 876 877 // Try to widen vector elements until the element type is a power of two and 878 // promote it to a legal type later on, for example: 879 // <3 x i8> -> <4 x i8> -> <4 x i32> 880 if (EltVT.isInteger()) { 881 // Vectors with a number of elements that is not a power of two are always 882 // widened, for example <3 x i8> -> <4 x i8>. 883 if (!VT.isPow2VectorType()) { 884 NumElts = NumElts.coefficientNextPowerOf2(); 885 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); 886 return LegalizeKind(TypeWidenVector, NVT); 887 } 888 889 // Examine the element type. 890 LegalizeKind LK = getTypeConversion(Context, EltVT); 891 892 // If type is to be expanded, split the vector. 893 // <4 x i140> -> <2 x i140> 894 if (LK.first == TypeExpandInteger) 895 return LegalizeKind(TypeSplitVector, 896 VT.getHalfNumVectorElementsVT(Context)); 897 898 // Promote the integer element types until a legal vector type is found 899 // or until the element integer type is too big. If a legal type was not 900 // found, fallback to the usual mechanism of widening/splitting the 901 // vector. 902 EVT OldEltVT = EltVT; 903 while (true) { 904 // Increase the bitwidth of the element to the next pow-of-two 905 // (which is greater than 8 bits). 906 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits()) 907 .getRoundIntegerType(Context); 908 909 // Stop trying when getting a non-simple element type. 910 // Note that vector elements may be greater than legal vector element 911 // types. Example: X86 XMM registers hold 64bit element on 32bit 912 // systems. 913 if (!EltVT.isSimple()) 914 break; 915 916 // Build a new vector type and check if it is legal. 917 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); 918 // Found a legal promoted vector type. 919 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal) 920 return LegalizeKind(TypePromoteInteger, 921 EVT::getVectorVT(Context, EltVT, NumElts)); 922 } 923 924 // Reset the type to the unexpanded type if we did not find a legal vector 925 // type with a promoted vector element type. 926 EltVT = OldEltVT; 927 } 928 929 // Try to widen the vector until a legal type is found. 930 // If there is no wider legal type, split the vector. 931 while (true) { 932 // Round up to the next power of 2. 933 NumElts = NumElts.coefficientNextPowerOf2(); 934 935 // If there is no simple vector type with this many elements then there 936 // cannot be a larger legal vector type. Note that this assumes that 937 // there are no skipped intermediate vector types in the simple types. 938 if (!EltVT.isSimple()) 939 break; 940 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); 941 if (LargerVector == MVT()) 942 break; 943 944 // If this type is legal then widen the vector. 945 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal) 946 return LegalizeKind(TypeWidenVector, LargerVector); 947 } 948 949 // Widen odd vectors to next power of two. 950 if (!VT.isPow2VectorType()) { 951 EVT NVT = VT.getPow2VectorType(Context); 952 return LegalizeKind(TypeWidenVector, NVT); 953 } 954 955 // Vectors with illegal element types are expanded. 956 EVT NVT = EVT::getVectorVT(Context, EltVT, 957 VT.getVectorElementCount().divideCoefficientBy(2)); 958 return LegalizeKind(TypeSplitVector, NVT); 959 } 960 961 static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, 962 unsigned &NumIntermediates, 963 MVT &RegisterVT, 964 TargetLoweringBase *TLI) { 965 // Figure out the right, legal destination reg to copy into. 966 ElementCount EC = VT.getVectorElementCount(); 967 MVT EltTy = VT.getVectorElementType(); 968 969 unsigned NumVectorRegs = 1; 970 971 // Scalable vectors cannot be scalarized, so splitting or widening is 972 // required. 973 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue())) 974 llvm_unreachable( 975 "Splitting or widening of non-power-of-2 MVTs is not implemented."); 976 977 // FIXME: We don't support non-power-of-2-sized vectors for now. 978 // Ideally we could break down into LHS/RHS like LegalizeDAG does. 979 if (!isPowerOf2_32(EC.getKnownMinValue())) { 980 // Split EC to unit size (scalable property is preserved). 981 NumVectorRegs = EC.getKnownMinValue(); 982 EC = ElementCount::getFixed(1); 983 } 984 985 // Divide the input until we get to a supported size. This will 986 // always end up with an EC that represent a scalar or a scalable 987 // scalar. 988 while (EC.getKnownMinValue() > 1 && 989 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) { 990 EC = EC.divideCoefficientBy(2); 991 NumVectorRegs <<= 1; 992 } 993 994 NumIntermediates = NumVectorRegs; 995 996 MVT NewVT = MVT::getVectorVT(EltTy, EC); 997 if (!TLI->isTypeLegal(NewVT)) 998 NewVT = EltTy; 999 IntermediateVT = NewVT; 1000 1001 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits(); 1002 1003 // Convert sizes such as i33 to i64. 1004 if (!isPowerOf2_32(LaneSizeInBits)) 1005 LaneSizeInBits = NextPowerOf2(LaneSizeInBits); 1006 1007 MVT DestVT = TLI->getRegisterType(NewVT); 1008 RegisterVT = DestVT; 1009 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. 1010 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits()); 1011 1012 // Otherwise, promotion or legal types use the same number of registers as 1013 // the vector decimated to the appropriate level. 1014 return NumVectorRegs; 1015 } 1016 1017 /// isLegalRC - Return true if the value types that can be represented by the 1018 /// specified register class are all legal. 1019 bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI, 1020 const TargetRegisterClass &RC) const { 1021 for (auto I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I) 1022 if (isTypeLegal(*I)) 1023 return true; 1024 return false; 1025 } 1026 1027 /// Replace/modify any TargetFrameIndex operands with a targte-dependent 1028 /// sequence of memory operands that is recognized by PrologEpilogInserter. 1029 MachineBasicBlock * 1030 TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, 1031 MachineBasicBlock *MBB) const { 1032 MachineInstr *MI = &InitialMI; 1033 MachineFunction &MF = *MI->getMF(); 1034 MachineFrameInfo &MFI = MF.getFrameInfo(); 1035 1036 // We're handling multiple types of operands here: 1037 // PATCHPOINT MetaArgs - live-in, read only, direct 1038 // STATEPOINT Deopt Spill - live-through, read only, indirect 1039 // STATEPOINT Deopt Alloca - live-through, read only, direct 1040 // (We're currently conservative and mark the deopt slots read/write in 1041 // practice.) 1042 // STATEPOINT GC Spill - live-through, read/write, indirect 1043 // STATEPOINT GC Alloca - live-through, read/write, direct 1044 // The live-in vs live-through is handled already (the live through ones are 1045 // all stack slots), but we need to handle the different type of stackmap 1046 // operands and memory effects here. 1047 1048 if (!llvm::any_of(MI->operands(), 1049 [](MachineOperand &Operand) { return Operand.isFI(); })) 1050 return MBB; 1051 1052 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc()); 1053 1054 // Inherit previous memory operands. 1055 MIB.cloneMemRefs(*MI); 1056 1057 for (unsigned i = 0; i < MI->getNumOperands(); ++i) { 1058 MachineOperand &MO = MI->getOperand(i); 1059 if (!MO.isFI()) { 1060 // Index of Def operand this Use it tied to. 1061 // Since Defs are coming before Uses, if Use is tied, then 1062 // index of Def must be smaller that index of that Use. 1063 // Also, Defs preserve their position in new MI. 1064 unsigned TiedTo = i; 1065 if (MO.isReg() && MO.isTied()) 1066 TiedTo = MI->findTiedOperandIdx(i); 1067 MIB.add(MO); 1068 if (TiedTo < i) 1069 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1); 1070 continue; 1071 } 1072 1073 // foldMemoryOperand builds a new MI after replacing a single FI operand 1074 // with the canonical set of five x86 addressing-mode operands. 1075 int FI = MO.getIndex(); 1076 1077 // Add frame index operands recognized by stackmaps.cpp 1078 if (MFI.isStatepointSpillSlotObjectIndex(FI)) { 1079 // indirect-mem-ref tag, size, #FI, offset. 1080 // Used for spills inserted by StatepointLowering. This codepath is not 1081 // used for patchpoints/stackmaps at all, for these spilling is done via 1082 // foldMemoryOperand callback only. 1083 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity"); 1084 MIB.addImm(StackMaps::IndirectMemRefOp); 1085 MIB.addImm(MFI.getObjectSize(FI)); 1086 MIB.add(MO); 1087 MIB.addImm(0); 1088 } else { 1089 // direct-mem-ref tag, #FI, offset. 1090 // Used by patchpoint, and direct alloca arguments to statepoints 1091 MIB.addImm(StackMaps::DirectMemRefOp); 1092 MIB.add(MO); 1093 MIB.addImm(0); 1094 } 1095 1096 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!"); 1097 1098 // Add a new memory operand for this FI. 1099 assert(MFI.getObjectOffset(FI) != -1); 1100 1101 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and 1102 // PATCHPOINT should be updated to do the same. (TODO) 1103 if (MI->getOpcode() != TargetOpcode::STATEPOINT) { 1104 auto Flags = MachineMemOperand::MOLoad; 1105 MachineMemOperand *MMO = MF.getMachineMemOperand( 1106 MachinePointerInfo::getFixedStack(MF, FI), Flags, 1107 MF.getDataLayout().getPointerSize(), MFI.getObjectAlign(FI)); 1108 MIB->addMemOperand(MF, MMO); 1109 } 1110 } 1111 MBB->insert(MachineBasicBlock::iterator(MI), MIB); 1112 MI->eraseFromParent(); 1113 return MBB; 1114 } 1115 1116 MachineBasicBlock * 1117 TargetLoweringBase::emitXRayCustomEvent(MachineInstr &MI, 1118 MachineBasicBlock *MBB) const { 1119 assert(MI.getOpcode() == TargetOpcode::PATCHABLE_EVENT_CALL && 1120 "Called emitXRayCustomEvent on the wrong MI!"); 1121 auto &MF = *MI.getMF(); 1122 auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc()); 1123 for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx) 1124 MIB.add(MI.getOperand(OpIdx)); 1125 1126 MBB->insert(MachineBasicBlock::iterator(MI), MIB); 1127 MI.eraseFromParent(); 1128 return MBB; 1129 } 1130 1131 MachineBasicBlock * 1132 TargetLoweringBase::emitXRayTypedEvent(MachineInstr &MI, 1133 MachineBasicBlock *MBB) const { 1134 assert(MI.getOpcode() == TargetOpcode::PATCHABLE_TYPED_EVENT_CALL && 1135 "Called emitXRayTypedEvent on the wrong MI!"); 1136 auto &MF = *MI.getMF(); 1137 auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc()); 1138 for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx) 1139 MIB.add(MI.getOperand(OpIdx)); 1140 1141 MBB->insert(MachineBasicBlock::iterator(MI), MIB); 1142 MI.eraseFromParent(); 1143 return MBB; 1144 } 1145 1146 /// findRepresentativeClass - Return the largest legal super-reg register class 1147 /// of the register class for the specified type and its associated "cost". 1148 // This function is in TargetLowering because it uses RegClassForVT which would 1149 // need to be moved to TargetRegisterInfo and would necessitate moving 1150 // isTypeLegal over as well - a massive change that would just require 1151 // TargetLowering having a TargetRegisterInfo class member that it would use. 1152 std::pair<const TargetRegisterClass *, uint8_t> 1153 TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI, 1154 MVT VT) const { 1155 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy]; 1156 if (!RC) 1157 return std::make_pair(RC, 0); 1158 1159 // Compute the set of all super-register classes. 1160 BitVector SuperRegRC(TRI->getNumRegClasses()); 1161 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI) 1162 SuperRegRC.setBitsInMask(RCI.getMask()); 1163 1164 // Find the first legal register class with the largest spill size. 1165 const TargetRegisterClass *BestRC = RC; 1166 for (unsigned i : SuperRegRC.set_bits()) { 1167 const TargetRegisterClass *SuperRC = TRI->getRegClass(i); 1168 // We want the largest possible spill size. 1169 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC)) 1170 continue; 1171 if (!isLegalRC(*TRI, *SuperRC)) 1172 continue; 1173 BestRC = SuperRC; 1174 } 1175 return std::make_pair(BestRC, 1); 1176 } 1177 1178 /// computeRegisterProperties - Once all of the register classes are added, 1179 /// this allows us to compute derived properties we expose. 1180 void TargetLoweringBase::computeRegisterProperties( 1181 const TargetRegisterInfo *TRI) { 1182 static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE, 1183 "Too many value types for ValueTypeActions to hold!"); 1184 1185 // Everything defaults to needing one register. 1186 for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { 1187 NumRegistersForVT[i] = 1; 1188 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i; 1189 } 1190 // ...except isVoid, which doesn't need any registers. 1191 NumRegistersForVT[MVT::isVoid] = 0; 1192 1193 // Find the largest integer register class. 1194 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE; 1195 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg) 1196 assert(LargestIntReg != MVT::i1 && "No integer registers defined!"); 1197 1198 // Every integer value type larger than this largest register takes twice as 1199 // many registers to represent as the previous ValueType. 1200 for (unsigned ExpandedReg = LargestIntReg + 1; 1201 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) { 1202 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1]; 1203 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg; 1204 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1); 1205 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg, 1206 TypeExpandInteger); 1207 } 1208 1209 // Inspect all of the ValueType's smaller than the largest integer 1210 // register to see which ones need promotion. 1211 unsigned LegalIntReg = LargestIntReg; 1212 for (unsigned IntReg = LargestIntReg - 1; 1213 IntReg >= (unsigned)MVT::i1; --IntReg) { 1214 MVT IVT = (MVT::SimpleValueType)IntReg; 1215 if (isTypeLegal(IVT)) { 1216 LegalIntReg = IntReg; 1217 } else { 1218 RegisterTypeForVT[IntReg] = TransformToType[IntReg] = 1219 (MVT::SimpleValueType)LegalIntReg; 1220 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger); 1221 } 1222 } 1223 1224 // ppcf128 type is really two f64's. 1225 if (!isTypeLegal(MVT::ppcf128)) { 1226 if (isTypeLegal(MVT::f64)) { 1227 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64]; 1228 RegisterTypeForVT[MVT::ppcf128] = MVT::f64; 1229 TransformToType[MVT::ppcf128] = MVT::f64; 1230 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat); 1231 } else { 1232 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128]; 1233 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128]; 1234 TransformToType[MVT::ppcf128] = MVT::i128; 1235 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat); 1236 } 1237 } 1238 1239 // Decide how to handle f128. If the target does not have native f128 support, 1240 // expand it to i128 and we will be generating soft float library calls. 1241 if (!isTypeLegal(MVT::f128)) { 1242 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128]; 1243 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128]; 1244 TransformToType[MVT::f128] = MVT::i128; 1245 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); 1246 } 1247 1248 // Decide how to handle f64. If the target does not have native f64 support, 1249 // expand it to i64 and we will be generating soft float library calls. 1250 if (!isTypeLegal(MVT::f64)) { 1251 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64]; 1252 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64]; 1253 TransformToType[MVT::f64] = MVT::i64; 1254 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); 1255 } 1256 1257 // Decide how to handle f32. If the target does not have native f32 support, 1258 // expand it to i32 and we will be generating soft float library calls. 1259 if (!isTypeLegal(MVT::f32)) { 1260 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; 1261 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; 1262 TransformToType[MVT::f32] = MVT::i32; 1263 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); 1264 } 1265 1266 // Decide how to handle f16. If the target does not have native f16 support, 1267 // promote it to f32, because there are no f16 library calls (except for 1268 // conversions). 1269 if (!isTypeLegal(MVT::f16)) { 1270 // Allow targets to control how we legalize half. 1271 if (softPromoteHalfType()) { 1272 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16]; 1273 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16]; 1274 TransformToType[MVT::f16] = MVT::f32; 1275 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf); 1276 } else { 1277 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32]; 1278 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32]; 1279 TransformToType[MVT::f16] = MVT::f32; 1280 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat); 1281 } 1282 } 1283 1284 // Loop over all of the vector value types to see which need transformations. 1285 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; 1286 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { 1287 MVT VT = (MVT::SimpleValueType) i; 1288 if (isTypeLegal(VT)) 1289 continue; 1290 1291 MVT EltVT = VT.getVectorElementType(); 1292 ElementCount EC = VT.getVectorElementCount(); 1293 bool IsLegalWiderType = false; 1294 bool IsScalable = VT.isScalableVector(); 1295 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT); 1296 switch (PreferredAction) { 1297 case TypePromoteInteger: { 1298 MVT::SimpleValueType EndVT = IsScalable ? 1299 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE : 1300 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE; 1301 // Try to promote the elements of integer vectors. If no legal 1302 // promotion was found, fall through to the widen-vector method. 1303 for (unsigned nVT = i + 1; 1304 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) { 1305 MVT SVT = (MVT::SimpleValueType) nVT; 1306 // Promote vectors of integers to vectors with the same number 1307 // of elements, with a wider element type. 1308 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() && 1309 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) { 1310 TransformToType[i] = SVT; 1311 RegisterTypeForVT[i] = SVT; 1312 NumRegistersForVT[i] = 1; 1313 ValueTypeActions.setTypeAction(VT, TypePromoteInteger); 1314 IsLegalWiderType = true; 1315 break; 1316 } 1317 } 1318 if (IsLegalWiderType) 1319 break; 1320 LLVM_FALLTHROUGH; 1321 } 1322 1323 case TypeWidenVector: 1324 if (isPowerOf2_32(EC.getKnownMinValue())) { 1325 // Try to widen the vector. 1326 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { 1327 MVT SVT = (MVT::SimpleValueType) nVT; 1328 if (SVT.getVectorElementType() == EltVT && 1329 SVT.isScalableVector() == IsScalable && 1330 SVT.getVectorElementCount().getKnownMinValue() > 1331 EC.getKnownMinValue() && 1332 isTypeLegal(SVT)) { 1333 TransformToType[i] = SVT; 1334 RegisterTypeForVT[i] = SVT; 1335 NumRegistersForVT[i] = 1; 1336 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1337 IsLegalWiderType = true; 1338 break; 1339 } 1340 } 1341 if (IsLegalWiderType) 1342 break; 1343 } else { 1344 // Only widen to the next power of 2 to keep consistency with EVT. 1345 MVT NVT = VT.getPow2VectorType(); 1346 if (isTypeLegal(NVT)) { 1347 TransformToType[i] = NVT; 1348 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1349 RegisterTypeForVT[i] = NVT; 1350 NumRegistersForVT[i] = 1; 1351 break; 1352 } 1353 } 1354 LLVM_FALLTHROUGH; 1355 1356 case TypeSplitVector: 1357 case TypeScalarizeVector: { 1358 MVT IntermediateVT; 1359 MVT RegisterVT; 1360 unsigned NumIntermediates; 1361 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT, 1362 NumIntermediates, RegisterVT, this); 1363 NumRegistersForVT[i] = NumRegisters; 1364 assert(NumRegistersForVT[i] == NumRegisters && 1365 "NumRegistersForVT size cannot represent NumRegisters!"); 1366 RegisterTypeForVT[i] = RegisterVT; 1367 1368 MVT NVT = VT.getPow2VectorType(); 1369 if (NVT == VT) { 1370 // Type is already a power of 2. The default action is to split. 1371 TransformToType[i] = MVT::Other; 1372 if (PreferredAction == TypeScalarizeVector) 1373 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); 1374 else if (PreferredAction == TypeSplitVector) 1375 ValueTypeActions.setTypeAction(VT, TypeSplitVector); 1376 else if (EC.getKnownMinValue() > 1) 1377 ValueTypeActions.setTypeAction(VT, TypeSplitVector); 1378 else 1379 ValueTypeActions.setTypeAction(VT, EC.isScalable() 1380 ? TypeScalarizeScalableVector 1381 : TypeScalarizeVector); 1382 } else { 1383 TransformToType[i] = NVT; 1384 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1385 } 1386 break; 1387 } 1388 default: 1389 llvm_unreachable("Unknown vector legalization action!"); 1390 } 1391 } 1392 1393 // Determine the 'representative' register class for each value type. 1394 // An representative register class is the largest (meaning one which is 1395 // not a sub-register class / subreg register class) legal register class for 1396 // a group of value types. For example, on i386, i8, i16, and i32 1397 // representative would be GR32; while on x86_64 it's GR64. 1398 for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { 1399 const TargetRegisterClass* RRC; 1400 uint8_t Cost; 1401 std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i); 1402 RepRegClassForVT[i] = RRC; 1403 RepRegClassCostForVT[i] = Cost; 1404 } 1405 } 1406 1407 EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1408 EVT VT) const { 1409 assert(!VT.isVector() && "No default SetCC type for vectors!"); 1410 return getPointerTy(DL).SimpleTy; 1411 } 1412 1413 MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const { 1414 return MVT::i32; // return the default value 1415 } 1416 1417 /// getVectorTypeBreakdown - Vector types are broken down into some number of 1418 /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 1419 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. 1420 /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. 1421 /// 1422 /// This method returns the number of registers needed, and the VT for each 1423 /// register. It also returns the VT and quantity of the intermediate values 1424 /// before they are promoted/expanded. 1425 unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT, 1426 EVT &IntermediateVT, 1427 unsigned &NumIntermediates, 1428 MVT &RegisterVT) const { 1429 ElementCount EltCnt = VT.getVectorElementCount(); 1430 1431 // If there is a wider vector type with the same element type as this one, 1432 // or a promoted vector type that has the same number of elements which 1433 // are wider, then we should convert to that legal vector type. 1434 // This handles things like <2 x float> -> <4 x float> and 1435 // <4 x i1> -> <4 x i32>. 1436 LegalizeTypeAction TA = getTypeAction(Context, VT); 1437 if (EltCnt.getKnownMinValue() != 1 && 1438 (TA == TypeWidenVector || TA == TypePromoteInteger)) { 1439 EVT RegisterEVT = getTypeToTransformTo(Context, VT); 1440 if (isTypeLegal(RegisterEVT)) { 1441 IntermediateVT = RegisterEVT; 1442 RegisterVT = RegisterEVT.getSimpleVT(); 1443 NumIntermediates = 1; 1444 return 1; 1445 } 1446 } 1447 1448 // Figure out the right, legal destination reg to copy into. 1449 EVT EltTy = VT.getVectorElementType(); 1450 1451 unsigned NumVectorRegs = 1; 1452 1453 // Scalable vectors cannot be scalarized, so handle the legalisation of the 1454 // types like done elsewhere in SelectionDAG. 1455 if (VT.isScalableVector() && !isPowerOf2_32(EltCnt.getKnownMinValue())) { 1456 LegalizeKind LK; 1457 EVT PartVT = VT; 1458 do { 1459 // Iterate until we've found a legal (part) type to hold VT. 1460 LK = getTypeConversion(Context, PartVT); 1461 PartVT = LK.second; 1462 } while (LK.first != TypeLegal); 1463 1464 NumIntermediates = VT.getVectorElementCount().getKnownMinValue() / 1465 PartVT.getVectorElementCount().getKnownMinValue(); 1466 1467 // FIXME: This code needs to be extended to handle more complex vector 1468 // breakdowns, like nxv7i64 -> nxv8i64 -> 4 x nxv2i64. Currently the only 1469 // supported cases are vectors that are broken down into equal parts 1470 // such as nxv6i64 -> 3 x nxv2i64. 1471 assert((PartVT.getVectorElementCount() * NumIntermediates) == 1472 VT.getVectorElementCount() && 1473 "Expected an integer multiple of PartVT"); 1474 IntermediateVT = PartVT; 1475 RegisterVT = getRegisterType(Context, IntermediateVT); 1476 return NumIntermediates; 1477 } 1478 1479 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally 1480 // we could break down into LHS/RHS like LegalizeDAG does. 1481 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) { 1482 NumVectorRegs = EltCnt.getKnownMinValue(); 1483 EltCnt = ElementCount::getFixed(1); 1484 } 1485 1486 // Divide the input until we get to a supported size. This will always 1487 // end with a scalar if the target doesn't support vectors. 1488 while (EltCnt.getKnownMinValue() > 1 && 1489 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) { 1490 EltCnt = EltCnt.divideCoefficientBy(2); 1491 NumVectorRegs <<= 1; 1492 } 1493 1494 NumIntermediates = NumVectorRegs; 1495 1496 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt); 1497 if (!isTypeLegal(NewVT)) 1498 NewVT = EltTy; 1499 IntermediateVT = NewVT; 1500 1501 MVT DestVT = getRegisterType(Context, NewVT); 1502 RegisterVT = DestVT; 1503 1504 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16. 1505 TypeSize NewVTSize = NewVT.getSizeInBits(); 1506 // Convert sizes such as i33 to i64. 1507 if (!isPowerOf2_32(NewVTSize.getKnownMinSize())) 1508 NewVTSize = NewVTSize.coefficientNextPowerOf2(); 1509 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); 1510 } 1511 1512 // Otherwise, promotion or legal types use the same number of registers as 1513 // the vector decimated to the appropriate level. 1514 return NumVectorRegs; 1515 } 1516 1517 bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst *SI, 1518 uint64_t NumCases, 1519 uint64_t Range, 1520 ProfileSummaryInfo *PSI, 1521 BlockFrequencyInfo *BFI) const { 1522 // FIXME: This function check the maximum table size and density, but the 1523 // minimum size is not checked. It would be nice if the minimum size is 1524 // also combined within this function. Currently, the minimum size check is 1525 // performed in findJumpTable() in SelectionDAGBuiler and 1526 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl. 1527 const bool OptForSize = 1528 SI->getParent()->getParent()->hasOptSize() || 1529 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI); 1530 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize); 1531 const unsigned MaxJumpTableSize = getMaximumJumpTableSize(); 1532 1533 // Check whether the number of cases is small enough and 1534 // the range is dense enough for a jump table. 1535 return (OptForSize || Range <= MaxJumpTableSize) && 1536 (NumCases * 100 >= Range * MinDensity); 1537 } 1538 1539 /// Get the EVTs and ArgFlags collections that represent the legalized return 1540 /// type of the given function. This does not require a DAG or a return value, 1541 /// and is suitable for use before any DAGs for the function are constructed. 1542 /// TODO: Move this out of TargetLowering.cpp. 1543 void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType, 1544 AttributeList attr, 1545 SmallVectorImpl<ISD::OutputArg> &Outs, 1546 const TargetLowering &TLI, const DataLayout &DL) { 1547 SmallVector<EVT, 4> ValueVTs; 1548 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs); 1549 unsigned NumValues = ValueVTs.size(); 1550 if (NumValues == 0) return; 1551 1552 for (unsigned j = 0, f = NumValues; j != f; ++j) { 1553 EVT VT = ValueVTs[j]; 1554 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 1555 1556 if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) 1557 ExtendKind = ISD::SIGN_EXTEND; 1558 else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt)) 1559 ExtendKind = ISD::ZERO_EXTEND; 1560 1561 // FIXME: C calling convention requires the return type to be promoted to 1562 // at least 32-bit. But this is not necessary for non-C calling 1563 // conventions. The frontend should mark functions whose return values 1564 // require promoting with signext or zeroext attributes. 1565 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { 1566 MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); 1567 if (VT.bitsLT(MinVT)) 1568 VT = MinVT; 1569 } 1570 1571 unsigned NumParts = 1572 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT); 1573 MVT PartVT = 1574 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT); 1575 1576 // 'inreg' on function refers to return value 1577 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1578 if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::InReg)) 1579 Flags.setInReg(); 1580 1581 // Propagate extension type if any 1582 if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) 1583 Flags.setSExt(); 1584 else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt)) 1585 Flags.setZExt(); 1586 1587 for (unsigned i = 0; i < NumParts; ++i) 1588 Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0)); 1589 } 1590 } 1591 1592 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1593 /// function arguments in the caller parameter area. This is the actual 1594 /// alignment, not its logarithm. 1595 unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty, 1596 const DataLayout &DL) const { 1597 return DL.getABITypeAlign(Ty).value(); 1598 } 1599 1600 bool TargetLoweringBase::allowsMemoryAccessForAlignment( 1601 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, 1602 Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const { 1603 // Check if the specified alignment is sufficient based on the data layout. 1604 // TODO: While using the data layout works in practice, a better solution 1605 // would be to implement this check directly (make this a virtual function). 1606 // For example, the ABI alignment may change based on software platform while 1607 // this function should only be affected by hardware implementation. 1608 Type *Ty = VT.getTypeForEVT(Context); 1609 if (Alignment >= DL.getABITypeAlign(Ty)) { 1610 // Assume that an access that meets the ABI-specified alignment is fast. 1611 if (Fast != nullptr) 1612 *Fast = true; 1613 return true; 1614 } 1615 1616 // This is a misaligned access. 1617 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment.value(), Flags, 1618 Fast); 1619 } 1620 1621 bool TargetLoweringBase::allowsMemoryAccessForAlignment( 1622 LLVMContext &Context, const DataLayout &DL, EVT VT, 1623 const MachineMemOperand &MMO, bool *Fast) const { 1624 return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(), 1625 MMO.getAlign(), MMO.getFlags(), Fast); 1626 } 1627 1628 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, 1629 const DataLayout &DL, EVT VT, 1630 unsigned AddrSpace, Align Alignment, 1631 MachineMemOperand::Flags Flags, 1632 bool *Fast) const { 1633 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment, 1634 Flags, Fast); 1635 } 1636 1637 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, 1638 const DataLayout &DL, EVT VT, 1639 const MachineMemOperand &MMO, 1640 bool *Fast) const { 1641 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(), 1642 MMO.getFlags(), Fast); 1643 } 1644 1645 BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const { 1646 return BranchProbability(MinPercentageForPredictableBranch, 100); 1647 } 1648 1649 //===----------------------------------------------------------------------===// 1650 // TargetTransformInfo Helpers 1651 //===----------------------------------------------------------------------===// 1652 1653 int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { 1654 enum InstructionOpcodes { 1655 #define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM, 1656 #define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM 1657 #include "llvm/IR/Instruction.def" 1658 }; 1659 switch (static_cast<InstructionOpcodes>(Opcode)) { 1660 case Ret: return 0; 1661 case Br: return 0; 1662 case Switch: return 0; 1663 case IndirectBr: return 0; 1664 case Invoke: return 0; 1665 case CallBr: return 0; 1666 case Resume: return 0; 1667 case Unreachable: return 0; 1668 case CleanupRet: return 0; 1669 case CatchRet: return 0; 1670 case CatchPad: return 0; 1671 case CatchSwitch: return 0; 1672 case CleanupPad: return 0; 1673 case FNeg: return ISD::FNEG; 1674 case Add: return ISD::ADD; 1675 case FAdd: return ISD::FADD; 1676 case Sub: return ISD::SUB; 1677 case FSub: return ISD::FSUB; 1678 case Mul: return ISD::MUL; 1679 case FMul: return ISD::FMUL; 1680 case UDiv: return ISD::UDIV; 1681 case SDiv: return ISD::SDIV; 1682 case FDiv: return ISD::FDIV; 1683 case URem: return ISD::UREM; 1684 case SRem: return ISD::SREM; 1685 case FRem: return ISD::FREM; 1686 case Shl: return ISD::SHL; 1687 case LShr: return ISD::SRL; 1688 case AShr: return ISD::SRA; 1689 case And: return ISD::AND; 1690 case Or: return ISD::OR; 1691 case Xor: return ISD::XOR; 1692 case Alloca: return 0; 1693 case Load: return ISD::LOAD; 1694 case Store: return ISD::STORE; 1695 case GetElementPtr: return 0; 1696 case Fence: return 0; 1697 case AtomicCmpXchg: return 0; 1698 case AtomicRMW: return 0; 1699 case Trunc: return ISD::TRUNCATE; 1700 case ZExt: return ISD::ZERO_EXTEND; 1701 case SExt: return ISD::SIGN_EXTEND; 1702 case FPToUI: return ISD::FP_TO_UINT; 1703 case FPToSI: return ISD::FP_TO_SINT; 1704 case UIToFP: return ISD::UINT_TO_FP; 1705 case SIToFP: return ISD::SINT_TO_FP; 1706 case FPTrunc: return ISD::FP_ROUND; 1707 case FPExt: return ISD::FP_EXTEND; 1708 case PtrToInt: return ISD::BITCAST; 1709 case IntToPtr: return ISD::BITCAST; 1710 case BitCast: return ISD::BITCAST; 1711 case AddrSpaceCast: return ISD::ADDRSPACECAST; 1712 case ICmp: return ISD::SETCC; 1713 case FCmp: return ISD::SETCC; 1714 case PHI: return 0; 1715 case Call: return 0; 1716 case Select: return ISD::SELECT; 1717 case UserOp1: return 0; 1718 case UserOp2: return 0; 1719 case VAArg: return 0; 1720 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT; 1721 case InsertElement: return ISD::INSERT_VECTOR_ELT; 1722 case ShuffleVector: return ISD::VECTOR_SHUFFLE; 1723 case ExtractValue: return ISD::MERGE_VALUES; 1724 case InsertValue: return ISD::MERGE_VALUES; 1725 case LandingPad: return 0; 1726 case Freeze: return ISD::FREEZE; 1727 } 1728 1729 llvm_unreachable("Unknown instruction type encountered!"); 1730 } 1731 1732 std::pair<int, MVT> 1733 TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL, 1734 Type *Ty) const { 1735 LLVMContext &C = Ty->getContext(); 1736 EVT MTy = getValueType(DL, Ty); 1737 1738 int Cost = 1; 1739 // We keep legalizing the type until we find a legal kind. We assume that 1740 // the only operation that costs anything is the split. After splitting 1741 // we need to handle two types. 1742 while (true) { 1743 LegalizeKind LK = getTypeConversion(C, MTy); 1744 1745 if (LK.first == TypeLegal) 1746 return std::make_pair(Cost, MTy.getSimpleVT()); 1747 1748 if (LK.first == TypeSplitVector || LK.first == TypeExpandInteger) 1749 Cost *= 2; 1750 1751 // Do not loop with f128 type. 1752 if (MTy == LK.second) 1753 return std::make_pair(Cost, MTy.getSimpleVT()); 1754 1755 // Keep legalizing the type. 1756 MTy = LK.second; 1757 } 1758 } 1759 1760 Value *TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilder<> &IRB, 1761 bool UseTLS) const { 1762 // compiler-rt provides a variable with a magic name. Targets that do not 1763 // link with compiler-rt may also provide such a variable. 1764 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 1765 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr"; 1766 auto UnsafeStackPtr = 1767 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar)); 1768 1769 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); 1770 1771 if (!UnsafeStackPtr) { 1772 auto TLSModel = UseTLS ? 1773 GlobalValue::InitialExecTLSModel : 1774 GlobalValue::NotThreadLocal; 1775 // The global variable is not defined yet, define it ourselves. 1776 // We use the initial-exec TLS model because we do not support the 1777 // variable living anywhere other than in the main executable. 1778 UnsafeStackPtr = new GlobalVariable( 1779 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, 1780 UnsafeStackPtrVar, nullptr, TLSModel); 1781 } else { 1782 // The variable exists, check its type and attributes. 1783 if (UnsafeStackPtr->getValueType() != StackPtrTy) 1784 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type"); 1785 if (UseTLS != UnsafeStackPtr->isThreadLocal()) 1786 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " + 1787 (UseTLS ? "" : "not ") + "be thread-local"); 1788 } 1789 return UnsafeStackPtr; 1790 } 1791 1792 Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const { 1793 if (!TM.getTargetTriple().isAndroid()) 1794 return getDefaultSafeStackPointerLocation(IRB, true); 1795 1796 // Android provides a libc function to retrieve the address of the current 1797 // thread's unsafe stack pointer. 1798 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 1799 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); 1800 FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address", 1801 StackPtrTy->getPointerTo(0)); 1802 return IRB.CreateCall(Fn); 1803 } 1804 1805 //===----------------------------------------------------------------------===// 1806 // Loop Strength Reduction hooks 1807 //===----------------------------------------------------------------------===// 1808 1809 /// isLegalAddressingMode - Return true if the addressing mode represented 1810 /// by AM is legal for this target, for a load/store of the specified type. 1811 bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL, 1812 const AddrMode &AM, Type *Ty, 1813 unsigned AS, Instruction *I) const { 1814 // The default implementation of this implements a conservative RISCy, r+r and 1815 // r+i addr mode. 1816 1817 // Allows a sign-extended 16-bit immediate field. 1818 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 1819 return false; 1820 1821 // No global is ever allowed as a base. 1822 if (AM.BaseGV) 1823 return false; 1824 1825 // Only support r+r, 1826 switch (AM.Scale) { 1827 case 0: // "r+i" or just "i", depending on HasBaseReg. 1828 break; 1829 case 1: 1830 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 1831 return false; 1832 // Otherwise we have r+r or r+i. 1833 break; 1834 case 2: 1835 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 1836 return false; 1837 // Allow 2*r as r+r. 1838 break; 1839 default: // Don't allow n * r 1840 return false; 1841 } 1842 1843 return true; 1844 } 1845 1846 //===----------------------------------------------------------------------===// 1847 // Stack Protector 1848 //===----------------------------------------------------------------------===// 1849 1850 // For OpenBSD return its special guard variable. Otherwise return nullptr, 1851 // so that SelectionDAG handle SSP. 1852 Value *TargetLoweringBase::getIRStackGuard(IRBuilder<> &IRB) const { 1853 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) { 1854 Module &M = *IRB.GetInsertBlock()->getParent()->getParent(); 1855 PointerType *PtrTy = Type::getInt8PtrTy(M.getContext()); 1856 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy); 1857 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C)) 1858 G->setVisibility(GlobalValue::HiddenVisibility); 1859 return C; 1860 } 1861 return nullptr; 1862 } 1863 1864 // Currently only support "standard" __stack_chk_guard. 1865 // TODO: add LOAD_STACK_GUARD support. 1866 void TargetLoweringBase::insertSSPDeclarations(Module &M) const { 1867 if (!M.getNamedValue("__stack_chk_guard")) 1868 new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false, 1869 GlobalVariable::ExternalLinkage, 1870 nullptr, "__stack_chk_guard"); 1871 } 1872 1873 // Currently only support "standard" __stack_chk_guard. 1874 // TODO: add LOAD_STACK_GUARD support. 1875 Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const { 1876 return M.getNamedValue("__stack_chk_guard"); 1877 } 1878 1879 Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const { 1880 return nullptr; 1881 } 1882 1883 unsigned TargetLoweringBase::getMinimumJumpTableEntries() const { 1884 return MinimumJumpTableEntries; 1885 } 1886 1887 void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) { 1888 MinimumJumpTableEntries = Val; 1889 } 1890 1891 unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const { 1892 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity; 1893 } 1894 1895 unsigned TargetLoweringBase::getMaximumJumpTableSize() const { 1896 return MaximumJumpTableSize; 1897 } 1898 1899 void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) { 1900 MaximumJumpTableSize = Val; 1901 } 1902 1903 bool TargetLoweringBase::isJumpTableRelative() const { 1904 return getTargetMachine().isPositionIndependent(); 1905 } 1906 1907 //===----------------------------------------------------------------------===// 1908 // Reciprocal Estimates 1909 //===----------------------------------------------------------------------===// 1910 1911 /// Get the reciprocal estimate attribute string for a function that will 1912 /// override the target defaults. 1913 static StringRef getRecipEstimateForFunc(MachineFunction &MF) { 1914 const Function &F = MF.getFunction(); 1915 return F.getFnAttribute("reciprocal-estimates").getValueAsString(); 1916 } 1917 1918 /// Construct a string for the given reciprocal operation of the given type. 1919 /// This string should match the corresponding option to the front-end's 1920 /// "-mrecip" flag assuming those strings have been passed through in an 1921 /// attribute string. For example, "vec-divf" for a division of a vXf32. 1922 static std::string getReciprocalOpName(bool IsSqrt, EVT VT) { 1923 std::string Name = VT.isVector() ? "vec-" : ""; 1924 1925 Name += IsSqrt ? "sqrt" : "div"; 1926 1927 // TODO: Handle "half" or other float types? 1928 if (VT.getScalarType() == MVT::f64) { 1929 Name += "d"; 1930 } else { 1931 assert(VT.getScalarType() == MVT::f32 && 1932 "Unexpected FP type for reciprocal estimate"); 1933 Name += "f"; 1934 } 1935 1936 return Name; 1937 } 1938 1939 /// Return the character position and value (a single numeric character) of a 1940 /// customized refinement operation in the input string if it exists. Return 1941 /// false if there is no customized refinement step count. 1942 static bool parseRefinementStep(StringRef In, size_t &Position, 1943 uint8_t &Value) { 1944 const char RefStepToken = ':'; 1945 Position = In.find(RefStepToken); 1946 if (Position == StringRef::npos) 1947 return false; 1948 1949 StringRef RefStepString = In.substr(Position + 1); 1950 // Allow exactly one numeric character for the additional refinement 1951 // step parameter. 1952 if (RefStepString.size() == 1) { 1953 char RefStepChar = RefStepString[0]; 1954 if (RefStepChar >= '0' && RefStepChar <= '9') { 1955 Value = RefStepChar - '0'; 1956 return true; 1957 } 1958 } 1959 report_fatal_error("Invalid refinement step for -recip."); 1960 } 1961 1962 /// For the input attribute string, return one of the ReciprocalEstimate enum 1963 /// status values (enabled, disabled, or not specified) for this operation on 1964 /// the specified data type. 1965 static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) { 1966 if (Override.empty()) 1967 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 1968 1969 SmallVector<StringRef, 4> OverrideVector; 1970 Override.split(OverrideVector, ','); 1971 unsigned NumArgs = OverrideVector.size(); 1972 1973 // Check if "all", "none", or "default" was specified. 1974 if (NumArgs == 1) { 1975 // Look for an optional setting of the number of refinement steps needed 1976 // for this type of reciprocal operation. 1977 size_t RefPos; 1978 uint8_t RefSteps; 1979 if (parseRefinementStep(Override, RefPos, RefSteps)) { 1980 // Split the string for further processing. 1981 Override = Override.substr(0, RefPos); 1982 } 1983 1984 // All reciprocal types are enabled. 1985 if (Override == "all") 1986 return TargetLoweringBase::ReciprocalEstimate::Enabled; 1987 1988 // All reciprocal types are disabled. 1989 if (Override == "none") 1990 return TargetLoweringBase::ReciprocalEstimate::Disabled; 1991 1992 // Target defaults for enablement are used. 1993 if (Override == "default") 1994 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 1995 } 1996 1997 // The attribute string may omit the size suffix ('f'/'d'). 1998 std::string VTName = getReciprocalOpName(IsSqrt, VT); 1999 std::string VTNameNoSize = VTName; 2000 VTNameNoSize.pop_back(); 2001 static const char DisabledPrefix = '!'; 2002 2003 for (StringRef RecipType : OverrideVector) { 2004 size_t RefPos; 2005 uint8_t RefSteps; 2006 if (parseRefinementStep(RecipType, RefPos, RefSteps)) 2007 RecipType = RecipType.substr(0, RefPos); 2008 2009 // Ignore the disablement token for string matching. 2010 bool IsDisabled = RecipType[0] == DisabledPrefix; 2011 if (IsDisabled) 2012 RecipType = RecipType.substr(1); 2013 2014 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)) 2015 return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled 2016 : TargetLoweringBase::ReciprocalEstimate::Enabled; 2017 } 2018 2019 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2020 } 2021 2022 /// For the input attribute string, return the customized refinement step count 2023 /// for this operation on the specified data type. If the step count does not 2024 /// exist, return the ReciprocalEstimate enum value for unspecified. 2025 static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) { 2026 if (Override.empty()) 2027 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2028 2029 SmallVector<StringRef, 4> OverrideVector; 2030 Override.split(OverrideVector, ','); 2031 unsigned NumArgs = OverrideVector.size(); 2032 2033 // Check if "all", "default", or "none" was specified. 2034 if (NumArgs == 1) { 2035 // Look for an optional setting of the number of refinement steps needed 2036 // for this type of reciprocal operation. 2037 size_t RefPos; 2038 uint8_t RefSteps; 2039 if (!parseRefinementStep(Override, RefPos, RefSteps)) 2040 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2041 2042 // Split the string for further processing. 2043 Override = Override.substr(0, RefPos); 2044 assert(Override != "none" && 2045 "Disabled reciprocals, but specifed refinement steps?"); 2046 2047 // If this is a general override, return the specified number of steps. 2048 if (Override == "all" || Override == "default") 2049 return RefSteps; 2050 } 2051 2052 // The attribute string may omit the size suffix ('f'/'d'). 2053 std::string VTName = getReciprocalOpName(IsSqrt, VT); 2054 std::string VTNameNoSize = VTName; 2055 VTNameNoSize.pop_back(); 2056 2057 for (StringRef RecipType : OverrideVector) { 2058 size_t RefPos; 2059 uint8_t RefSteps; 2060 if (!parseRefinementStep(RecipType, RefPos, RefSteps)) 2061 continue; 2062 2063 RecipType = RecipType.substr(0, RefPos); 2064 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)) 2065 return RefSteps; 2066 } 2067 2068 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2069 } 2070 2071 int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT, 2072 MachineFunction &MF) const { 2073 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF)); 2074 } 2075 2076 int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT, 2077 MachineFunction &MF) const { 2078 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF)); 2079 } 2080 2081 int TargetLoweringBase::getSqrtRefinementSteps(EVT VT, 2082 MachineFunction &MF) const { 2083 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF)); 2084 } 2085 2086 int TargetLoweringBase::getDivRefinementSteps(EVT VT, 2087 MachineFunction &MF) const { 2088 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF)); 2089 } 2090 2091 void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const { 2092 MF.getRegInfo().freezeReservedRegs(MF); 2093 } 2094 2095 MachineMemOperand::Flags 2096 TargetLoweringBase::getLoadMemOperandFlags(const LoadInst &LI, 2097 const DataLayout &DL) const { 2098 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad; 2099 if (LI.isVolatile()) 2100 Flags |= MachineMemOperand::MOVolatile; 2101 2102 if (LI.hasMetadata(LLVMContext::MD_nontemporal)) 2103 Flags |= MachineMemOperand::MONonTemporal; 2104 2105 if (LI.hasMetadata(LLVMContext::MD_invariant_load)) 2106 Flags |= MachineMemOperand::MOInvariant; 2107 2108 if (isDereferenceablePointer(LI.getPointerOperand(), LI.getType(), DL)) 2109 Flags |= MachineMemOperand::MODereferenceable; 2110 2111 Flags |= getTargetMMOFlags(LI); 2112 return Flags; 2113 } 2114 2115 MachineMemOperand::Flags 2116 TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI, 2117 const DataLayout &DL) const { 2118 MachineMemOperand::Flags Flags = MachineMemOperand::MOStore; 2119 2120 if (SI.isVolatile()) 2121 Flags |= MachineMemOperand::MOVolatile; 2122 2123 if (SI.hasMetadata(LLVMContext::MD_nontemporal)) 2124 Flags |= MachineMemOperand::MONonTemporal; 2125 2126 // FIXME: Not preserving dereferenceable 2127 Flags |= getTargetMMOFlags(SI); 2128 return Flags; 2129 } 2130 2131 MachineMemOperand::Flags 2132 TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI, 2133 const DataLayout &DL) const { 2134 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 2135 2136 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) { 2137 if (RMW->isVolatile()) 2138 Flags |= MachineMemOperand::MOVolatile; 2139 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) { 2140 if (CmpX->isVolatile()) 2141 Flags |= MachineMemOperand::MOVolatile; 2142 } else 2143 llvm_unreachable("not an atomic instruction"); 2144 2145 // FIXME: Not preserving dereferenceable 2146 Flags |= getTargetMMOFlags(AI); 2147 return Flags; 2148 } 2149 2150 //===----------------------------------------------------------------------===// 2151 // GlobalISel Hooks 2152 //===----------------------------------------------------------------------===// 2153 2154 bool TargetLoweringBase::shouldLocalize(const MachineInstr &MI, 2155 const TargetTransformInfo *TTI) const { 2156 auto &MF = *MI.getMF(); 2157 auto &MRI = MF.getRegInfo(); 2158 // Assuming a spill and reload of a value has a cost of 1 instruction each, 2159 // this helper function computes the maximum number of uses we should consider 2160 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We 2161 // break even in terms of code size when the original MI has 2 users vs 2162 // choosing to potentially spill. Any more than 2 users we we have a net code 2163 // size increase. This doesn't take into account register pressure though. 2164 auto maxUses = [](unsigned RematCost) { 2165 // A cost of 1 means remats are basically free. 2166 if (RematCost == 1) 2167 return UINT_MAX; 2168 if (RematCost == 2) 2169 return 2U; 2170 2171 // Remat is too expensive, only sink if there's one user. 2172 if (RematCost > 2) 2173 return 1U; 2174 llvm_unreachable("Unexpected remat cost"); 2175 }; 2176 2177 // Helper to walk through uses and terminate if we've reached a limit. Saves 2178 // us spending time traversing uses if all we want to know is if it's >= min. 2179 auto isUsesAtMost = [&](unsigned Reg, unsigned MaxUses) { 2180 unsigned NumUses = 0; 2181 auto UI = MRI.use_instr_nodbg_begin(Reg), UE = MRI.use_instr_nodbg_end(); 2182 for (; UI != UE && NumUses < MaxUses; ++UI) { 2183 NumUses++; 2184 } 2185 // If we haven't reached the end yet then there are more than MaxUses users. 2186 return UI == UE; 2187 }; 2188 2189 switch (MI.getOpcode()) { 2190 default: 2191 return false; 2192 // Constants-like instructions should be close to their users. 2193 // We don't want long live-ranges for them. 2194 case TargetOpcode::G_CONSTANT: 2195 case TargetOpcode::G_FCONSTANT: 2196 case TargetOpcode::G_FRAME_INDEX: 2197 case TargetOpcode::G_INTTOPTR: 2198 return true; 2199 case TargetOpcode::G_GLOBAL_VALUE: { 2200 unsigned RematCost = TTI->getGISelRematGlobalCost(); 2201 Register Reg = MI.getOperand(0).getReg(); 2202 unsigned MaxUses = maxUses(RematCost); 2203 if (MaxUses == UINT_MAX) 2204 return true; // Remats are "free" so always localize. 2205 bool B = isUsesAtMost(Reg, MaxUses); 2206 return B; 2207 } 2208 } 2209 } 2210