1 //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This implements the TargetLoweringBase class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/BitVector.h" 14 #include "llvm/ADT/STLExtras.h" 15 #include "llvm/ADT/SmallVector.h" 16 #include "llvm/ADT/StringExtras.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/ADT/Triple.h" 19 #include "llvm/ADT/Twine.h" 20 #include "llvm/Analysis/Loads.h" 21 #include "llvm/Analysis/TargetTransformInfo.h" 22 #include "llvm/CodeGen/Analysis.h" 23 #include "llvm/CodeGen/ISDOpcodes.h" 24 #include "llvm/CodeGen/MachineBasicBlock.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineInstr.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineMemOperand.h" 30 #include "llvm/CodeGen/MachineOperand.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/RuntimeLibcalls.h" 33 #include "llvm/CodeGen/StackMaps.h" 34 #include "llvm/CodeGen/TargetLowering.h" 35 #include "llvm/CodeGen/TargetOpcodes.h" 36 #include "llvm/CodeGen/TargetRegisterInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/CallingConv.h" 40 #include "llvm/IR/DataLayout.h" 41 #include "llvm/IR/DerivedTypes.h" 42 #include "llvm/IR/Function.h" 43 #include "llvm/IR/GlobalValue.h" 44 #include "llvm/IR/GlobalVariable.h" 45 #include "llvm/IR/IRBuilder.h" 46 #include "llvm/IR/Module.h" 47 #include "llvm/IR/Type.h" 48 #include "llvm/Support/Casting.h" 49 #include "llvm/Support/CommandLine.h" 50 #include "llvm/Support/Compiler.h" 51 #include "llvm/Support/ErrorHandling.h" 52 #include "llvm/Support/MachineValueType.h" 53 #include "llvm/Support/MathExtras.h" 54 #include "llvm/Target/TargetMachine.h" 55 #include "llvm/Target/TargetOptions.h" 56 #include "llvm/Transforms/Utils/SizeOpts.h" 57 #include <algorithm> 58 #include <cassert> 59 #include <cstddef> 60 #include <cstdint> 61 #include <cstring> 62 #include <iterator> 63 #include <string> 64 #include <tuple> 65 #include <utility> 66 67 using namespace llvm; 68 69 static cl::opt<bool> JumpIsExpensiveOverride( 70 "jump-is-expensive", cl::init(false), 71 cl::desc("Do not create extra branches to split comparison logic."), 72 cl::Hidden); 73 74 static cl::opt<unsigned> MinimumJumpTableEntries 75 ("min-jump-table-entries", cl::init(4), cl::Hidden, 76 cl::desc("Set minimum number of entries to use a jump table.")); 77 78 static cl::opt<unsigned> MaximumJumpTableSize 79 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, 80 cl::desc("Set maximum size of jump tables.")); 81 82 /// Minimum jump table density for normal functions. 83 static cl::opt<unsigned> 84 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, 85 cl::desc("Minimum density for building a jump table in " 86 "a normal function")); 87 88 /// Minimum jump table density for -Os or -Oz functions. 89 static cl::opt<unsigned> OptsizeJumpTableDensity( 90 "optsize-jump-table-density", cl::init(40), cl::Hidden, 91 cl::desc("Minimum density for building a jump table in " 92 "an optsize function")); 93 94 // FIXME: This option is only to test if the strict fp operation processed 95 // correctly by preventing mutating strict fp operation to normal fp operation 96 // during development. When the backend supports strict float operation, this 97 // option will be meaningless. 98 static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation", 99 cl::desc("Don't mutate strict-float node to a legalize node"), 100 cl::init(false), cl::Hidden); 101 102 static bool darwinHasSinCos(const Triple &TT) { 103 assert(TT.isOSDarwin() && "should be called with darwin triple"); 104 // Don't bother with 32 bit x86. 105 if (TT.getArch() == Triple::x86) 106 return false; 107 // Macos < 10.9 has no sincos_stret. 108 if (TT.isMacOSX()) 109 return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit(); 110 // iOS < 7.0 has no sincos_stret. 111 if (TT.isiOS()) 112 return !TT.isOSVersionLT(7, 0); 113 // Any other darwin such as WatchOS/TvOS is new enough. 114 return true; 115 } 116 117 void TargetLoweringBase::InitLibcalls(const Triple &TT) { 118 #define HANDLE_LIBCALL(code, name) \ 119 setLibcallName(RTLIB::code, name); 120 #include "llvm/IR/RuntimeLibcalls.def" 121 #undef HANDLE_LIBCALL 122 // Initialize calling conventions to their default. 123 for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC) 124 setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C); 125 126 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf". 127 if (TT.isPPC()) { 128 setLibcallName(RTLIB::ADD_F128, "__addkf3"); 129 setLibcallName(RTLIB::SUB_F128, "__subkf3"); 130 setLibcallName(RTLIB::MUL_F128, "__mulkf3"); 131 setLibcallName(RTLIB::DIV_F128, "__divkf3"); 132 setLibcallName(RTLIB::POWI_F128, "__powikf2"); 133 setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2"); 134 setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2"); 135 setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2"); 136 setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2"); 137 setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi"); 138 setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi"); 139 setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti"); 140 setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi"); 141 setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi"); 142 setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti"); 143 setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf"); 144 setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf"); 145 setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf"); 146 setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf"); 147 setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf"); 148 setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf"); 149 setLibcallName(RTLIB::OEQ_F128, "__eqkf2"); 150 setLibcallName(RTLIB::UNE_F128, "__nekf2"); 151 setLibcallName(RTLIB::OGE_F128, "__gekf2"); 152 setLibcallName(RTLIB::OLT_F128, "__ltkf2"); 153 setLibcallName(RTLIB::OLE_F128, "__lekf2"); 154 setLibcallName(RTLIB::OGT_F128, "__gtkf2"); 155 setLibcallName(RTLIB::UO_F128, "__unordkf2"); 156 } 157 158 // A few names are different on particular architectures or environments. 159 if (TT.isOSDarwin()) { 160 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead 161 // of the gnueabi-style __gnu_*_ieee. 162 // FIXME: What about other targets? 163 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); 164 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); 165 166 // Some darwins have an optimized __bzero/bzero function. 167 switch (TT.getArch()) { 168 case Triple::x86: 169 case Triple::x86_64: 170 if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6)) 171 setLibcallName(RTLIB::BZERO, "__bzero"); 172 break; 173 case Triple::aarch64: 174 case Triple::aarch64_32: 175 setLibcallName(RTLIB::BZERO, "bzero"); 176 break; 177 default: 178 break; 179 } 180 181 if (darwinHasSinCos(TT)) { 182 setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret"); 183 setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret"); 184 if (TT.isWatchABI()) { 185 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32, 186 CallingConv::ARM_AAPCS_VFP); 187 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64, 188 CallingConv::ARM_AAPCS_VFP); 189 } 190 } 191 } else { 192 setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee"); 193 setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee"); 194 } 195 196 if (TT.isGNUEnvironment() || TT.isOSFuchsia() || 197 (TT.isAndroid() && !TT.isAndroidVersionLT(9))) { 198 setLibcallName(RTLIB::SINCOS_F32, "sincosf"); 199 setLibcallName(RTLIB::SINCOS_F64, "sincos"); 200 setLibcallName(RTLIB::SINCOS_F80, "sincosl"); 201 setLibcallName(RTLIB::SINCOS_F128, "sincosl"); 202 setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl"); 203 } 204 205 if (TT.isPS4CPU()) { 206 setLibcallName(RTLIB::SINCOS_F32, "sincosf"); 207 setLibcallName(RTLIB::SINCOS_F64, "sincos"); 208 } 209 210 if (TT.isOSOpenBSD()) { 211 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr); 212 } 213 } 214 215 /// GetFPLibCall - Helper to return the right libcall for the given floating 216 /// point type, or UNKNOWN_LIBCALL if there is none. 217 RTLIB::Libcall RTLIB::getFPLibCall(EVT VT, 218 RTLIB::Libcall Call_F32, 219 RTLIB::Libcall Call_F64, 220 RTLIB::Libcall Call_F80, 221 RTLIB::Libcall Call_F128, 222 RTLIB::Libcall Call_PPCF128) { 223 return 224 VT == MVT::f32 ? Call_F32 : 225 VT == MVT::f64 ? Call_F64 : 226 VT == MVT::f80 ? Call_F80 : 227 VT == MVT::f128 ? Call_F128 : 228 VT == MVT::ppcf128 ? Call_PPCF128 : 229 RTLIB::UNKNOWN_LIBCALL; 230 } 231 232 /// getFPEXT - Return the FPEXT_*_* value for the given types, or 233 /// UNKNOWN_LIBCALL if there is none. 234 RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { 235 if (OpVT == MVT::f16) { 236 if (RetVT == MVT::f32) 237 return FPEXT_F16_F32; 238 if (RetVT == MVT::f64) 239 return FPEXT_F16_F64; 240 if (RetVT == MVT::f128) 241 return FPEXT_F16_F128; 242 } else if (OpVT == MVT::f32) { 243 if (RetVT == MVT::f64) 244 return FPEXT_F32_F64; 245 if (RetVT == MVT::f128) 246 return FPEXT_F32_F128; 247 if (RetVT == MVT::ppcf128) 248 return FPEXT_F32_PPCF128; 249 } else if (OpVT == MVT::f64) { 250 if (RetVT == MVT::f128) 251 return FPEXT_F64_F128; 252 else if (RetVT == MVT::ppcf128) 253 return FPEXT_F64_PPCF128; 254 } else if (OpVT == MVT::f80) { 255 if (RetVT == MVT::f128) 256 return FPEXT_F80_F128; 257 } 258 259 return UNKNOWN_LIBCALL; 260 } 261 262 /// getFPROUND - Return the FPROUND_*_* value for the given types, or 263 /// UNKNOWN_LIBCALL if there is none. 264 RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) { 265 if (RetVT == MVT::f16) { 266 if (OpVT == MVT::f32) 267 return FPROUND_F32_F16; 268 if (OpVT == MVT::f64) 269 return FPROUND_F64_F16; 270 if (OpVT == MVT::f80) 271 return FPROUND_F80_F16; 272 if (OpVT == MVT::f128) 273 return FPROUND_F128_F16; 274 if (OpVT == MVT::ppcf128) 275 return FPROUND_PPCF128_F16; 276 } else if (RetVT == MVT::f32) { 277 if (OpVT == MVT::f64) 278 return FPROUND_F64_F32; 279 if (OpVT == MVT::f80) 280 return FPROUND_F80_F32; 281 if (OpVT == MVT::f128) 282 return FPROUND_F128_F32; 283 if (OpVT == MVT::ppcf128) 284 return FPROUND_PPCF128_F32; 285 } else if (RetVT == MVT::f64) { 286 if (OpVT == MVT::f80) 287 return FPROUND_F80_F64; 288 if (OpVT == MVT::f128) 289 return FPROUND_F128_F64; 290 if (OpVT == MVT::ppcf128) 291 return FPROUND_PPCF128_F64; 292 } else if (RetVT == MVT::f80) { 293 if (OpVT == MVT::f128) 294 return FPROUND_F128_F80; 295 } 296 297 return UNKNOWN_LIBCALL; 298 } 299 300 /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or 301 /// UNKNOWN_LIBCALL if there is none. 302 RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) { 303 if (OpVT == MVT::f16) { 304 if (RetVT == MVT::i32) 305 return FPTOSINT_F16_I32; 306 if (RetVT == MVT::i64) 307 return FPTOSINT_F16_I64; 308 if (RetVT == MVT::i128) 309 return FPTOSINT_F16_I128; 310 } else if (OpVT == MVT::f32) { 311 if (RetVT == MVT::i32) 312 return FPTOSINT_F32_I32; 313 if (RetVT == MVT::i64) 314 return FPTOSINT_F32_I64; 315 if (RetVT == MVT::i128) 316 return FPTOSINT_F32_I128; 317 } else if (OpVT == MVT::f64) { 318 if (RetVT == MVT::i32) 319 return FPTOSINT_F64_I32; 320 if (RetVT == MVT::i64) 321 return FPTOSINT_F64_I64; 322 if (RetVT == MVT::i128) 323 return FPTOSINT_F64_I128; 324 } else if (OpVT == MVT::f80) { 325 if (RetVT == MVT::i32) 326 return FPTOSINT_F80_I32; 327 if (RetVT == MVT::i64) 328 return FPTOSINT_F80_I64; 329 if (RetVT == MVT::i128) 330 return FPTOSINT_F80_I128; 331 } else if (OpVT == MVT::f128) { 332 if (RetVT == MVT::i32) 333 return FPTOSINT_F128_I32; 334 if (RetVT == MVT::i64) 335 return FPTOSINT_F128_I64; 336 if (RetVT == MVT::i128) 337 return FPTOSINT_F128_I128; 338 } else if (OpVT == MVT::ppcf128) { 339 if (RetVT == MVT::i32) 340 return FPTOSINT_PPCF128_I32; 341 if (RetVT == MVT::i64) 342 return FPTOSINT_PPCF128_I64; 343 if (RetVT == MVT::i128) 344 return FPTOSINT_PPCF128_I128; 345 } 346 return UNKNOWN_LIBCALL; 347 } 348 349 /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or 350 /// UNKNOWN_LIBCALL if there is none. 351 RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) { 352 if (OpVT == MVT::f16) { 353 if (RetVT == MVT::i32) 354 return FPTOUINT_F16_I32; 355 if (RetVT == MVT::i64) 356 return FPTOUINT_F16_I64; 357 if (RetVT == MVT::i128) 358 return FPTOUINT_F16_I128; 359 } else if (OpVT == MVT::f32) { 360 if (RetVT == MVT::i32) 361 return FPTOUINT_F32_I32; 362 if (RetVT == MVT::i64) 363 return FPTOUINT_F32_I64; 364 if (RetVT == MVT::i128) 365 return FPTOUINT_F32_I128; 366 } else if (OpVT == MVT::f64) { 367 if (RetVT == MVT::i32) 368 return FPTOUINT_F64_I32; 369 if (RetVT == MVT::i64) 370 return FPTOUINT_F64_I64; 371 if (RetVT == MVT::i128) 372 return FPTOUINT_F64_I128; 373 } else if (OpVT == MVT::f80) { 374 if (RetVT == MVT::i32) 375 return FPTOUINT_F80_I32; 376 if (RetVT == MVT::i64) 377 return FPTOUINT_F80_I64; 378 if (RetVT == MVT::i128) 379 return FPTOUINT_F80_I128; 380 } else if (OpVT == MVT::f128) { 381 if (RetVT == MVT::i32) 382 return FPTOUINT_F128_I32; 383 if (RetVT == MVT::i64) 384 return FPTOUINT_F128_I64; 385 if (RetVT == MVT::i128) 386 return FPTOUINT_F128_I128; 387 } else if (OpVT == MVT::ppcf128) { 388 if (RetVT == MVT::i32) 389 return FPTOUINT_PPCF128_I32; 390 if (RetVT == MVT::i64) 391 return FPTOUINT_PPCF128_I64; 392 if (RetVT == MVT::i128) 393 return FPTOUINT_PPCF128_I128; 394 } 395 return UNKNOWN_LIBCALL; 396 } 397 398 /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or 399 /// UNKNOWN_LIBCALL if there is none. 400 RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) { 401 if (OpVT == MVT::i32) { 402 if (RetVT == MVT::f16) 403 return SINTTOFP_I32_F16; 404 if (RetVT == MVT::f32) 405 return SINTTOFP_I32_F32; 406 if (RetVT == MVT::f64) 407 return SINTTOFP_I32_F64; 408 if (RetVT == MVT::f80) 409 return SINTTOFP_I32_F80; 410 if (RetVT == MVT::f128) 411 return SINTTOFP_I32_F128; 412 if (RetVT == MVT::ppcf128) 413 return SINTTOFP_I32_PPCF128; 414 } else if (OpVT == MVT::i64) { 415 if (RetVT == MVT::f16) 416 return SINTTOFP_I64_F16; 417 if (RetVT == MVT::f32) 418 return SINTTOFP_I64_F32; 419 if (RetVT == MVT::f64) 420 return SINTTOFP_I64_F64; 421 if (RetVT == MVT::f80) 422 return SINTTOFP_I64_F80; 423 if (RetVT == MVT::f128) 424 return SINTTOFP_I64_F128; 425 if (RetVT == MVT::ppcf128) 426 return SINTTOFP_I64_PPCF128; 427 } else if (OpVT == MVT::i128) { 428 if (RetVT == MVT::f16) 429 return SINTTOFP_I128_F16; 430 if (RetVT == MVT::f32) 431 return SINTTOFP_I128_F32; 432 if (RetVT == MVT::f64) 433 return SINTTOFP_I128_F64; 434 if (RetVT == MVT::f80) 435 return SINTTOFP_I128_F80; 436 if (RetVT == MVT::f128) 437 return SINTTOFP_I128_F128; 438 if (RetVT == MVT::ppcf128) 439 return SINTTOFP_I128_PPCF128; 440 } 441 return UNKNOWN_LIBCALL; 442 } 443 444 /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or 445 /// UNKNOWN_LIBCALL if there is none. 446 RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { 447 if (OpVT == MVT::i32) { 448 if (RetVT == MVT::f16) 449 return UINTTOFP_I32_F16; 450 if (RetVT == MVT::f32) 451 return UINTTOFP_I32_F32; 452 if (RetVT == MVT::f64) 453 return UINTTOFP_I32_F64; 454 if (RetVT == MVT::f80) 455 return UINTTOFP_I32_F80; 456 if (RetVT == MVT::f128) 457 return UINTTOFP_I32_F128; 458 if (RetVT == MVT::ppcf128) 459 return UINTTOFP_I32_PPCF128; 460 } else if (OpVT == MVT::i64) { 461 if (RetVT == MVT::f16) 462 return UINTTOFP_I64_F16; 463 if (RetVT == MVT::f32) 464 return UINTTOFP_I64_F32; 465 if (RetVT == MVT::f64) 466 return UINTTOFP_I64_F64; 467 if (RetVT == MVT::f80) 468 return UINTTOFP_I64_F80; 469 if (RetVT == MVT::f128) 470 return UINTTOFP_I64_F128; 471 if (RetVT == MVT::ppcf128) 472 return UINTTOFP_I64_PPCF128; 473 } else if (OpVT == MVT::i128) { 474 if (RetVT == MVT::f16) 475 return UINTTOFP_I128_F16; 476 if (RetVT == MVT::f32) 477 return UINTTOFP_I128_F32; 478 if (RetVT == MVT::f64) 479 return UINTTOFP_I128_F64; 480 if (RetVT == MVT::f80) 481 return UINTTOFP_I128_F80; 482 if (RetVT == MVT::f128) 483 return UINTTOFP_I128_F128; 484 if (RetVT == MVT::ppcf128) 485 return UINTTOFP_I128_PPCF128; 486 } 487 return UNKNOWN_LIBCALL; 488 } 489 490 RTLIB::Libcall RTLIB::getPOWI(EVT RetVT) { 491 return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128, 492 POWI_PPCF128); 493 } 494 495 RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, 496 MVT VT) { 497 unsigned ModeN, ModelN; 498 switch (VT.SimpleTy) { 499 case MVT::i8: 500 ModeN = 0; 501 break; 502 case MVT::i16: 503 ModeN = 1; 504 break; 505 case MVT::i32: 506 ModeN = 2; 507 break; 508 case MVT::i64: 509 ModeN = 3; 510 break; 511 case MVT::i128: 512 ModeN = 4; 513 break; 514 default: 515 return UNKNOWN_LIBCALL; 516 } 517 518 switch (Order) { 519 case AtomicOrdering::Monotonic: 520 ModelN = 0; 521 break; 522 case AtomicOrdering::Acquire: 523 ModelN = 1; 524 break; 525 case AtomicOrdering::Release: 526 ModelN = 2; 527 break; 528 case AtomicOrdering::AcquireRelease: 529 case AtomicOrdering::SequentiallyConsistent: 530 ModelN = 3; 531 break; 532 default: 533 return UNKNOWN_LIBCALL; 534 } 535 536 #define LCALLS(A, B) \ 537 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL } 538 #define LCALL5(A) \ 539 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16) 540 switch (Opc) { 541 case ISD::ATOMIC_CMP_SWAP: { 542 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)}; 543 return LC[ModeN][ModelN]; 544 } 545 case ISD::ATOMIC_SWAP: { 546 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)}; 547 return LC[ModeN][ModelN]; 548 } 549 case ISD::ATOMIC_LOAD_ADD: { 550 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)}; 551 return LC[ModeN][ModelN]; 552 } 553 case ISD::ATOMIC_LOAD_OR: { 554 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)}; 555 return LC[ModeN][ModelN]; 556 } 557 case ISD::ATOMIC_LOAD_CLR: { 558 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)}; 559 return LC[ModeN][ModelN]; 560 } 561 case ISD::ATOMIC_LOAD_XOR: { 562 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)}; 563 return LC[ModeN][ModelN]; 564 } 565 default: 566 return UNKNOWN_LIBCALL; 567 } 568 #undef LCALLS 569 #undef LCALL5 570 } 571 572 RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) { 573 #define OP_TO_LIBCALL(Name, Enum) \ 574 case Name: \ 575 switch (VT.SimpleTy) { \ 576 default: \ 577 return UNKNOWN_LIBCALL; \ 578 case MVT::i8: \ 579 return Enum##_1; \ 580 case MVT::i16: \ 581 return Enum##_2; \ 582 case MVT::i32: \ 583 return Enum##_4; \ 584 case MVT::i64: \ 585 return Enum##_8; \ 586 case MVT::i128: \ 587 return Enum##_16; \ 588 } 589 590 switch (Opc) { 591 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET) 592 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP) 593 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD) 594 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB) 595 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND) 596 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR) 597 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR) 598 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND) 599 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX) 600 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX) 601 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN) 602 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN) 603 } 604 605 #undef OP_TO_LIBCALL 606 607 return UNKNOWN_LIBCALL; 608 } 609 610 RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 611 switch (ElementSize) { 612 case 1: 613 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1; 614 case 2: 615 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2; 616 case 4: 617 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4; 618 case 8: 619 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8; 620 case 16: 621 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16; 622 default: 623 return UNKNOWN_LIBCALL; 624 } 625 } 626 627 RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 628 switch (ElementSize) { 629 case 1: 630 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1; 631 case 2: 632 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2; 633 case 4: 634 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4; 635 case 8: 636 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8; 637 case 16: 638 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16; 639 default: 640 return UNKNOWN_LIBCALL; 641 } 642 } 643 644 RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { 645 switch (ElementSize) { 646 case 1: 647 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1; 648 case 2: 649 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2; 650 case 4: 651 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4; 652 case 8: 653 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8; 654 case 16: 655 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16; 656 default: 657 return UNKNOWN_LIBCALL; 658 } 659 } 660 661 /// InitCmpLibcallCCs - Set default comparison libcall CC. 662 static void InitCmpLibcallCCs(ISD::CondCode *CCs) { 663 memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL); 664 CCs[RTLIB::OEQ_F32] = ISD::SETEQ; 665 CCs[RTLIB::OEQ_F64] = ISD::SETEQ; 666 CCs[RTLIB::OEQ_F128] = ISD::SETEQ; 667 CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ; 668 CCs[RTLIB::UNE_F32] = ISD::SETNE; 669 CCs[RTLIB::UNE_F64] = ISD::SETNE; 670 CCs[RTLIB::UNE_F128] = ISD::SETNE; 671 CCs[RTLIB::UNE_PPCF128] = ISD::SETNE; 672 CCs[RTLIB::OGE_F32] = ISD::SETGE; 673 CCs[RTLIB::OGE_F64] = ISD::SETGE; 674 CCs[RTLIB::OGE_F128] = ISD::SETGE; 675 CCs[RTLIB::OGE_PPCF128] = ISD::SETGE; 676 CCs[RTLIB::OLT_F32] = ISD::SETLT; 677 CCs[RTLIB::OLT_F64] = ISD::SETLT; 678 CCs[RTLIB::OLT_F128] = ISD::SETLT; 679 CCs[RTLIB::OLT_PPCF128] = ISD::SETLT; 680 CCs[RTLIB::OLE_F32] = ISD::SETLE; 681 CCs[RTLIB::OLE_F64] = ISD::SETLE; 682 CCs[RTLIB::OLE_F128] = ISD::SETLE; 683 CCs[RTLIB::OLE_PPCF128] = ISD::SETLE; 684 CCs[RTLIB::OGT_F32] = ISD::SETGT; 685 CCs[RTLIB::OGT_F64] = ISD::SETGT; 686 CCs[RTLIB::OGT_F128] = ISD::SETGT; 687 CCs[RTLIB::OGT_PPCF128] = ISD::SETGT; 688 CCs[RTLIB::UO_F32] = ISD::SETNE; 689 CCs[RTLIB::UO_F64] = ISD::SETNE; 690 CCs[RTLIB::UO_F128] = ISD::SETNE; 691 CCs[RTLIB::UO_PPCF128] = ISD::SETNE; 692 } 693 694 /// NOTE: The TargetMachine owns TLOF. 695 TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { 696 initActions(); 697 698 // Perform these initializations only once. 699 MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = 700 MaxLoadsPerMemcmp = 8; 701 MaxGluedStoresPerMemcpy = 0; 702 MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize = 703 MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4; 704 HasMultipleConditionRegisters = false; 705 HasExtractBitsInsn = false; 706 JumpIsExpensive = JumpIsExpensiveOverride; 707 PredictableSelectIsExpensive = false; 708 EnableExtLdPromotion = false; 709 StackPointerRegisterToSaveRestore = 0; 710 BooleanContents = UndefinedBooleanContent; 711 BooleanFloatContents = UndefinedBooleanContent; 712 BooleanVectorContents = UndefinedBooleanContent; 713 SchedPreferenceInfo = Sched::ILP; 714 GatherAllAliasesMaxDepth = 18; 715 IsStrictFPEnabled = DisableStrictNodeMutation; 716 // TODO: the default will be switched to 0 in the next commit, along 717 // with the Target-specific changes necessary. 718 MaxAtomicSizeInBitsSupported = 1024; 719 720 MinCmpXchgSizeInBits = 0; 721 SupportsUnalignedAtomics = false; 722 723 std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr); 724 725 InitLibcalls(TM.getTargetTriple()); 726 InitCmpLibcallCCs(CmpLibcallCCs); 727 } 728 729 void TargetLoweringBase::initActions() { 730 // All operations default to being supported. 731 memset(OpActions, 0, sizeof(OpActions)); 732 memset(LoadExtActions, 0, sizeof(LoadExtActions)); 733 memset(TruncStoreActions, 0, sizeof(TruncStoreActions)); 734 memset(IndexedModeActions, 0, sizeof(IndexedModeActions)); 735 memset(CondCodeActions, 0, sizeof(CondCodeActions)); 736 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr); 737 std::fill(std::begin(TargetDAGCombineArray), 738 std::end(TargetDAGCombineArray), 0); 739 740 for (MVT VT : MVT::fp_valuetypes()) { 741 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits()); 742 if (IntVT.isValid()) { 743 setOperationAction(ISD::ATOMIC_SWAP, VT, Promote); 744 AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT); 745 } 746 } 747 748 // Set default actions for various operations. 749 for (MVT VT : MVT::all_valuetypes()) { 750 // Default all indexed load / store to expand. 751 for (unsigned IM = (unsigned)ISD::PRE_INC; 752 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) { 753 setIndexedLoadAction(IM, VT, Expand); 754 setIndexedStoreAction(IM, VT, Expand); 755 setIndexedMaskedLoadAction(IM, VT, Expand); 756 setIndexedMaskedStoreAction(IM, VT, Expand); 757 } 758 759 // Most backends expect to see the node which just returns the value loaded. 760 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand); 761 762 // These operations default to expand. 763 setOperationAction(ISD::FGETSIGN, VT, Expand); 764 setOperationAction(ISD::ISNAN, VT, Expand); 765 setOperationAction(ISD::CONCAT_VECTORS, VT, Expand); 766 setOperationAction(ISD::FMINNUM, VT, Expand); 767 setOperationAction(ISD::FMAXNUM, VT, Expand); 768 setOperationAction(ISD::FMINNUM_IEEE, VT, Expand); 769 setOperationAction(ISD::FMAXNUM_IEEE, VT, Expand); 770 setOperationAction(ISD::FMINIMUM, VT, Expand); 771 setOperationAction(ISD::FMAXIMUM, VT, Expand); 772 setOperationAction(ISD::FMAD, VT, Expand); 773 setOperationAction(ISD::SMIN, VT, Expand); 774 setOperationAction(ISD::SMAX, VT, Expand); 775 setOperationAction(ISD::UMIN, VT, Expand); 776 setOperationAction(ISD::UMAX, VT, Expand); 777 setOperationAction(ISD::ABS, VT, Expand); 778 setOperationAction(ISD::FSHL, VT, Expand); 779 setOperationAction(ISD::FSHR, VT, Expand); 780 setOperationAction(ISD::SADDSAT, VT, Expand); 781 setOperationAction(ISD::UADDSAT, VT, Expand); 782 setOperationAction(ISD::SSUBSAT, VT, Expand); 783 setOperationAction(ISD::USUBSAT, VT, Expand); 784 setOperationAction(ISD::SSHLSAT, VT, Expand); 785 setOperationAction(ISD::USHLSAT, VT, Expand); 786 setOperationAction(ISD::SMULFIX, VT, Expand); 787 setOperationAction(ISD::SMULFIXSAT, VT, Expand); 788 setOperationAction(ISD::UMULFIX, VT, Expand); 789 setOperationAction(ISD::UMULFIXSAT, VT, Expand); 790 setOperationAction(ISD::SDIVFIX, VT, Expand); 791 setOperationAction(ISD::SDIVFIXSAT, VT, Expand); 792 setOperationAction(ISD::UDIVFIX, VT, Expand); 793 setOperationAction(ISD::UDIVFIXSAT, VT, Expand); 794 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Expand); 795 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Expand); 796 797 // Overflow operations default to expand 798 setOperationAction(ISD::SADDO, VT, Expand); 799 setOperationAction(ISD::SSUBO, VT, Expand); 800 setOperationAction(ISD::UADDO, VT, Expand); 801 setOperationAction(ISD::USUBO, VT, Expand); 802 setOperationAction(ISD::SMULO, VT, Expand); 803 setOperationAction(ISD::UMULO, VT, Expand); 804 805 // ADDCARRY operations default to expand 806 setOperationAction(ISD::ADDCARRY, VT, Expand); 807 setOperationAction(ISD::SUBCARRY, VT, Expand); 808 setOperationAction(ISD::SETCCCARRY, VT, Expand); 809 setOperationAction(ISD::SADDO_CARRY, VT, Expand); 810 setOperationAction(ISD::SSUBO_CARRY, VT, Expand); 811 812 // ADDC/ADDE/SUBC/SUBE default to expand. 813 setOperationAction(ISD::ADDC, VT, Expand); 814 setOperationAction(ISD::ADDE, VT, Expand); 815 setOperationAction(ISD::SUBC, VT, Expand); 816 setOperationAction(ISD::SUBE, VT, Expand); 817 818 // Absolute difference 819 setOperationAction(ISD::ABDS, VT, Expand); 820 setOperationAction(ISD::ABDU, VT, Expand); 821 822 // These default to Expand so they will be expanded to CTLZ/CTTZ by default. 823 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 824 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 825 826 setOperationAction(ISD::BITREVERSE, VT, Expand); 827 setOperationAction(ISD::PARITY, VT, Expand); 828 829 // These library functions default to expand. 830 setOperationAction(ISD::FROUND, VT, Expand); 831 setOperationAction(ISD::FROUNDEVEN, VT, Expand); 832 setOperationAction(ISD::FPOWI, VT, Expand); 833 834 // These operations default to expand for vector types. 835 if (VT.isVector()) { 836 setOperationAction(ISD::FCOPYSIGN, VT, Expand); 837 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 838 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Expand); 839 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Expand); 840 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand); 841 setOperationAction(ISD::SPLAT_VECTOR, VT, Expand); 842 } 843 844 // Constrained floating-point operations default to expand. 845 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 846 setOperationAction(ISD::STRICT_##DAGN, VT, Expand); 847 #include "llvm/IR/ConstrainedOps.def" 848 849 // For most targets @llvm.get.dynamic.area.offset just returns 0. 850 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand); 851 852 // Vector reduction default to expand. 853 setOperationAction(ISD::VECREDUCE_FADD, VT, Expand); 854 setOperationAction(ISD::VECREDUCE_FMUL, VT, Expand); 855 setOperationAction(ISD::VECREDUCE_ADD, VT, Expand); 856 setOperationAction(ISD::VECREDUCE_MUL, VT, Expand); 857 setOperationAction(ISD::VECREDUCE_AND, VT, Expand); 858 setOperationAction(ISD::VECREDUCE_OR, VT, Expand); 859 setOperationAction(ISD::VECREDUCE_XOR, VT, Expand); 860 setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand); 861 setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand); 862 setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand); 863 setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand); 864 setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand); 865 setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand); 866 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Expand); 867 setOperationAction(ISD::VECREDUCE_SEQ_FMUL, VT, Expand); 868 869 // Named vector shuffles default to expand. 870 setOperationAction(ISD::VECTOR_SPLICE, VT, Expand); 871 } 872 873 // Most targets ignore the @llvm.prefetch intrinsic. 874 setOperationAction(ISD::PREFETCH, MVT::Other, Expand); 875 876 // Most targets also ignore the @llvm.readcyclecounter intrinsic. 877 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand); 878 879 // ConstantFP nodes default to expand. Targets can either change this to 880 // Legal, in which case all fp constants are legal, or use isFPImmLegal() 881 // to optimize expansions for certain constants. 882 setOperationAction(ISD::ConstantFP, MVT::f16, Expand); 883 setOperationAction(ISD::ConstantFP, MVT::f32, Expand); 884 setOperationAction(ISD::ConstantFP, MVT::f64, Expand); 885 setOperationAction(ISD::ConstantFP, MVT::f80, Expand); 886 setOperationAction(ISD::ConstantFP, MVT::f128, Expand); 887 888 // These library functions default to expand. 889 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) { 890 setOperationAction(ISD::FCBRT, VT, Expand); 891 setOperationAction(ISD::FLOG , VT, Expand); 892 setOperationAction(ISD::FLOG2, VT, Expand); 893 setOperationAction(ISD::FLOG10, VT, Expand); 894 setOperationAction(ISD::FEXP , VT, Expand); 895 setOperationAction(ISD::FEXP2, VT, Expand); 896 setOperationAction(ISD::FFLOOR, VT, Expand); 897 setOperationAction(ISD::FNEARBYINT, VT, Expand); 898 setOperationAction(ISD::FCEIL, VT, Expand); 899 setOperationAction(ISD::FRINT, VT, Expand); 900 setOperationAction(ISD::FTRUNC, VT, Expand); 901 setOperationAction(ISD::FROUND, VT, Expand); 902 setOperationAction(ISD::FROUNDEVEN, VT, Expand); 903 setOperationAction(ISD::LROUND, VT, Expand); 904 setOperationAction(ISD::LLROUND, VT, Expand); 905 setOperationAction(ISD::LRINT, VT, Expand); 906 setOperationAction(ISD::LLRINT, VT, Expand); 907 } 908 909 // Default ISD::TRAP to expand (which turns it into abort). 910 setOperationAction(ISD::TRAP, MVT::Other, Expand); 911 912 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand" 913 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP. 914 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand); 915 916 setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand); 917 } 918 919 MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL, 920 EVT) const { 921 return MVT::getIntegerVT(DL.getPointerSizeInBits(0)); 922 } 923 924 EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL, 925 bool LegalTypes) const { 926 assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); 927 if (LHSTy.isVector()) 928 return LHSTy; 929 return LegalTypes ? getScalarShiftAmountTy(DL, LHSTy) 930 : getPointerTy(DL); 931 } 932 933 bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const { 934 assert(isTypeLegal(VT)); 935 switch (Op) { 936 default: 937 return false; 938 case ISD::SDIV: 939 case ISD::UDIV: 940 case ISD::SREM: 941 case ISD::UREM: 942 return true; 943 } 944 } 945 946 bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS, 947 unsigned DestAS) const { 948 return TM.isNoopAddrSpaceCast(SrcAS, DestAS); 949 } 950 951 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) { 952 // If the command-line option was specified, ignore this request. 953 if (!JumpIsExpensiveOverride.getNumOccurrences()) 954 JumpIsExpensive = isExpensive; 955 } 956 957 TargetLoweringBase::LegalizeKind 958 TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const { 959 // If this is a simple type, use the ComputeRegisterProp mechanism. 960 if (VT.isSimple()) { 961 MVT SVT = VT.getSimpleVT(); 962 assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType)); 963 MVT NVT = TransformToType[SVT.SimpleTy]; 964 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT); 965 966 assert((LA == TypeLegal || LA == TypeSoftenFloat || 967 LA == TypeSoftPromoteHalf || 968 (NVT.isVector() || 969 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) && 970 "Promote may not follow Expand or Promote"); 971 972 if (LA == TypeSplitVector) 973 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context)); 974 if (LA == TypeScalarizeVector) 975 return LegalizeKind(LA, SVT.getVectorElementType()); 976 return LegalizeKind(LA, NVT); 977 } 978 979 // Handle Extended Scalar Types. 980 if (!VT.isVector()) { 981 assert(VT.isInteger() && "Float types must be simple"); 982 unsigned BitSize = VT.getSizeInBits(); 983 // First promote to a power-of-two size, then expand if necessary. 984 if (BitSize < 8 || !isPowerOf2_32(BitSize)) { 985 EVT NVT = VT.getRoundIntegerType(Context); 986 assert(NVT != VT && "Unable to round integer VT"); 987 LegalizeKind NextStep = getTypeConversion(Context, NVT); 988 // Avoid multi-step promotion. 989 if (NextStep.first == TypePromoteInteger) 990 return NextStep; 991 // Return rounded integer type. 992 return LegalizeKind(TypePromoteInteger, NVT); 993 } 994 995 return LegalizeKind(TypeExpandInteger, 996 EVT::getIntegerVT(Context, VT.getSizeInBits() / 2)); 997 } 998 999 // Handle vector types. 1000 ElementCount NumElts = VT.getVectorElementCount(); 1001 EVT EltVT = VT.getVectorElementType(); 1002 1003 // Vectors with only one element are always scalarized. 1004 if (NumElts.isScalar()) 1005 return LegalizeKind(TypeScalarizeVector, EltVT); 1006 1007 // Try to widen vector elements until the element type is a power of two and 1008 // promote it to a legal type later on, for example: 1009 // <3 x i8> -> <4 x i8> -> <4 x i32> 1010 if (EltVT.isInteger()) { 1011 // Vectors with a number of elements that is not a power of two are always 1012 // widened, for example <3 x i8> -> <4 x i8>. 1013 if (!VT.isPow2VectorType()) { 1014 NumElts = NumElts.coefficientNextPowerOf2(); 1015 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); 1016 return LegalizeKind(TypeWidenVector, NVT); 1017 } 1018 1019 // Examine the element type. 1020 LegalizeKind LK = getTypeConversion(Context, EltVT); 1021 1022 // If type is to be expanded, split the vector. 1023 // <4 x i140> -> <2 x i140> 1024 if (LK.first == TypeExpandInteger) { 1025 if (VT.getVectorElementCount().isScalable()) 1026 return LegalizeKind(TypeScalarizeScalableVector, EltVT); 1027 return LegalizeKind(TypeSplitVector, 1028 VT.getHalfNumVectorElementsVT(Context)); 1029 } 1030 1031 // Promote the integer element types until a legal vector type is found 1032 // or until the element integer type is too big. If a legal type was not 1033 // found, fallback to the usual mechanism of widening/splitting the 1034 // vector. 1035 EVT OldEltVT = EltVT; 1036 while (true) { 1037 // Increase the bitwidth of the element to the next pow-of-two 1038 // (which is greater than 8 bits). 1039 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits()) 1040 .getRoundIntegerType(Context); 1041 1042 // Stop trying when getting a non-simple element type. 1043 // Note that vector elements may be greater than legal vector element 1044 // types. Example: X86 XMM registers hold 64bit element on 32bit 1045 // systems. 1046 if (!EltVT.isSimple()) 1047 break; 1048 1049 // Build a new vector type and check if it is legal. 1050 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); 1051 // Found a legal promoted vector type. 1052 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal) 1053 return LegalizeKind(TypePromoteInteger, 1054 EVT::getVectorVT(Context, EltVT, NumElts)); 1055 } 1056 1057 // Reset the type to the unexpanded type if we did not find a legal vector 1058 // type with a promoted vector element type. 1059 EltVT = OldEltVT; 1060 } 1061 1062 // Try to widen the vector until a legal type is found. 1063 // If there is no wider legal type, split the vector. 1064 while (true) { 1065 // Round up to the next power of 2. 1066 NumElts = NumElts.coefficientNextPowerOf2(); 1067 1068 // If there is no simple vector type with this many elements then there 1069 // cannot be a larger legal vector type. Note that this assumes that 1070 // there are no skipped intermediate vector types in the simple types. 1071 if (!EltVT.isSimple()) 1072 break; 1073 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); 1074 if (LargerVector == MVT()) 1075 break; 1076 1077 // If this type is legal then widen the vector. 1078 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal) 1079 return LegalizeKind(TypeWidenVector, LargerVector); 1080 } 1081 1082 // Widen odd vectors to next power of two. 1083 if (!VT.isPow2VectorType()) { 1084 EVT NVT = VT.getPow2VectorType(Context); 1085 return LegalizeKind(TypeWidenVector, NVT); 1086 } 1087 1088 if (VT.getVectorElementCount() == ElementCount::getScalable(1)) 1089 return LegalizeKind(TypeScalarizeScalableVector, EltVT); 1090 1091 // Vectors with illegal element types are expanded. 1092 EVT NVT = EVT::getVectorVT(Context, EltVT, 1093 VT.getVectorElementCount().divideCoefficientBy(2)); 1094 return LegalizeKind(TypeSplitVector, NVT); 1095 } 1096 1097 static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, 1098 unsigned &NumIntermediates, 1099 MVT &RegisterVT, 1100 TargetLoweringBase *TLI) { 1101 // Figure out the right, legal destination reg to copy into. 1102 ElementCount EC = VT.getVectorElementCount(); 1103 MVT EltTy = VT.getVectorElementType(); 1104 1105 unsigned NumVectorRegs = 1; 1106 1107 // Scalable vectors cannot be scalarized, so splitting or widening is 1108 // required. 1109 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue())) 1110 llvm_unreachable( 1111 "Splitting or widening of non-power-of-2 MVTs is not implemented."); 1112 1113 // FIXME: We don't support non-power-of-2-sized vectors for now. 1114 // Ideally we could break down into LHS/RHS like LegalizeDAG does. 1115 if (!isPowerOf2_32(EC.getKnownMinValue())) { 1116 // Split EC to unit size (scalable property is preserved). 1117 NumVectorRegs = EC.getKnownMinValue(); 1118 EC = ElementCount::getFixed(1); 1119 } 1120 1121 // Divide the input until we get to a supported size. This will 1122 // always end up with an EC that represent a scalar or a scalable 1123 // scalar. 1124 while (EC.getKnownMinValue() > 1 && 1125 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) { 1126 EC = EC.divideCoefficientBy(2); 1127 NumVectorRegs <<= 1; 1128 } 1129 1130 NumIntermediates = NumVectorRegs; 1131 1132 MVT NewVT = MVT::getVectorVT(EltTy, EC); 1133 if (!TLI->isTypeLegal(NewVT)) 1134 NewVT = EltTy; 1135 IntermediateVT = NewVT; 1136 1137 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits(); 1138 1139 // Convert sizes such as i33 to i64. 1140 if (!isPowerOf2_32(LaneSizeInBits)) 1141 LaneSizeInBits = NextPowerOf2(LaneSizeInBits); 1142 1143 MVT DestVT = TLI->getRegisterType(NewVT); 1144 RegisterVT = DestVT; 1145 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. 1146 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits()); 1147 1148 // Otherwise, promotion or legal types use the same number of registers as 1149 // the vector decimated to the appropriate level. 1150 return NumVectorRegs; 1151 } 1152 1153 /// isLegalRC - Return true if the value types that can be represented by the 1154 /// specified register class are all legal. 1155 bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI, 1156 const TargetRegisterClass &RC) const { 1157 for (auto I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I) 1158 if (isTypeLegal(*I)) 1159 return true; 1160 return false; 1161 } 1162 1163 /// Replace/modify any TargetFrameIndex operands with a targte-dependent 1164 /// sequence of memory operands that is recognized by PrologEpilogInserter. 1165 MachineBasicBlock * 1166 TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, 1167 MachineBasicBlock *MBB) const { 1168 MachineInstr *MI = &InitialMI; 1169 MachineFunction &MF = *MI->getMF(); 1170 MachineFrameInfo &MFI = MF.getFrameInfo(); 1171 1172 // We're handling multiple types of operands here: 1173 // PATCHPOINT MetaArgs - live-in, read only, direct 1174 // STATEPOINT Deopt Spill - live-through, read only, indirect 1175 // STATEPOINT Deopt Alloca - live-through, read only, direct 1176 // (We're currently conservative and mark the deopt slots read/write in 1177 // practice.) 1178 // STATEPOINT GC Spill - live-through, read/write, indirect 1179 // STATEPOINT GC Alloca - live-through, read/write, direct 1180 // The live-in vs live-through is handled already (the live through ones are 1181 // all stack slots), but we need to handle the different type of stackmap 1182 // operands and memory effects here. 1183 1184 if (!llvm::any_of(MI->operands(), 1185 [](MachineOperand &Operand) { return Operand.isFI(); })) 1186 return MBB; 1187 1188 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc()); 1189 1190 // Inherit previous memory operands. 1191 MIB.cloneMemRefs(*MI); 1192 1193 for (unsigned i = 0; i < MI->getNumOperands(); ++i) { 1194 MachineOperand &MO = MI->getOperand(i); 1195 if (!MO.isFI()) { 1196 // Index of Def operand this Use it tied to. 1197 // Since Defs are coming before Uses, if Use is tied, then 1198 // index of Def must be smaller that index of that Use. 1199 // Also, Defs preserve their position in new MI. 1200 unsigned TiedTo = i; 1201 if (MO.isReg() && MO.isTied()) 1202 TiedTo = MI->findTiedOperandIdx(i); 1203 MIB.add(MO); 1204 if (TiedTo < i) 1205 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1); 1206 continue; 1207 } 1208 1209 // foldMemoryOperand builds a new MI after replacing a single FI operand 1210 // with the canonical set of five x86 addressing-mode operands. 1211 int FI = MO.getIndex(); 1212 1213 // Add frame index operands recognized by stackmaps.cpp 1214 if (MFI.isStatepointSpillSlotObjectIndex(FI)) { 1215 // indirect-mem-ref tag, size, #FI, offset. 1216 // Used for spills inserted by StatepointLowering. This codepath is not 1217 // used for patchpoints/stackmaps at all, for these spilling is done via 1218 // foldMemoryOperand callback only. 1219 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity"); 1220 MIB.addImm(StackMaps::IndirectMemRefOp); 1221 MIB.addImm(MFI.getObjectSize(FI)); 1222 MIB.add(MO); 1223 MIB.addImm(0); 1224 } else { 1225 // direct-mem-ref tag, #FI, offset. 1226 // Used by patchpoint, and direct alloca arguments to statepoints 1227 MIB.addImm(StackMaps::DirectMemRefOp); 1228 MIB.add(MO); 1229 MIB.addImm(0); 1230 } 1231 1232 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!"); 1233 1234 // Add a new memory operand for this FI. 1235 assert(MFI.getObjectOffset(FI) != -1); 1236 1237 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and 1238 // PATCHPOINT should be updated to do the same. (TODO) 1239 if (MI->getOpcode() != TargetOpcode::STATEPOINT) { 1240 auto Flags = MachineMemOperand::MOLoad; 1241 MachineMemOperand *MMO = MF.getMachineMemOperand( 1242 MachinePointerInfo::getFixedStack(MF, FI), Flags, 1243 MF.getDataLayout().getPointerSize(), MFI.getObjectAlign(FI)); 1244 MIB->addMemOperand(MF, MMO); 1245 } 1246 } 1247 MBB->insert(MachineBasicBlock::iterator(MI), MIB); 1248 MI->eraseFromParent(); 1249 return MBB; 1250 } 1251 1252 /// findRepresentativeClass - Return the largest legal super-reg register class 1253 /// of the register class for the specified type and its associated "cost". 1254 // This function is in TargetLowering because it uses RegClassForVT which would 1255 // need to be moved to TargetRegisterInfo and would necessitate moving 1256 // isTypeLegal over as well - a massive change that would just require 1257 // TargetLowering having a TargetRegisterInfo class member that it would use. 1258 std::pair<const TargetRegisterClass *, uint8_t> 1259 TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI, 1260 MVT VT) const { 1261 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy]; 1262 if (!RC) 1263 return std::make_pair(RC, 0); 1264 1265 // Compute the set of all super-register classes. 1266 BitVector SuperRegRC(TRI->getNumRegClasses()); 1267 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI) 1268 SuperRegRC.setBitsInMask(RCI.getMask()); 1269 1270 // Find the first legal register class with the largest spill size. 1271 const TargetRegisterClass *BestRC = RC; 1272 for (unsigned i : SuperRegRC.set_bits()) { 1273 const TargetRegisterClass *SuperRC = TRI->getRegClass(i); 1274 // We want the largest possible spill size. 1275 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC)) 1276 continue; 1277 if (!isLegalRC(*TRI, *SuperRC)) 1278 continue; 1279 BestRC = SuperRC; 1280 } 1281 return std::make_pair(BestRC, 1); 1282 } 1283 1284 /// computeRegisterProperties - Once all of the register classes are added, 1285 /// this allows us to compute derived properties we expose. 1286 void TargetLoweringBase::computeRegisterProperties( 1287 const TargetRegisterInfo *TRI) { 1288 static_assert(MVT::VALUETYPE_SIZE <= MVT::MAX_ALLOWED_VALUETYPE, 1289 "Too many value types for ValueTypeActions to hold!"); 1290 1291 // Everything defaults to needing one register. 1292 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) { 1293 NumRegistersForVT[i] = 1; 1294 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i; 1295 } 1296 // ...except isVoid, which doesn't need any registers. 1297 NumRegistersForVT[MVT::isVoid] = 0; 1298 1299 // Find the largest integer register class. 1300 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE; 1301 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg) 1302 assert(LargestIntReg != MVT::i1 && "No integer registers defined!"); 1303 1304 // Every integer value type larger than this largest register takes twice as 1305 // many registers to represent as the previous ValueType. 1306 for (unsigned ExpandedReg = LargestIntReg + 1; 1307 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) { 1308 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1]; 1309 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg; 1310 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1); 1311 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg, 1312 TypeExpandInteger); 1313 } 1314 1315 // Inspect all of the ValueType's smaller than the largest integer 1316 // register to see which ones need promotion. 1317 unsigned LegalIntReg = LargestIntReg; 1318 for (unsigned IntReg = LargestIntReg - 1; 1319 IntReg >= (unsigned)MVT::i1; --IntReg) { 1320 MVT IVT = (MVT::SimpleValueType)IntReg; 1321 if (isTypeLegal(IVT)) { 1322 LegalIntReg = IntReg; 1323 } else { 1324 RegisterTypeForVT[IntReg] = TransformToType[IntReg] = 1325 (MVT::SimpleValueType)LegalIntReg; 1326 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger); 1327 } 1328 } 1329 1330 // ppcf128 type is really two f64's. 1331 if (!isTypeLegal(MVT::ppcf128)) { 1332 if (isTypeLegal(MVT::f64)) { 1333 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64]; 1334 RegisterTypeForVT[MVT::ppcf128] = MVT::f64; 1335 TransformToType[MVT::ppcf128] = MVT::f64; 1336 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat); 1337 } else { 1338 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128]; 1339 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128]; 1340 TransformToType[MVT::ppcf128] = MVT::i128; 1341 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat); 1342 } 1343 } 1344 1345 // Decide how to handle f128. If the target does not have native f128 support, 1346 // expand it to i128 and we will be generating soft float library calls. 1347 if (!isTypeLegal(MVT::f128)) { 1348 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128]; 1349 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128]; 1350 TransformToType[MVT::f128] = MVT::i128; 1351 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); 1352 } 1353 1354 // Decide how to handle f64. If the target does not have native f64 support, 1355 // expand it to i64 and we will be generating soft float library calls. 1356 if (!isTypeLegal(MVT::f64)) { 1357 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64]; 1358 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64]; 1359 TransformToType[MVT::f64] = MVT::i64; 1360 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); 1361 } 1362 1363 // Decide how to handle f32. If the target does not have native f32 support, 1364 // expand it to i32 and we will be generating soft float library calls. 1365 if (!isTypeLegal(MVT::f32)) { 1366 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; 1367 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; 1368 TransformToType[MVT::f32] = MVT::i32; 1369 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); 1370 } 1371 1372 // Decide how to handle f16. If the target does not have native f16 support, 1373 // promote it to f32, because there are no f16 library calls (except for 1374 // conversions). 1375 if (!isTypeLegal(MVT::f16)) { 1376 // Allow targets to control how we legalize half. 1377 if (softPromoteHalfType()) { 1378 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16]; 1379 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16]; 1380 TransformToType[MVT::f16] = MVT::f32; 1381 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf); 1382 } else { 1383 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32]; 1384 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32]; 1385 TransformToType[MVT::f16] = MVT::f32; 1386 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat); 1387 } 1388 } 1389 1390 // Loop over all of the vector value types to see which need transformations. 1391 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; 1392 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { 1393 MVT VT = (MVT::SimpleValueType) i; 1394 if (isTypeLegal(VT)) 1395 continue; 1396 1397 MVT EltVT = VT.getVectorElementType(); 1398 ElementCount EC = VT.getVectorElementCount(); 1399 bool IsLegalWiderType = false; 1400 bool IsScalable = VT.isScalableVector(); 1401 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT); 1402 switch (PreferredAction) { 1403 case TypePromoteInteger: { 1404 MVT::SimpleValueType EndVT = IsScalable ? 1405 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE : 1406 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE; 1407 // Try to promote the elements of integer vectors. If no legal 1408 // promotion was found, fall through to the widen-vector method. 1409 for (unsigned nVT = i + 1; 1410 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) { 1411 MVT SVT = (MVT::SimpleValueType) nVT; 1412 // Promote vectors of integers to vectors with the same number 1413 // of elements, with a wider element type. 1414 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() && 1415 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) { 1416 TransformToType[i] = SVT; 1417 RegisterTypeForVT[i] = SVT; 1418 NumRegistersForVT[i] = 1; 1419 ValueTypeActions.setTypeAction(VT, TypePromoteInteger); 1420 IsLegalWiderType = true; 1421 break; 1422 } 1423 } 1424 if (IsLegalWiderType) 1425 break; 1426 LLVM_FALLTHROUGH; 1427 } 1428 1429 case TypeWidenVector: 1430 if (isPowerOf2_32(EC.getKnownMinValue())) { 1431 // Try to widen the vector. 1432 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { 1433 MVT SVT = (MVT::SimpleValueType) nVT; 1434 if (SVT.getVectorElementType() == EltVT && 1435 SVT.isScalableVector() == IsScalable && 1436 SVT.getVectorElementCount().getKnownMinValue() > 1437 EC.getKnownMinValue() && 1438 isTypeLegal(SVT)) { 1439 TransformToType[i] = SVT; 1440 RegisterTypeForVT[i] = SVT; 1441 NumRegistersForVT[i] = 1; 1442 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1443 IsLegalWiderType = true; 1444 break; 1445 } 1446 } 1447 if (IsLegalWiderType) 1448 break; 1449 } else { 1450 // Only widen to the next power of 2 to keep consistency with EVT. 1451 MVT NVT = VT.getPow2VectorType(); 1452 if (isTypeLegal(NVT)) { 1453 TransformToType[i] = NVT; 1454 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1455 RegisterTypeForVT[i] = NVT; 1456 NumRegistersForVT[i] = 1; 1457 break; 1458 } 1459 } 1460 LLVM_FALLTHROUGH; 1461 1462 case TypeSplitVector: 1463 case TypeScalarizeVector: { 1464 MVT IntermediateVT; 1465 MVT RegisterVT; 1466 unsigned NumIntermediates; 1467 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT, 1468 NumIntermediates, RegisterVT, this); 1469 NumRegistersForVT[i] = NumRegisters; 1470 assert(NumRegistersForVT[i] == NumRegisters && 1471 "NumRegistersForVT size cannot represent NumRegisters!"); 1472 RegisterTypeForVT[i] = RegisterVT; 1473 1474 MVT NVT = VT.getPow2VectorType(); 1475 if (NVT == VT) { 1476 // Type is already a power of 2. The default action is to split. 1477 TransformToType[i] = MVT::Other; 1478 if (PreferredAction == TypeScalarizeVector) 1479 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); 1480 else if (PreferredAction == TypeSplitVector) 1481 ValueTypeActions.setTypeAction(VT, TypeSplitVector); 1482 else if (EC.getKnownMinValue() > 1) 1483 ValueTypeActions.setTypeAction(VT, TypeSplitVector); 1484 else 1485 ValueTypeActions.setTypeAction(VT, EC.isScalable() 1486 ? TypeScalarizeScalableVector 1487 : TypeScalarizeVector); 1488 } else { 1489 TransformToType[i] = NVT; 1490 ValueTypeActions.setTypeAction(VT, TypeWidenVector); 1491 } 1492 break; 1493 } 1494 default: 1495 llvm_unreachable("Unknown vector legalization action!"); 1496 } 1497 } 1498 1499 // Determine the 'representative' register class for each value type. 1500 // An representative register class is the largest (meaning one which is 1501 // not a sub-register class / subreg register class) legal register class for 1502 // a group of value types. For example, on i386, i8, i16, and i32 1503 // representative would be GR32; while on x86_64 it's GR64. 1504 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) { 1505 const TargetRegisterClass* RRC; 1506 uint8_t Cost; 1507 std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i); 1508 RepRegClassForVT[i] = RRC; 1509 RepRegClassCostForVT[i] = Cost; 1510 } 1511 } 1512 1513 EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1514 EVT VT) const { 1515 assert(!VT.isVector() && "No default SetCC type for vectors!"); 1516 return getPointerTy(DL).SimpleTy; 1517 } 1518 1519 MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const { 1520 return MVT::i32; // return the default value 1521 } 1522 1523 /// getVectorTypeBreakdown - Vector types are broken down into some number of 1524 /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 1525 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. 1526 /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. 1527 /// 1528 /// This method returns the number of registers needed, and the VT for each 1529 /// register. It also returns the VT and quantity of the intermediate values 1530 /// before they are promoted/expanded. 1531 unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, 1532 EVT VT, EVT &IntermediateVT, 1533 unsigned &NumIntermediates, 1534 MVT &RegisterVT) const { 1535 ElementCount EltCnt = VT.getVectorElementCount(); 1536 1537 // If there is a wider vector type with the same element type as this one, 1538 // or a promoted vector type that has the same number of elements which 1539 // are wider, then we should convert to that legal vector type. 1540 // This handles things like <2 x float> -> <4 x float> and 1541 // <4 x i1> -> <4 x i32>. 1542 LegalizeTypeAction TA = getTypeAction(Context, VT); 1543 if (!EltCnt.isScalar() && 1544 (TA == TypeWidenVector || TA == TypePromoteInteger)) { 1545 EVT RegisterEVT = getTypeToTransformTo(Context, VT); 1546 if (isTypeLegal(RegisterEVT)) { 1547 IntermediateVT = RegisterEVT; 1548 RegisterVT = RegisterEVT.getSimpleVT(); 1549 NumIntermediates = 1; 1550 return 1; 1551 } 1552 } 1553 1554 // Figure out the right, legal destination reg to copy into. 1555 EVT EltTy = VT.getVectorElementType(); 1556 1557 unsigned NumVectorRegs = 1; 1558 1559 // Scalable vectors cannot be scalarized, so handle the legalisation of the 1560 // types like done elsewhere in SelectionDAG. 1561 if (EltCnt.isScalable()) { 1562 LegalizeKind LK; 1563 EVT PartVT = VT; 1564 do { 1565 // Iterate until we've found a legal (part) type to hold VT. 1566 LK = getTypeConversion(Context, PartVT); 1567 PartVT = LK.second; 1568 } while (LK.first != TypeLegal); 1569 1570 if (!PartVT.isVector()) { 1571 report_fatal_error( 1572 "Don't know how to legalize this scalable vector type"); 1573 } 1574 1575 NumIntermediates = 1576 divideCeil(VT.getVectorElementCount().getKnownMinValue(), 1577 PartVT.getVectorElementCount().getKnownMinValue()); 1578 IntermediateVT = PartVT; 1579 RegisterVT = getRegisterType(Context, IntermediateVT); 1580 return NumIntermediates; 1581 } 1582 1583 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally 1584 // we could break down into LHS/RHS like LegalizeDAG does. 1585 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) { 1586 NumVectorRegs = EltCnt.getKnownMinValue(); 1587 EltCnt = ElementCount::getFixed(1); 1588 } 1589 1590 // Divide the input until we get to a supported size. This will always 1591 // end with a scalar if the target doesn't support vectors. 1592 while (EltCnt.getKnownMinValue() > 1 && 1593 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) { 1594 EltCnt = EltCnt.divideCoefficientBy(2); 1595 NumVectorRegs <<= 1; 1596 } 1597 1598 NumIntermediates = NumVectorRegs; 1599 1600 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt); 1601 if (!isTypeLegal(NewVT)) 1602 NewVT = EltTy; 1603 IntermediateVT = NewVT; 1604 1605 MVT DestVT = getRegisterType(Context, NewVT); 1606 RegisterVT = DestVT; 1607 1608 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16. 1609 TypeSize NewVTSize = NewVT.getSizeInBits(); 1610 // Convert sizes such as i33 to i64. 1611 if (!isPowerOf2_32(NewVTSize.getKnownMinSize())) 1612 NewVTSize = NewVTSize.coefficientNextPowerOf2(); 1613 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); 1614 } 1615 1616 // Otherwise, promotion or legal types use the same number of registers as 1617 // the vector decimated to the appropriate level. 1618 return NumVectorRegs; 1619 } 1620 1621 bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst *SI, 1622 uint64_t NumCases, 1623 uint64_t Range, 1624 ProfileSummaryInfo *PSI, 1625 BlockFrequencyInfo *BFI) const { 1626 // FIXME: This function check the maximum table size and density, but the 1627 // minimum size is not checked. It would be nice if the minimum size is 1628 // also combined within this function. Currently, the minimum size check is 1629 // performed in findJumpTable() in SelectionDAGBuiler and 1630 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl. 1631 const bool OptForSize = 1632 SI->getParent()->getParent()->hasOptSize() || 1633 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI); 1634 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize); 1635 const unsigned MaxJumpTableSize = getMaximumJumpTableSize(); 1636 1637 // Check whether the number of cases is small enough and 1638 // the range is dense enough for a jump table. 1639 return (OptForSize || Range <= MaxJumpTableSize) && 1640 (NumCases * 100 >= Range * MinDensity); 1641 } 1642 1643 /// Get the EVTs and ArgFlags collections that represent the legalized return 1644 /// type of the given function. This does not require a DAG or a return value, 1645 /// and is suitable for use before any DAGs for the function are constructed. 1646 /// TODO: Move this out of TargetLowering.cpp. 1647 void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType, 1648 AttributeList attr, 1649 SmallVectorImpl<ISD::OutputArg> &Outs, 1650 const TargetLowering &TLI, const DataLayout &DL) { 1651 SmallVector<EVT, 4> ValueVTs; 1652 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs); 1653 unsigned NumValues = ValueVTs.size(); 1654 if (NumValues == 0) return; 1655 1656 for (unsigned j = 0, f = NumValues; j != f; ++j) { 1657 EVT VT = ValueVTs[j]; 1658 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 1659 1660 if (attr.hasRetAttr(Attribute::SExt)) 1661 ExtendKind = ISD::SIGN_EXTEND; 1662 else if (attr.hasRetAttr(Attribute::ZExt)) 1663 ExtendKind = ISD::ZERO_EXTEND; 1664 1665 // FIXME: C calling convention requires the return type to be promoted to 1666 // at least 32-bit. But this is not necessary for non-C calling 1667 // conventions. The frontend should mark functions whose return values 1668 // require promoting with signext or zeroext attributes. 1669 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { 1670 MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); 1671 if (VT.bitsLT(MinVT)) 1672 VT = MinVT; 1673 } 1674 1675 unsigned NumParts = 1676 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT); 1677 MVT PartVT = 1678 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT); 1679 1680 // 'inreg' on function refers to return value 1681 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1682 if (attr.hasRetAttr(Attribute::InReg)) 1683 Flags.setInReg(); 1684 1685 // Propagate extension type if any 1686 if (attr.hasRetAttr(Attribute::SExt)) 1687 Flags.setSExt(); 1688 else if (attr.hasRetAttr(Attribute::ZExt)) 1689 Flags.setZExt(); 1690 1691 for (unsigned i = 0; i < NumParts; ++i) 1692 Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0)); 1693 } 1694 } 1695 1696 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1697 /// function arguments in the caller parameter area. This is the actual 1698 /// alignment, not its logarithm. 1699 unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty, 1700 const DataLayout &DL) const { 1701 return DL.getABITypeAlign(Ty).value(); 1702 } 1703 1704 bool TargetLoweringBase::allowsMemoryAccessForAlignment( 1705 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, 1706 Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const { 1707 // Check if the specified alignment is sufficient based on the data layout. 1708 // TODO: While using the data layout works in practice, a better solution 1709 // would be to implement this check directly (make this a virtual function). 1710 // For example, the ABI alignment may change based on software platform while 1711 // this function should only be affected by hardware implementation. 1712 Type *Ty = VT.getTypeForEVT(Context); 1713 if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) { 1714 // Assume that an access that meets the ABI-specified alignment is fast. 1715 if (Fast != nullptr) 1716 *Fast = true; 1717 return true; 1718 } 1719 1720 // This is a misaligned access. 1721 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast); 1722 } 1723 1724 bool TargetLoweringBase::allowsMemoryAccessForAlignment( 1725 LLVMContext &Context, const DataLayout &DL, EVT VT, 1726 const MachineMemOperand &MMO, bool *Fast) const { 1727 return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(), 1728 MMO.getAlign(), MMO.getFlags(), Fast); 1729 } 1730 1731 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, 1732 const DataLayout &DL, EVT VT, 1733 unsigned AddrSpace, Align Alignment, 1734 MachineMemOperand::Flags Flags, 1735 bool *Fast) const { 1736 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment, 1737 Flags, Fast); 1738 } 1739 1740 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, 1741 const DataLayout &DL, EVT VT, 1742 const MachineMemOperand &MMO, 1743 bool *Fast) const { 1744 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(), 1745 MMO.getFlags(), Fast); 1746 } 1747 1748 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, 1749 const DataLayout &DL, LLT Ty, 1750 const MachineMemOperand &MMO, 1751 bool *Fast) const { 1752 EVT VT = getApproximateEVTForLLT(Ty, DL, Context); 1753 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(), 1754 MMO.getFlags(), Fast); 1755 } 1756 1757 //===----------------------------------------------------------------------===// 1758 // TargetTransformInfo Helpers 1759 //===----------------------------------------------------------------------===// 1760 1761 int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { 1762 enum InstructionOpcodes { 1763 #define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM, 1764 #define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM 1765 #include "llvm/IR/Instruction.def" 1766 }; 1767 switch (static_cast<InstructionOpcodes>(Opcode)) { 1768 case Ret: return 0; 1769 case Br: return 0; 1770 case Switch: return 0; 1771 case IndirectBr: return 0; 1772 case Invoke: return 0; 1773 case CallBr: return 0; 1774 case Resume: return 0; 1775 case Unreachable: return 0; 1776 case CleanupRet: return 0; 1777 case CatchRet: return 0; 1778 case CatchPad: return 0; 1779 case CatchSwitch: return 0; 1780 case CleanupPad: return 0; 1781 case FNeg: return ISD::FNEG; 1782 case Add: return ISD::ADD; 1783 case FAdd: return ISD::FADD; 1784 case Sub: return ISD::SUB; 1785 case FSub: return ISD::FSUB; 1786 case Mul: return ISD::MUL; 1787 case FMul: return ISD::FMUL; 1788 case UDiv: return ISD::UDIV; 1789 case SDiv: return ISD::SDIV; 1790 case FDiv: return ISD::FDIV; 1791 case URem: return ISD::UREM; 1792 case SRem: return ISD::SREM; 1793 case FRem: return ISD::FREM; 1794 case Shl: return ISD::SHL; 1795 case LShr: return ISD::SRL; 1796 case AShr: return ISD::SRA; 1797 case And: return ISD::AND; 1798 case Or: return ISD::OR; 1799 case Xor: return ISD::XOR; 1800 case Alloca: return 0; 1801 case Load: return ISD::LOAD; 1802 case Store: return ISD::STORE; 1803 case GetElementPtr: return 0; 1804 case Fence: return 0; 1805 case AtomicCmpXchg: return 0; 1806 case AtomicRMW: return 0; 1807 case Trunc: return ISD::TRUNCATE; 1808 case ZExt: return ISD::ZERO_EXTEND; 1809 case SExt: return ISD::SIGN_EXTEND; 1810 case FPToUI: return ISD::FP_TO_UINT; 1811 case FPToSI: return ISD::FP_TO_SINT; 1812 case UIToFP: return ISD::UINT_TO_FP; 1813 case SIToFP: return ISD::SINT_TO_FP; 1814 case FPTrunc: return ISD::FP_ROUND; 1815 case FPExt: return ISD::FP_EXTEND; 1816 case PtrToInt: return ISD::BITCAST; 1817 case IntToPtr: return ISD::BITCAST; 1818 case BitCast: return ISD::BITCAST; 1819 case AddrSpaceCast: return ISD::ADDRSPACECAST; 1820 case ICmp: return ISD::SETCC; 1821 case FCmp: return ISD::SETCC; 1822 case PHI: return 0; 1823 case Call: return 0; 1824 case Select: return ISD::SELECT; 1825 case UserOp1: return 0; 1826 case UserOp2: return 0; 1827 case VAArg: return 0; 1828 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT; 1829 case InsertElement: return ISD::INSERT_VECTOR_ELT; 1830 case ShuffleVector: return ISD::VECTOR_SHUFFLE; 1831 case ExtractValue: return ISD::MERGE_VALUES; 1832 case InsertValue: return ISD::MERGE_VALUES; 1833 case LandingPad: return 0; 1834 case Freeze: return ISD::FREEZE; 1835 } 1836 1837 llvm_unreachable("Unknown instruction type encountered!"); 1838 } 1839 1840 std::pair<InstructionCost, MVT> 1841 TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL, 1842 Type *Ty) const { 1843 LLVMContext &C = Ty->getContext(); 1844 EVT MTy = getValueType(DL, Ty); 1845 1846 InstructionCost Cost = 1; 1847 // We keep legalizing the type until we find a legal kind. We assume that 1848 // the only operation that costs anything is the split. After splitting 1849 // we need to handle two types. 1850 while (true) { 1851 LegalizeKind LK = getTypeConversion(C, MTy); 1852 1853 if (LK.first == TypeScalarizeScalableVector) 1854 return std::make_pair(InstructionCost::getInvalid(), MVT::getVT(Ty)); 1855 1856 if (LK.first == TypeLegal) 1857 return std::make_pair(Cost, MTy.getSimpleVT()); 1858 1859 if (LK.first == TypeSplitVector || LK.first == TypeExpandInteger) 1860 Cost *= 2; 1861 1862 // Do not loop with f128 type. 1863 if (MTy == LK.second) 1864 return std::make_pair(Cost, MTy.getSimpleVT()); 1865 1866 // Keep legalizing the type. 1867 MTy = LK.second; 1868 } 1869 } 1870 1871 Value * 1872 TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, 1873 bool UseTLS) const { 1874 // compiler-rt provides a variable with a magic name. Targets that do not 1875 // link with compiler-rt may also provide such a variable. 1876 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 1877 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr"; 1878 auto UnsafeStackPtr = 1879 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar)); 1880 1881 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); 1882 1883 if (!UnsafeStackPtr) { 1884 auto TLSModel = UseTLS ? 1885 GlobalValue::InitialExecTLSModel : 1886 GlobalValue::NotThreadLocal; 1887 // The global variable is not defined yet, define it ourselves. 1888 // We use the initial-exec TLS model because we do not support the 1889 // variable living anywhere other than in the main executable. 1890 UnsafeStackPtr = new GlobalVariable( 1891 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, 1892 UnsafeStackPtrVar, nullptr, TLSModel); 1893 } else { 1894 // The variable exists, check its type and attributes. 1895 if (UnsafeStackPtr->getValueType() != StackPtrTy) 1896 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type"); 1897 if (UseTLS != UnsafeStackPtr->isThreadLocal()) 1898 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " + 1899 (UseTLS ? "" : "not ") + "be thread-local"); 1900 } 1901 return UnsafeStackPtr; 1902 } 1903 1904 Value * 1905 TargetLoweringBase::getSafeStackPointerLocation(IRBuilderBase &IRB) const { 1906 if (!TM.getTargetTriple().isAndroid()) 1907 return getDefaultSafeStackPointerLocation(IRB, true); 1908 1909 // Android provides a libc function to retrieve the address of the current 1910 // thread's unsafe stack pointer. 1911 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 1912 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); 1913 FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address", 1914 StackPtrTy->getPointerTo(0)); 1915 return IRB.CreateCall(Fn); 1916 } 1917 1918 //===----------------------------------------------------------------------===// 1919 // Loop Strength Reduction hooks 1920 //===----------------------------------------------------------------------===// 1921 1922 /// isLegalAddressingMode - Return true if the addressing mode represented 1923 /// by AM is legal for this target, for a load/store of the specified type. 1924 bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL, 1925 const AddrMode &AM, Type *Ty, 1926 unsigned AS, Instruction *I) const { 1927 // The default implementation of this implements a conservative RISCy, r+r and 1928 // r+i addr mode. 1929 1930 // Allows a sign-extended 16-bit immediate field. 1931 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 1932 return false; 1933 1934 // No global is ever allowed as a base. 1935 if (AM.BaseGV) 1936 return false; 1937 1938 // Only support r+r, 1939 switch (AM.Scale) { 1940 case 0: // "r+i" or just "i", depending on HasBaseReg. 1941 break; 1942 case 1: 1943 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 1944 return false; 1945 // Otherwise we have r+r or r+i. 1946 break; 1947 case 2: 1948 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 1949 return false; 1950 // Allow 2*r as r+r. 1951 break; 1952 default: // Don't allow n * r 1953 return false; 1954 } 1955 1956 return true; 1957 } 1958 1959 //===----------------------------------------------------------------------===// 1960 // Stack Protector 1961 //===----------------------------------------------------------------------===// 1962 1963 // For OpenBSD return its special guard variable. Otherwise return nullptr, 1964 // so that SelectionDAG handle SSP. 1965 Value *TargetLoweringBase::getIRStackGuard(IRBuilderBase &IRB) const { 1966 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) { 1967 Module &M = *IRB.GetInsertBlock()->getParent()->getParent(); 1968 PointerType *PtrTy = Type::getInt8PtrTy(M.getContext()); 1969 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy); 1970 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C)) 1971 G->setVisibility(GlobalValue::HiddenVisibility); 1972 return C; 1973 } 1974 return nullptr; 1975 } 1976 1977 // Currently only support "standard" __stack_chk_guard. 1978 // TODO: add LOAD_STACK_GUARD support. 1979 void TargetLoweringBase::insertSSPDeclarations(Module &M) const { 1980 if (!M.getNamedValue("__stack_chk_guard")) { 1981 auto *GV = new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false, 1982 GlobalVariable::ExternalLinkage, nullptr, 1983 "__stack_chk_guard"); 1984 if (TM.getRelocationModel() == Reloc::Static && 1985 !TM.getTargetTriple().isWindowsGNUEnvironment()) 1986 GV->setDSOLocal(true); 1987 } 1988 } 1989 1990 // Currently only support "standard" __stack_chk_guard. 1991 // TODO: add LOAD_STACK_GUARD support. 1992 Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const { 1993 return M.getNamedValue("__stack_chk_guard"); 1994 } 1995 1996 Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const { 1997 return nullptr; 1998 } 1999 2000 unsigned TargetLoweringBase::getMinimumJumpTableEntries() const { 2001 return MinimumJumpTableEntries; 2002 } 2003 2004 void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) { 2005 MinimumJumpTableEntries = Val; 2006 } 2007 2008 unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const { 2009 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity; 2010 } 2011 2012 unsigned TargetLoweringBase::getMaximumJumpTableSize() const { 2013 return MaximumJumpTableSize; 2014 } 2015 2016 void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) { 2017 MaximumJumpTableSize = Val; 2018 } 2019 2020 bool TargetLoweringBase::isJumpTableRelative() const { 2021 return getTargetMachine().isPositionIndependent(); 2022 } 2023 2024 Align TargetLoweringBase::getPrefLoopAlignment(MachineLoop *ML) const { 2025 if (TM.Options.LoopAlignment) 2026 return Align(TM.Options.LoopAlignment); 2027 return PrefLoopAlignment; 2028 } 2029 2030 //===----------------------------------------------------------------------===// 2031 // Reciprocal Estimates 2032 //===----------------------------------------------------------------------===// 2033 2034 /// Get the reciprocal estimate attribute string for a function that will 2035 /// override the target defaults. 2036 static StringRef getRecipEstimateForFunc(MachineFunction &MF) { 2037 const Function &F = MF.getFunction(); 2038 return F.getFnAttribute("reciprocal-estimates").getValueAsString(); 2039 } 2040 2041 /// Construct a string for the given reciprocal operation of the given type. 2042 /// This string should match the corresponding option to the front-end's 2043 /// "-mrecip" flag assuming those strings have been passed through in an 2044 /// attribute string. For example, "vec-divf" for a division of a vXf32. 2045 static std::string getReciprocalOpName(bool IsSqrt, EVT VT) { 2046 std::string Name = VT.isVector() ? "vec-" : ""; 2047 2048 Name += IsSqrt ? "sqrt" : "div"; 2049 2050 // TODO: Handle "half" or other float types? 2051 if (VT.getScalarType() == MVT::f64) { 2052 Name += "d"; 2053 } else { 2054 assert(VT.getScalarType() == MVT::f32 && 2055 "Unexpected FP type for reciprocal estimate"); 2056 Name += "f"; 2057 } 2058 2059 return Name; 2060 } 2061 2062 /// Return the character position and value (a single numeric character) of a 2063 /// customized refinement operation in the input string if it exists. Return 2064 /// false if there is no customized refinement step count. 2065 static bool parseRefinementStep(StringRef In, size_t &Position, 2066 uint8_t &Value) { 2067 const char RefStepToken = ':'; 2068 Position = In.find(RefStepToken); 2069 if (Position == StringRef::npos) 2070 return false; 2071 2072 StringRef RefStepString = In.substr(Position + 1); 2073 // Allow exactly one numeric character for the additional refinement 2074 // step parameter. 2075 if (RefStepString.size() == 1) { 2076 char RefStepChar = RefStepString[0]; 2077 if (isDigit(RefStepChar)) { 2078 Value = RefStepChar - '0'; 2079 return true; 2080 } 2081 } 2082 report_fatal_error("Invalid refinement step for -recip."); 2083 } 2084 2085 /// For the input attribute string, return one of the ReciprocalEstimate enum 2086 /// status values (enabled, disabled, or not specified) for this operation on 2087 /// the specified data type. 2088 static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) { 2089 if (Override.empty()) 2090 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2091 2092 SmallVector<StringRef, 4> OverrideVector; 2093 Override.split(OverrideVector, ','); 2094 unsigned NumArgs = OverrideVector.size(); 2095 2096 // Check if "all", "none", or "default" was specified. 2097 if (NumArgs == 1) { 2098 // Look for an optional setting of the number of refinement steps needed 2099 // for this type of reciprocal operation. 2100 size_t RefPos; 2101 uint8_t RefSteps; 2102 if (parseRefinementStep(Override, RefPos, RefSteps)) { 2103 // Split the string for further processing. 2104 Override = Override.substr(0, RefPos); 2105 } 2106 2107 // All reciprocal types are enabled. 2108 if (Override == "all") 2109 return TargetLoweringBase::ReciprocalEstimate::Enabled; 2110 2111 // All reciprocal types are disabled. 2112 if (Override == "none") 2113 return TargetLoweringBase::ReciprocalEstimate::Disabled; 2114 2115 // Target defaults for enablement are used. 2116 if (Override == "default") 2117 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2118 } 2119 2120 // The attribute string may omit the size suffix ('f'/'d'). 2121 std::string VTName = getReciprocalOpName(IsSqrt, VT); 2122 std::string VTNameNoSize = VTName; 2123 VTNameNoSize.pop_back(); 2124 static const char DisabledPrefix = '!'; 2125 2126 for (StringRef RecipType : OverrideVector) { 2127 size_t RefPos; 2128 uint8_t RefSteps; 2129 if (parseRefinementStep(RecipType, RefPos, RefSteps)) 2130 RecipType = RecipType.substr(0, RefPos); 2131 2132 // Ignore the disablement token for string matching. 2133 bool IsDisabled = RecipType[0] == DisabledPrefix; 2134 if (IsDisabled) 2135 RecipType = RecipType.substr(1); 2136 2137 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)) 2138 return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled 2139 : TargetLoweringBase::ReciprocalEstimate::Enabled; 2140 } 2141 2142 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2143 } 2144 2145 /// For the input attribute string, return the customized refinement step count 2146 /// for this operation on the specified data type. If the step count does not 2147 /// exist, return the ReciprocalEstimate enum value for unspecified. 2148 static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) { 2149 if (Override.empty()) 2150 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2151 2152 SmallVector<StringRef, 4> OverrideVector; 2153 Override.split(OverrideVector, ','); 2154 unsigned NumArgs = OverrideVector.size(); 2155 2156 // Check if "all", "default", or "none" was specified. 2157 if (NumArgs == 1) { 2158 // Look for an optional setting of the number of refinement steps needed 2159 // for this type of reciprocal operation. 2160 size_t RefPos; 2161 uint8_t RefSteps; 2162 if (!parseRefinementStep(Override, RefPos, RefSteps)) 2163 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2164 2165 // Split the string for further processing. 2166 Override = Override.substr(0, RefPos); 2167 assert(Override != "none" && 2168 "Disabled reciprocals, but specifed refinement steps?"); 2169 2170 // If this is a general override, return the specified number of steps. 2171 if (Override == "all" || Override == "default") 2172 return RefSteps; 2173 } 2174 2175 // The attribute string may omit the size suffix ('f'/'d'). 2176 std::string VTName = getReciprocalOpName(IsSqrt, VT); 2177 std::string VTNameNoSize = VTName; 2178 VTNameNoSize.pop_back(); 2179 2180 for (StringRef RecipType : OverrideVector) { 2181 size_t RefPos; 2182 uint8_t RefSteps; 2183 if (!parseRefinementStep(RecipType, RefPos, RefSteps)) 2184 continue; 2185 2186 RecipType = RecipType.substr(0, RefPos); 2187 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)) 2188 return RefSteps; 2189 } 2190 2191 return TargetLoweringBase::ReciprocalEstimate::Unspecified; 2192 } 2193 2194 int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT, 2195 MachineFunction &MF) const { 2196 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF)); 2197 } 2198 2199 int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT, 2200 MachineFunction &MF) const { 2201 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF)); 2202 } 2203 2204 int TargetLoweringBase::getSqrtRefinementSteps(EVT VT, 2205 MachineFunction &MF) const { 2206 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF)); 2207 } 2208 2209 int TargetLoweringBase::getDivRefinementSteps(EVT VT, 2210 MachineFunction &MF) const { 2211 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF)); 2212 } 2213 2214 void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const { 2215 MF.getRegInfo().freezeReservedRegs(MF); 2216 } 2217 2218 MachineMemOperand::Flags 2219 TargetLoweringBase::getLoadMemOperandFlags(const LoadInst &LI, 2220 const DataLayout &DL) const { 2221 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad; 2222 if (LI.isVolatile()) 2223 Flags |= MachineMemOperand::MOVolatile; 2224 2225 if (LI.hasMetadata(LLVMContext::MD_nontemporal)) 2226 Flags |= MachineMemOperand::MONonTemporal; 2227 2228 if (LI.hasMetadata(LLVMContext::MD_invariant_load)) 2229 Flags |= MachineMemOperand::MOInvariant; 2230 2231 if (isDereferenceablePointer(LI.getPointerOperand(), LI.getType(), DL)) 2232 Flags |= MachineMemOperand::MODereferenceable; 2233 2234 Flags |= getTargetMMOFlags(LI); 2235 return Flags; 2236 } 2237 2238 MachineMemOperand::Flags 2239 TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI, 2240 const DataLayout &DL) const { 2241 MachineMemOperand::Flags Flags = MachineMemOperand::MOStore; 2242 2243 if (SI.isVolatile()) 2244 Flags |= MachineMemOperand::MOVolatile; 2245 2246 if (SI.hasMetadata(LLVMContext::MD_nontemporal)) 2247 Flags |= MachineMemOperand::MONonTemporal; 2248 2249 // FIXME: Not preserving dereferenceable 2250 Flags |= getTargetMMOFlags(SI); 2251 return Flags; 2252 } 2253 2254 MachineMemOperand::Flags 2255 TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI, 2256 const DataLayout &DL) const { 2257 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 2258 2259 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) { 2260 if (RMW->isVolatile()) 2261 Flags |= MachineMemOperand::MOVolatile; 2262 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) { 2263 if (CmpX->isVolatile()) 2264 Flags |= MachineMemOperand::MOVolatile; 2265 } else 2266 llvm_unreachable("not an atomic instruction"); 2267 2268 // FIXME: Not preserving dereferenceable 2269 Flags |= getTargetMMOFlags(AI); 2270 return Flags; 2271 } 2272 2273 Instruction *TargetLoweringBase::emitLeadingFence(IRBuilderBase &Builder, 2274 Instruction *Inst, 2275 AtomicOrdering Ord) const { 2276 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore()) 2277 return Builder.CreateFence(Ord); 2278 else 2279 return nullptr; 2280 } 2281 2282 Instruction *TargetLoweringBase::emitTrailingFence(IRBuilderBase &Builder, 2283 Instruction *Inst, 2284 AtomicOrdering Ord) const { 2285 if (isAcquireOrStronger(Ord)) 2286 return Builder.CreateFence(Ord); 2287 else 2288 return nullptr; 2289 } 2290 2291 //===----------------------------------------------------------------------===// 2292 // GlobalISel Hooks 2293 //===----------------------------------------------------------------------===// 2294 2295 bool TargetLoweringBase::shouldLocalize(const MachineInstr &MI, 2296 const TargetTransformInfo *TTI) const { 2297 auto &MF = *MI.getMF(); 2298 auto &MRI = MF.getRegInfo(); 2299 // Assuming a spill and reload of a value has a cost of 1 instruction each, 2300 // this helper function computes the maximum number of uses we should consider 2301 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We 2302 // break even in terms of code size when the original MI has 2 users vs 2303 // choosing to potentially spill. Any more than 2 users we we have a net code 2304 // size increase. This doesn't take into account register pressure though. 2305 auto maxUses = [](unsigned RematCost) { 2306 // A cost of 1 means remats are basically free. 2307 if (RematCost == 1) 2308 return UINT_MAX; 2309 if (RematCost == 2) 2310 return 2U; 2311 2312 // Remat is too expensive, only sink if there's one user. 2313 if (RematCost > 2) 2314 return 1U; 2315 llvm_unreachable("Unexpected remat cost"); 2316 }; 2317 2318 // Helper to walk through uses and terminate if we've reached a limit. Saves 2319 // us spending time traversing uses if all we want to know is if it's >= min. 2320 auto isUsesAtMost = [&](unsigned Reg, unsigned MaxUses) { 2321 unsigned NumUses = 0; 2322 auto UI = MRI.use_instr_nodbg_begin(Reg), UE = MRI.use_instr_nodbg_end(); 2323 for (; UI != UE && NumUses < MaxUses; ++UI) { 2324 NumUses++; 2325 } 2326 // If we haven't reached the end yet then there are more than MaxUses users. 2327 return UI == UE; 2328 }; 2329 2330 switch (MI.getOpcode()) { 2331 default: 2332 return false; 2333 // Constants-like instructions should be close to their users. 2334 // We don't want long live-ranges for them. 2335 case TargetOpcode::G_CONSTANT: 2336 case TargetOpcode::G_FCONSTANT: 2337 case TargetOpcode::G_FRAME_INDEX: 2338 case TargetOpcode::G_INTTOPTR: 2339 return true; 2340 case TargetOpcode::G_GLOBAL_VALUE: { 2341 unsigned RematCost = TTI->getGISelRematGlobalCost(); 2342 Register Reg = MI.getOperand(0).getReg(); 2343 unsigned MaxUses = maxUses(RematCost); 2344 if (MaxUses == UINT_MAX) 2345 return true; // Remats are "free" so always localize. 2346 bool B = isUsesAtMost(Reg, MaxUses); 2347 return B; 2348 } 2349 } 2350 } 2351