1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This implements the SelectionDAG class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/CodeGen/SelectionDAG.h" 14 #include "SDNodeDbgValue.h" 15 #include "llvm/ADT/APFloat.h" 16 #include "llvm/ADT/APInt.h" 17 #include "llvm/ADT/APSInt.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/BitVector.h" 20 #include "llvm/ADT/FoldingSet.h" 21 #include "llvm/ADT/None.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Triple.h" 26 #include "llvm/ADT/Twine.h" 27 #include "llvm/Analysis/BlockFrequencyInfo.h" 28 #include "llvm/Analysis/MemoryLocation.h" 29 #include "llvm/Analysis/ProfileSummaryInfo.h" 30 #include "llvm/Analysis/ValueTracking.h" 31 #include "llvm/CodeGen/FunctionLoweringInfo.h" 32 #include "llvm/CodeGen/ISDOpcodes.h" 33 #include "llvm/CodeGen/MachineBasicBlock.h" 34 #include "llvm/CodeGen/MachineConstantPool.h" 35 #include "llvm/CodeGen/MachineFrameInfo.h" 36 #include "llvm/CodeGen/MachineFunction.h" 37 #include "llvm/CodeGen/MachineMemOperand.h" 38 #include "llvm/CodeGen/RuntimeLibcalls.h" 39 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" 40 #include "llvm/CodeGen/SelectionDAGNodes.h" 41 #include "llvm/CodeGen/SelectionDAGTargetInfo.h" 42 #include "llvm/CodeGen/TargetFrameLowering.h" 43 #include "llvm/CodeGen/TargetLowering.h" 44 #include "llvm/CodeGen/TargetRegisterInfo.h" 45 #include "llvm/CodeGen/TargetSubtargetInfo.h" 46 #include "llvm/CodeGen/ValueTypes.h" 47 #include "llvm/IR/Constant.h" 48 #include "llvm/IR/Constants.h" 49 #include "llvm/IR/DataLayout.h" 50 #include "llvm/IR/DebugInfoMetadata.h" 51 #include "llvm/IR/DebugLoc.h" 52 #include "llvm/IR/DerivedTypes.h" 53 #include "llvm/IR/Function.h" 54 #include "llvm/IR/GlobalValue.h" 55 #include "llvm/IR/Metadata.h" 56 #include "llvm/IR/Type.h" 57 #include "llvm/IR/Value.h" 58 #include "llvm/Support/Casting.h" 59 #include "llvm/Support/CodeGen.h" 60 #include "llvm/Support/Compiler.h" 61 #include "llvm/Support/Debug.h" 62 #include "llvm/Support/ErrorHandling.h" 63 #include "llvm/Support/KnownBits.h" 64 #include "llvm/Support/MachineValueType.h" 65 #include "llvm/Support/ManagedStatic.h" 66 #include "llvm/Support/MathExtras.h" 67 #include "llvm/Support/Mutex.h" 68 #include "llvm/Support/raw_ostream.h" 69 #include "llvm/Target/TargetMachine.h" 70 #include "llvm/Target/TargetOptions.h" 71 #include "llvm/Transforms/Utils/SizeOpts.h" 72 #include <algorithm> 73 #include <cassert> 74 #include <cstdint> 75 #include <cstdlib> 76 #include <limits> 77 #include <set> 78 #include <string> 79 #include <utility> 80 #include <vector> 81 82 using namespace llvm; 83 84 /// makeVTList - Return an instance of the SDVTList struct initialized with the 85 /// specified members. 86 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 87 SDVTList Res = {VTs, NumVTs}; 88 return Res; 89 } 90 91 // Default null implementations of the callbacks. 92 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 93 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 94 void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} 95 96 void SelectionDAG::DAGNodeDeletedListener::anchor() {} 97 98 #define DEBUG_TYPE "selectiondag" 99 100 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt", 101 cl::Hidden, cl::init(true), 102 cl::desc("Gang up loads and stores generated by inlining of memcpy")); 103 104 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max", 105 cl::desc("Number limit for gluing ld/st of memcpy."), 106 cl::Hidden, cl::init(0)); 107 108 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { 109 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); 110 } 111 112 //===----------------------------------------------------------------------===// 113 // ConstantFPSDNode Class 114 //===----------------------------------------------------------------------===// 115 116 /// isExactlyValue - We don't rely on operator== working on double values, as 117 /// it returns true for things that are clearly not equal, like -0.0 and 0.0. 118 /// As such, this method can be used to do an exact bit-for-bit comparison of 119 /// two floating point values. 120 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 121 return getValueAPF().bitwiseIsEqual(V); 122 } 123 124 bool ConstantFPSDNode::isValueValidForType(EVT VT, 125 const APFloat& Val) { 126 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 127 128 // convert modifies in place, so make a copy. 129 APFloat Val2 = APFloat(Val); 130 bool losesInfo; 131 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 132 APFloat::rmNearestTiesToEven, 133 &losesInfo); 134 return !losesInfo; 135 } 136 137 //===----------------------------------------------------------------------===// 138 // ISD Namespace 139 //===----------------------------------------------------------------------===// 140 141 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { 142 auto *BV = dyn_cast<BuildVectorSDNode>(N); 143 if (!BV) 144 return false; 145 146 APInt SplatUndef; 147 unsigned SplatBitSize; 148 bool HasUndefs; 149 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 150 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, 151 EltSize) && 152 EltSize == SplatBitSize; 153 } 154 155 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be 156 // specializations of the more general isConstantSplatVector()? 157 158 bool ISD::isBuildVectorAllOnes(const SDNode *N) { 159 // Look through a bit convert. 160 while (N->getOpcode() == ISD::BITCAST) 161 N = N->getOperand(0).getNode(); 162 163 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 164 165 unsigned i = 0, e = N->getNumOperands(); 166 167 // Skip over all of the undef values. 168 while (i != e && N->getOperand(i).isUndef()) 169 ++i; 170 171 // Do not accept an all-undef vector. 172 if (i == e) return false; 173 174 // Do not accept build_vectors that aren't all constants or which have non-~0 175 // elements. We have to be a bit careful here, as the type of the constant 176 // may not be the same as the type of the vector elements due to type 177 // legalization (the elements are promoted to a legal type for the target and 178 // a vector of a type may be legal when the base element type is not). 179 // We only want to check enough bits to cover the vector elements, because 180 // we care if the resultant vector is all ones, not whether the individual 181 // constants are. 182 SDValue NotZero = N->getOperand(i); 183 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 184 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 185 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 186 return false; 187 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 188 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 189 return false; 190 } else 191 return false; 192 193 // Okay, we have at least one ~0 value, check to see if the rest match or are 194 // undefs. Even with the above element type twiddling, this should be OK, as 195 // the same type legalization should have applied to all the elements. 196 for (++i; i != e; ++i) 197 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef()) 198 return false; 199 return true; 200 } 201 202 bool ISD::isBuildVectorAllZeros(const SDNode *N) { 203 // Look through a bit convert. 204 while (N->getOpcode() == ISD::BITCAST) 205 N = N->getOperand(0).getNode(); 206 207 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 208 209 bool IsAllUndef = true; 210 for (const SDValue &Op : N->op_values()) { 211 if (Op.isUndef()) 212 continue; 213 IsAllUndef = false; 214 // Do not accept build_vectors that aren't all constants or which have non-0 215 // elements. We have to be a bit careful here, as the type of the constant 216 // may not be the same as the type of the vector elements due to type 217 // legalization (the elements are promoted to a legal type for the target 218 // and a vector of a type may be legal when the base element type is not). 219 // We only want to check enough bits to cover the vector elements, because 220 // we care if the resultant vector is all zeros, not whether the individual 221 // constants are. 222 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 223 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) { 224 if (CN->getAPIntValue().countTrailingZeros() < EltSize) 225 return false; 226 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) { 227 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize) 228 return false; 229 } else 230 return false; 231 } 232 233 // Do not accept an all-undef vector. 234 if (IsAllUndef) 235 return false; 236 return true; 237 } 238 239 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { 240 if (N->getOpcode() != ISD::BUILD_VECTOR) 241 return false; 242 243 for (const SDValue &Op : N->op_values()) { 244 if (Op.isUndef()) 245 continue; 246 if (!isa<ConstantSDNode>(Op)) 247 return false; 248 } 249 return true; 250 } 251 252 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { 253 if (N->getOpcode() != ISD::BUILD_VECTOR) 254 return false; 255 256 for (const SDValue &Op : N->op_values()) { 257 if (Op.isUndef()) 258 continue; 259 if (!isa<ConstantFPSDNode>(Op)) 260 return false; 261 } 262 return true; 263 } 264 265 bool ISD::allOperandsUndef(const SDNode *N) { 266 // Return false if the node has no operands. 267 // This is "logically inconsistent" with the definition of "all" but 268 // is probably the desired behavior. 269 if (N->getNumOperands() == 0) 270 return false; 271 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); }); 272 } 273 274 bool ISD::matchUnaryPredicate(SDValue Op, 275 std::function<bool(ConstantSDNode *)> Match, 276 bool AllowUndefs) { 277 // FIXME: Add support for scalar UNDEF cases? 278 if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) 279 return Match(Cst); 280 281 // FIXME: Add support for vector UNDEF cases? 282 if (ISD::BUILD_VECTOR != Op.getOpcode()) 283 return false; 284 285 EVT SVT = Op.getValueType().getScalarType(); 286 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 287 if (AllowUndefs && Op.getOperand(i).isUndef()) { 288 if (!Match(nullptr)) 289 return false; 290 continue; 291 } 292 293 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i)); 294 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) 295 return false; 296 } 297 return true; 298 } 299 300 bool ISD::matchBinaryPredicate( 301 SDValue LHS, SDValue RHS, 302 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, 303 bool AllowUndefs, bool AllowTypeMismatch) { 304 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) 305 return false; 306 307 // TODO: Add support for scalar UNDEF cases? 308 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS)) 309 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS)) 310 return Match(LHSCst, RHSCst); 311 312 // TODO: Add support for vector UNDEF cases? 313 if (ISD::BUILD_VECTOR != LHS.getOpcode() || 314 ISD::BUILD_VECTOR != RHS.getOpcode()) 315 return false; 316 317 EVT SVT = LHS.getValueType().getScalarType(); 318 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { 319 SDValue LHSOp = LHS.getOperand(i); 320 SDValue RHSOp = RHS.getOperand(i); 321 bool LHSUndef = AllowUndefs && LHSOp.isUndef(); 322 bool RHSUndef = AllowUndefs && RHSOp.isUndef(); 323 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp); 324 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp); 325 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) 326 return false; 327 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || 328 LHSOp.getValueType() != RHSOp.getValueType())) 329 return false; 330 if (!Match(LHSCst, RHSCst)) 331 return false; 332 } 333 return true; 334 } 335 336 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { 337 switch (VecReduceOpcode) { 338 default: 339 llvm_unreachable("Expected VECREDUCE opcode"); 340 case ISD::VECREDUCE_FADD: 341 case ISD::VECREDUCE_SEQ_FADD: 342 return ISD::FADD; 343 case ISD::VECREDUCE_FMUL: 344 case ISD::VECREDUCE_SEQ_FMUL: 345 return ISD::FMUL; 346 case ISD::VECREDUCE_ADD: 347 return ISD::ADD; 348 case ISD::VECREDUCE_MUL: 349 return ISD::MUL; 350 case ISD::VECREDUCE_AND: 351 return ISD::AND; 352 case ISD::VECREDUCE_OR: 353 return ISD::OR; 354 case ISD::VECREDUCE_XOR: 355 return ISD::XOR; 356 case ISD::VECREDUCE_SMAX: 357 return ISD::SMAX; 358 case ISD::VECREDUCE_SMIN: 359 return ISD::SMIN; 360 case ISD::VECREDUCE_UMAX: 361 return ISD::UMAX; 362 case ISD::VECREDUCE_UMIN: 363 return ISD::UMIN; 364 case ISD::VECREDUCE_FMAX: 365 return ISD::FMAXNUM; 366 case ISD::VECREDUCE_FMIN: 367 return ISD::FMINNUM; 368 } 369 } 370 371 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { 372 switch (ExtType) { 373 case ISD::EXTLOAD: 374 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; 375 case ISD::SEXTLOAD: 376 return ISD::SIGN_EXTEND; 377 case ISD::ZEXTLOAD: 378 return ISD::ZERO_EXTEND; 379 default: 380 break; 381 } 382 383 llvm_unreachable("Invalid LoadExtType"); 384 } 385 386 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 387 // To perform this operation, we just need to swap the L and G bits of the 388 // operation. 389 unsigned OldL = (Operation >> 2) & 1; 390 unsigned OldG = (Operation >> 1) & 1; 391 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 392 (OldL << 1) | // New G bit 393 (OldG << 2)); // New L bit. 394 } 395 396 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { 397 unsigned Operation = Op; 398 if (isIntegerLike) 399 Operation ^= 7; // Flip L, G, E bits, but not U. 400 else 401 Operation ^= 15; // Flip all of the condition bits. 402 403 if (Operation > ISD::SETTRUE2) 404 Operation &= ~8; // Don't let N and U bits get set. 405 406 return ISD::CondCode(Operation); 407 } 408 409 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { 410 return getSetCCInverseImpl(Op, Type.isInteger()); 411 } 412 413 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, 414 bool isIntegerLike) { 415 return getSetCCInverseImpl(Op, isIntegerLike); 416 } 417 418 /// For an integer comparison, return 1 if the comparison is a signed operation 419 /// and 2 if the result is an unsigned comparison. Return zero if the operation 420 /// does not depend on the sign of the input (setne and seteq). 421 static int isSignedOp(ISD::CondCode Opcode) { 422 switch (Opcode) { 423 default: llvm_unreachable("Illegal integer setcc operation!"); 424 case ISD::SETEQ: 425 case ISD::SETNE: return 0; 426 case ISD::SETLT: 427 case ISD::SETLE: 428 case ISD::SETGT: 429 case ISD::SETGE: return 1; 430 case ISD::SETULT: 431 case ISD::SETULE: 432 case ISD::SETUGT: 433 case ISD::SETUGE: return 2; 434 } 435 } 436 437 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 438 EVT Type) { 439 bool IsInteger = Type.isInteger(); 440 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 441 // Cannot fold a signed integer setcc with an unsigned integer setcc. 442 return ISD::SETCC_INVALID; 443 444 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 445 446 // If the N and U bits get set, then the resultant comparison DOES suddenly 447 // care about orderedness, and it is true when ordered. 448 if (Op > ISD::SETTRUE2) 449 Op &= ~16; // Clear the U bit if the N bit is set. 450 451 // Canonicalize illegal integer setcc's. 452 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 453 Op = ISD::SETNE; 454 455 return ISD::CondCode(Op); 456 } 457 458 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 459 EVT Type) { 460 bool IsInteger = Type.isInteger(); 461 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 462 // Cannot fold a signed setcc with an unsigned setcc. 463 return ISD::SETCC_INVALID; 464 465 // Combine all of the condition bits. 466 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 467 468 // Canonicalize illegal integer setcc's. 469 if (IsInteger) { 470 switch (Result) { 471 default: break; 472 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 473 case ISD::SETOEQ: // SETEQ & SETU[LG]E 474 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 475 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 476 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 477 } 478 } 479 480 return Result; 481 } 482 483 //===----------------------------------------------------------------------===// 484 // SDNode Profile Support 485 //===----------------------------------------------------------------------===// 486 487 /// AddNodeIDOpcode - Add the node opcode to the NodeID data. 488 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 489 ID.AddInteger(OpC); 490 } 491 492 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 493 /// solely with their pointer. 494 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 495 ID.AddPointer(VTList.VTs); 496 } 497 498 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 499 static void AddNodeIDOperands(FoldingSetNodeID &ID, 500 ArrayRef<SDValue> Ops) { 501 for (auto& Op : Ops) { 502 ID.AddPointer(Op.getNode()); 503 ID.AddInteger(Op.getResNo()); 504 } 505 } 506 507 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 508 static void AddNodeIDOperands(FoldingSetNodeID &ID, 509 ArrayRef<SDUse> Ops) { 510 for (auto& Op : Ops) { 511 ID.AddPointer(Op.getNode()); 512 ID.AddInteger(Op.getResNo()); 513 } 514 } 515 516 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, 517 SDVTList VTList, ArrayRef<SDValue> OpList) { 518 AddNodeIDOpcode(ID, OpC); 519 AddNodeIDValueTypes(ID, VTList); 520 AddNodeIDOperands(ID, OpList); 521 } 522 523 /// If this is an SDNode with special info, add this info to the NodeID data. 524 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 525 switch (N->getOpcode()) { 526 case ISD::TargetExternalSymbol: 527 case ISD::ExternalSymbol: 528 case ISD::MCSymbol: 529 llvm_unreachable("Should only be used on nodes with operands"); 530 default: break; // Normal nodes don't need extra info. 531 case ISD::TargetConstant: 532 case ISD::Constant: { 533 const ConstantSDNode *C = cast<ConstantSDNode>(N); 534 ID.AddPointer(C->getConstantIntValue()); 535 ID.AddBoolean(C->isOpaque()); 536 break; 537 } 538 case ISD::TargetConstantFP: 539 case ISD::ConstantFP: 540 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 541 break; 542 case ISD::TargetGlobalAddress: 543 case ISD::GlobalAddress: 544 case ISD::TargetGlobalTLSAddress: 545 case ISD::GlobalTLSAddress: { 546 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 547 ID.AddPointer(GA->getGlobal()); 548 ID.AddInteger(GA->getOffset()); 549 ID.AddInteger(GA->getTargetFlags()); 550 break; 551 } 552 case ISD::BasicBlock: 553 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 554 break; 555 case ISD::Register: 556 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 557 break; 558 case ISD::RegisterMask: 559 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 560 break; 561 case ISD::SRCVALUE: 562 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 563 break; 564 case ISD::FrameIndex: 565 case ISD::TargetFrameIndex: 566 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 567 break; 568 case ISD::LIFETIME_START: 569 case ISD::LIFETIME_END: 570 if (cast<LifetimeSDNode>(N)->hasOffset()) { 571 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize()); 572 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset()); 573 } 574 break; 575 case ISD::JumpTable: 576 case ISD::TargetJumpTable: 577 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 578 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 579 break; 580 case ISD::ConstantPool: 581 case ISD::TargetConstantPool: { 582 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 583 ID.AddInteger(CP->getAlign().value()); 584 ID.AddInteger(CP->getOffset()); 585 if (CP->isMachineConstantPoolEntry()) 586 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 587 else 588 ID.AddPointer(CP->getConstVal()); 589 ID.AddInteger(CP->getTargetFlags()); 590 break; 591 } 592 case ISD::TargetIndex: { 593 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 594 ID.AddInteger(TI->getIndex()); 595 ID.AddInteger(TI->getOffset()); 596 ID.AddInteger(TI->getTargetFlags()); 597 break; 598 } 599 case ISD::LOAD: { 600 const LoadSDNode *LD = cast<LoadSDNode>(N); 601 ID.AddInteger(LD->getMemoryVT().getRawBits()); 602 ID.AddInteger(LD->getRawSubclassData()); 603 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 604 break; 605 } 606 case ISD::STORE: { 607 const StoreSDNode *ST = cast<StoreSDNode>(N); 608 ID.AddInteger(ST->getMemoryVT().getRawBits()); 609 ID.AddInteger(ST->getRawSubclassData()); 610 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 611 break; 612 } 613 case ISD::MLOAD: { 614 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N); 615 ID.AddInteger(MLD->getMemoryVT().getRawBits()); 616 ID.AddInteger(MLD->getRawSubclassData()); 617 ID.AddInteger(MLD->getPointerInfo().getAddrSpace()); 618 break; 619 } 620 case ISD::MSTORE: { 621 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 622 ID.AddInteger(MST->getMemoryVT().getRawBits()); 623 ID.AddInteger(MST->getRawSubclassData()); 624 ID.AddInteger(MST->getPointerInfo().getAddrSpace()); 625 break; 626 } 627 case ISD::MGATHER: { 628 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N); 629 ID.AddInteger(MG->getMemoryVT().getRawBits()); 630 ID.AddInteger(MG->getRawSubclassData()); 631 ID.AddInteger(MG->getPointerInfo().getAddrSpace()); 632 break; 633 } 634 case ISD::MSCATTER: { 635 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N); 636 ID.AddInteger(MS->getMemoryVT().getRawBits()); 637 ID.AddInteger(MS->getRawSubclassData()); 638 ID.AddInteger(MS->getPointerInfo().getAddrSpace()); 639 break; 640 } 641 case ISD::ATOMIC_CMP_SWAP: 642 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 643 case ISD::ATOMIC_SWAP: 644 case ISD::ATOMIC_LOAD_ADD: 645 case ISD::ATOMIC_LOAD_SUB: 646 case ISD::ATOMIC_LOAD_AND: 647 case ISD::ATOMIC_LOAD_CLR: 648 case ISD::ATOMIC_LOAD_OR: 649 case ISD::ATOMIC_LOAD_XOR: 650 case ISD::ATOMIC_LOAD_NAND: 651 case ISD::ATOMIC_LOAD_MIN: 652 case ISD::ATOMIC_LOAD_MAX: 653 case ISD::ATOMIC_LOAD_UMIN: 654 case ISD::ATOMIC_LOAD_UMAX: 655 case ISD::ATOMIC_LOAD: 656 case ISD::ATOMIC_STORE: { 657 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 658 ID.AddInteger(AT->getMemoryVT().getRawBits()); 659 ID.AddInteger(AT->getRawSubclassData()); 660 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 661 break; 662 } 663 case ISD::PREFETCH: { 664 const MemSDNode *PF = cast<MemSDNode>(N); 665 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 666 break; 667 } 668 case ISD::VECTOR_SHUFFLE: { 669 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 670 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 671 i != e; ++i) 672 ID.AddInteger(SVN->getMaskElt(i)); 673 break; 674 } 675 case ISD::TargetBlockAddress: 676 case ISD::BlockAddress: { 677 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 678 ID.AddPointer(BA->getBlockAddress()); 679 ID.AddInteger(BA->getOffset()); 680 ID.AddInteger(BA->getTargetFlags()); 681 break; 682 } 683 } // end switch (N->getOpcode()) 684 685 // Target specific memory nodes could also have address spaces to check. 686 if (N->isTargetMemoryOpcode()) 687 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 688 } 689 690 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 691 /// data. 692 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 693 AddNodeIDOpcode(ID, N->getOpcode()); 694 // Add the return value info. 695 AddNodeIDValueTypes(ID, N->getVTList()); 696 // Add the operand info. 697 AddNodeIDOperands(ID, N->ops()); 698 699 // Handle SDNode leafs with special info. 700 AddNodeIDCustom(ID, N); 701 } 702 703 //===----------------------------------------------------------------------===// 704 // SelectionDAG Class 705 //===----------------------------------------------------------------------===// 706 707 /// doNotCSE - Return true if CSE should not be performed for this node. 708 static bool doNotCSE(SDNode *N) { 709 if (N->getValueType(0) == MVT::Glue) 710 return true; // Never CSE anything that produces a flag. 711 712 switch (N->getOpcode()) { 713 default: break; 714 case ISD::HANDLENODE: 715 case ISD::EH_LABEL: 716 return true; // Never CSE these nodes. 717 } 718 719 // Check that remaining values produced are not flags. 720 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 721 if (N->getValueType(i) == MVT::Glue) 722 return true; // Never CSE anything that produces a flag. 723 724 return false; 725 } 726 727 /// RemoveDeadNodes - This method deletes all unreachable nodes in the 728 /// SelectionDAG. 729 void SelectionDAG::RemoveDeadNodes() { 730 // Create a dummy node (which is not added to allnodes), that adds a reference 731 // to the root node, preventing it from being deleted. 732 HandleSDNode Dummy(getRoot()); 733 734 SmallVector<SDNode*, 128> DeadNodes; 735 736 // Add all obviously-dead nodes to the DeadNodes worklist. 737 for (SDNode &Node : allnodes()) 738 if (Node.use_empty()) 739 DeadNodes.push_back(&Node); 740 741 RemoveDeadNodes(DeadNodes); 742 743 // If the root changed (e.g. it was a dead load, update the root). 744 setRoot(Dummy.getValue()); 745 } 746 747 /// RemoveDeadNodes - This method deletes the unreachable nodes in the 748 /// given list, and any nodes that become unreachable as a result. 749 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 750 751 // Process the worklist, deleting the nodes and adding their uses to the 752 // worklist. 753 while (!DeadNodes.empty()) { 754 SDNode *N = DeadNodes.pop_back_val(); 755 // Skip to next node if we've already managed to delete the node. This could 756 // happen if replacing a node causes a node previously added to the node to 757 // be deleted. 758 if (N->getOpcode() == ISD::DELETED_NODE) 759 continue; 760 761 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 762 DUL->NodeDeleted(N, nullptr); 763 764 // Take the node out of the appropriate CSE map. 765 RemoveNodeFromCSEMaps(N); 766 767 // Next, brutally remove the operand list. This is safe to do, as there are 768 // no cycles in the graph. 769 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 770 SDUse &Use = *I++; 771 SDNode *Operand = Use.getNode(); 772 Use.set(SDValue()); 773 774 // Now that we removed this operand, see if there are no uses of it left. 775 if (Operand->use_empty()) 776 DeadNodes.push_back(Operand); 777 } 778 779 DeallocateNode(N); 780 } 781 } 782 783 void SelectionDAG::RemoveDeadNode(SDNode *N){ 784 SmallVector<SDNode*, 16> DeadNodes(1, N); 785 786 // Create a dummy node that adds a reference to the root node, preventing 787 // it from being deleted. (This matters if the root is an operand of the 788 // dead node.) 789 HandleSDNode Dummy(getRoot()); 790 791 RemoveDeadNodes(DeadNodes); 792 } 793 794 void SelectionDAG::DeleteNode(SDNode *N) { 795 // First take this out of the appropriate CSE map. 796 RemoveNodeFromCSEMaps(N); 797 798 // Finally, remove uses due to operands of this node, remove from the 799 // AllNodes list, and delete the node. 800 DeleteNodeNotInCSEMaps(N); 801 } 802 803 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 804 assert(N->getIterator() != AllNodes.begin() && 805 "Cannot delete the entry node!"); 806 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 807 808 // Drop all of the operands and decrement used node's use counts. 809 N->DropOperands(); 810 811 DeallocateNode(N); 812 } 813 814 void SDDbgInfo::erase(const SDNode *Node) { 815 DbgValMapType::iterator I = DbgValMap.find(Node); 816 if (I == DbgValMap.end()) 817 return; 818 for (auto &Val: I->second) 819 Val->setIsInvalidated(); 820 DbgValMap.erase(I); 821 } 822 823 void SelectionDAG::DeallocateNode(SDNode *N) { 824 // If we have operands, deallocate them. 825 removeOperands(N); 826 827 NodeAllocator.Deallocate(AllNodes.remove(N)); 828 829 // Set the opcode to DELETED_NODE to help catch bugs when node 830 // memory is reallocated. 831 // FIXME: There are places in SDag that have grown a dependency on the opcode 832 // value in the released node. 833 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); 834 N->NodeType = ISD::DELETED_NODE; 835 836 // If any of the SDDbgValue nodes refer to this SDNode, invalidate 837 // them and forget about that node. 838 DbgInfo->erase(N); 839 } 840 841 #ifndef NDEBUG 842 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 843 static void VerifySDNode(SDNode *N) { 844 switch (N->getOpcode()) { 845 default: 846 break; 847 case ISD::BUILD_PAIR: { 848 EVT VT = N->getValueType(0); 849 assert(N->getNumValues() == 1 && "Too many results!"); 850 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 851 "Wrong return type!"); 852 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 853 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 854 "Mismatched operand types!"); 855 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 856 "Wrong operand type!"); 857 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 858 "Wrong return type size"); 859 break; 860 } 861 case ISD::BUILD_VECTOR: { 862 assert(N->getNumValues() == 1 && "Too many results!"); 863 assert(N->getValueType(0).isVector() && "Wrong return type!"); 864 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 865 "Wrong number of operands!"); 866 EVT EltVT = N->getValueType(0).getVectorElementType(); 867 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 868 assert((I->getValueType() == EltVT || 869 (EltVT.isInteger() && I->getValueType().isInteger() && 870 EltVT.bitsLE(I->getValueType()))) && 871 "Wrong operand type!"); 872 assert(I->getValueType() == N->getOperand(0).getValueType() && 873 "Operands must all have the same type"); 874 } 875 break; 876 } 877 } 878 } 879 #endif // NDEBUG 880 881 /// Insert a newly allocated node into the DAG. 882 /// 883 /// Handles insertion into the all nodes list and CSE map, as well as 884 /// verification and other common operations when a new node is allocated. 885 void SelectionDAG::InsertNode(SDNode *N) { 886 AllNodes.push_back(N); 887 #ifndef NDEBUG 888 N->PersistentId = NextPersistentId++; 889 VerifySDNode(N); 890 #endif 891 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 892 DUL->NodeInserted(N); 893 } 894 895 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 896 /// correspond to it. This is useful when we're about to delete or repurpose 897 /// the node. We don't want future request for structurally identical nodes 898 /// to return N anymore. 899 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 900 bool Erased = false; 901 switch (N->getOpcode()) { 902 case ISD::HANDLENODE: return false; // noop. 903 case ISD::CONDCODE: 904 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 905 "Cond code doesn't exist!"); 906 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr; 907 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr; 908 break; 909 case ISD::ExternalSymbol: 910 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 911 break; 912 case ISD::TargetExternalSymbol: { 913 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 914 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>( 915 ESN->getSymbol(), ESN->getTargetFlags())); 916 break; 917 } 918 case ISD::MCSymbol: { 919 auto *MCSN = cast<MCSymbolSDNode>(N); 920 Erased = MCSymbols.erase(MCSN->getMCSymbol()); 921 break; 922 } 923 case ISD::VALUETYPE: { 924 EVT VT = cast<VTSDNode>(N)->getVT(); 925 if (VT.isExtended()) { 926 Erased = ExtendedValueTypeNodes.erase(VT); 927 } else { 928 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; 929 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; 930 } 931 break; 932 } 933 default: 934 // Remove it from the CSE Map. 935 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 936 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 937 Erased = CSEMap.RemoveNode(N); 938 break; 939 } 940 #ifndef NDEBUG 941 // Verify that the node was actually in one of the CSE maps, unless it has a 942 // flag result (which cannot be CSE'd) or is one of the special cases that are 943 // not subject to CSE. 944 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 945 !N->isMachineOpcode() && !doNotCSE(N)) { 946 N->dump(this); 947 dbgs() << "\n"; 948 llvm_unreachable("Node is not in map!"); 949 } 950 #endif 951 return Erased; 952 } 953 954 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 955 /// maps and modified in place. Add it back to the CSE maps, unless an identical 956 /// node already exists, in which case transfer all its users to the existing 957 /// node. This transfer can potentially trigger recursive merging. 958 void 959 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 960 // For node types that aren't CSE'd, just act as if no identical node 961 // already exists. 962 if (!doNotCSE(N)) { 963 SDNode *Existing = CSEMap.GetOrInsertNode(N); 964 if (Existing != N) { 965 // If there was already an existing matching node, use ReplaceAllUsesWith 966 // to replace the dead one with the existing one. This can cause 967 // recursive merging of other unrelated nodes down the line. 968 ReplaceAllUsesWith(N, Existing); 969 970 // N is now dead. Inform the listeners and delete it. 971 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 972 DUL->NodeDeleted(N, Existing); 973 DeleteNodeNotInCSEMaps(N); 974 return; 975 } 976 } 977 978 // If the node doesn't already exist, we updated it. Inform listeners. 979 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 980 DUL->NodeUpdated(N); 981 } 982 983 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 984 /// were replaced with those specified. If this node is never memoized, 985 /// return null, otherwise return a pointer to the slot it would take. If a 986 /// node already exists with these operands, the slot will be non-null. 987 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 988 void *&InsertPos) { 989 if (doNotCSE(N)) 990 return nullptr; 991 992 SDValue Ops[] = { Op }; 993 FoldingSetNodeID ID; 994 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 995 AddNodeIDCustom(ID, N); 996 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 997 if (Node) 998 Node->intersectFlagsWith(N->getFlags()); 999 return Node; 1000 } 1001 1002 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1003 /// were replaced with those specified. If this node is never memoized, 1004 /// return null, otherwise return a pointer to the slot it would take. If a 1005 /// node already exists with these operands, the slot will be non-null. 1006 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 1007 SDValue Op1, SDValue Op2, 1008 void *&InsertPos) { 1009 if (doNotCSE(N)) 1010 return nullptr; 1011 1012 SDValue Ops[] = { Op1, Op2 }; 1013 FoldingSetNodeID ID; 1014 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 1015 AddNodeIDCustom(ID, N); 1016 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 1017 if (Node) 1018 Node->intersectFlagsWith(N->getFlags()); 1019 return Node; 1020 } 1021 1022 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1023 /// were replaced with those specified. If this node is never memoized, 1024 /// return null, otherwise return a pointer to the slot it would take. If a 1025 /// node already exists with these operands, the slot will be non-null. 1026 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 1027 void *&InsertPos) { 1028 if (doNotCSE(N)) 1029 return nullptr; 1030 1031 FoldingSetNodeID ID; 1032 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 1033 AddNodeIDCustom(ID, N); 1034 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 1035 if (Node) 1036 Node->intersectFlagsWith(N->getFlags()); 1037 return Node; 1038 } 1039 1040 Align SelectionDAG::getEVTAlign(EVT VT) const { 1041 Type *Ty = VT == MVT::iPTR ? 1042 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 1043 VT.getTypeForEVT(*getContext()); 1044 1045 return getDataLayout().getABITypeAlign(Ty); 1046 } 1047 1048 // EntryNode could meaningfully have debug info if we can find it... 1049 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 1050 : TM(tm), OptLevel(OL), 1051 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 1052 Root(getEntryNode()) { 1053 InsertNode(&EntryNode); 1054 DbgInfo = new SDDbgInfo(); 1055 } 1056 1057 void SelectionDAG::init(MachineFunction &NewMF, 1058 OptimizationRemarkEmitter &NewORE, 1059 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 1060 LegacyDivergenceAnalysis * Divergence, 1061 ProfileSummaryInfo *PSIin, 1062 BlockFrequencyInfo *BFIin) { 1063 MF = &NewMF; 1064 SDAGISelPass = PassPtr; 1065 ORE = &NewORE; 1066 TLI = getSubtarget().getTargetLowering(); 1067 TSI = getSubtarget().getSelectionDAGInfo(); 1068 LibInfo = LibraryInfo; 1069 Context = &MF->getFunction().getContext(); 1070 DA = Divergence; 1071 PSI = PSIin; 1072 BFI = BFIin; 1073 } 1074 1075 SelectionDAG::~SelectionDAG() { 1076 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 1077 allnodes_clear(); 1078 OperandRecycler.clear(OperandAllocator); 1079 delete DbgInfo; 1080 } 1081 1082 bool SelectionDAG::shouldOptForSize() const { 1083 return MF->getFunction().hasOptSize() || 1084 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI); 1085 } 1086 1087 void SelectionDAG::allnodes_clear() { 1088 assert(&*AllNodes.begin() == &EntryNode); 1089 AllNodes.remove(AllNodes.begin()); 1090 while (!AllNodes.empty()) 1091 DeallocateNode(&AllNodes.front()); 1092 #ifndef NDEBUG 1093 NextPersistentId = 0; 1094 #endif 1095 } 1096 1097 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1098 void *&InsertPos) { 1099 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1100 if (N) { 1101 switch (N->getOpcode()) { 1102 default: break; 1103 case ISD::Constant: 1104 case ISD::ConstantFP: 1105 llvm_unreachable("Querying for Constant and ConstantFP nodes requires " 1106 "debug location. Use another overload."); 1107 } 1108 } 1109 return N; 1110 } 1111 1112 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1113 const SDLoc &DL, void *&InsertPos) { 1114 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1115 if (N) { 1116 switch (N->getOpcode()) { 1117 case ISD::Constant: 1118 case ISD::ConstantFP: 1119 // Erase debug location from the node if the node is used at several 1120 // different places. Do not propagate one location to all uses as it 1121 // will cause a worse single stepping debugging experience. 1122 if (N->getDebugLoc() != DL.getDebugLoc()) 1123 N->setDebugLoc(DebugLoc()); 1124 break; 1125 default: 1126 // When the node's point of use is located earlier in the instruction 1127 // sequence than its prior point of use, update its debug info to the 1128 // earlier location. 1129 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) 1130 N->setDebugLoc(DL.getDebugLoc()); 1131 break; 1132 } 1133 } 1134 return N; 1135 } 1136 1137 void SelectionDAG::clear() { 1138 allnodes_clear(); 1139 OperandRecycler.clear(OperandAllocator); 1140 OperandAllocator.Reset(); 1141 CSEMap.clear(); 1142 1143 ExtendedValueTypeNodes.clear(); 1144 ExternalSymbols.clear(); 1145 TargetExternalSymbols.clear(); 1146 MCSymbols.clear(); 1147 SDCallSiteDbgInfo.clear(); 1148 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 1149 static_cast<CondCodeSDNode*>(nullptr)); 1150 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 1151 static_cast<SDNode*>(nullptr)); 1152 1153 EntryNode.UseList = nullptr; 1154 InsertNode(&EntryNode); 1155 Root = getEntryNode(); 1156 DbgInfo->clear(); 1157 } 1158 1159 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { 1160 return VT.bitsGT(Op.getValueType()) 1161 ? getNode(ISD::FP_EXTEND, DL, VT, Op) 1162 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); 1163 } 1164 1165 std::pair<SDValue, SDValue> 1166 SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, 1167 const SDLoc &DL, EVT VT) { 1168 assert(!VT.bitsEq(Op.getValueType()) && 1169 "Strict no-op FP extend/round not allowed."); 1170 SDValue Res = 1171 VT.bitsGT(Op.getValueType()) 1172 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) 1173 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, 1174 {Chain, Op, getIntPtrConstant(0, DL)}); 1175 1176 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1)); 1177 } 1178 1179 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1180 return VT.bitsGT(Op.getValueType()) ? 1181 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 1182 getNode(ISD::TRUNCATE, DL, VT, Op); 1183 } 1184 1185 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1186 return VT.bitsGT(Op.getValueType()) ? 1187 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 1188 getNode(ISD::TRUNCATE, DL, VT, Op); 1189 } 1190 1191 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1192 return VT.bitsGT(Op.getValueType()) ? 1193 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 1194 getNode(ISD::TRUNCATE, DL, VT, Op); 1195 } 1196 1197 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, 1198 EVT OpVT) { 1199 if (VT.bitsLE(Op.getValueType())) 1200 return getNode(ISD::TRUNCATE, SL, VT, Op); 1201 1202 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); 1203 return getNode(TLI->getExtendForContent(BType), SL, VT, Op); 1204 } 1205 1206 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { 1207 EVT OpVT = Op.getValueType(); 1208 assert(VT.isInteger() && OpVT.isInteger() && 1209 "Cannot getZeroExtendInReg FP types"); 1210 assert(VT.isVector() == OpVT.isVector() && 1211 "getZeroExtendInReg type should be vector iff the operand " 1212 "type is vector!"); 1213 assert((!VT.isVector() || 1214 VT.getVectorElementCount() == OpVT.getVectorElementCount()) && 1215 "Vector element counts must match in getZeroExtendInReg"); 1216 assert(VT.bitsLE(OpVT) && "Not extending!"); 1217 if (OpVT == VT) 1218 return Op; 1219 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(), 1220 VT.getScalarSizeInBits()); 1221 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); 1222 } 1223 1224 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1225 // Only unsigned pointer semantics are supported right now. In the future this 1226 // might delegate to TLI to check pointer signedness. 1227 return getZExtOrTrunc(Op, DL, VT); 1228 } 1229 1230 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { 1231 // Only unsigned pointer semantics are supported right now. In the future this 1232 // might delegate to TLI to check pointer signedness. 1233 return getZeroExtendInReg(Op, DL, VT); 1234 } 1235 1236 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 1237 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1238 EVT EltVT = VT.getScalarType(); 1239 SDValue NegOne = 1240 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT); 1241 return getNode(ISD::XOR, DL, VT, Val, NegOne); 1242 } 1243 1244 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1245 SDValue TrueValue = getBoolConstant(true, DL, VT, VT); 1246 return getNode(ISD::XOR, DL, VT, Val, TrueValue); 1247 } 1248 1249 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, 1250 EVT OpVT) { 1251 if (!V) 1252 return getConstant(0, DL, VT); 1253 1254 switch (TLI->getBooleanContents(OpVT)) { 1255 case TargetLowering::ZeroOrOneBooleanContent: 1256 case TargetLowering::UndefinedBooleanContent: 1257 return getConstant(1, DL, VT); 1258 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1259 return getAllOnesConstant(DL, VT); 1260 } 1261 llvm_unreachable("Unexpected boolean content enum!"); 1262 } 1263 1264 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 1265 bool isT, bool isO) { 1266 EVT EltVT = VT.getScalarType(); 1267 assert((EltVT.getSizeInBits() >= 64 || 1268 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 1269 "getConstant with a uint64_t value that doesn't fit in the type!"); 1270 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); 1271 } 1272 1273 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 1274 bool isT, bool isO) { 1275 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); 1276 } 1277 1278 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, 1279 EVT VT, bool isT, bool isO) { 1280 assert(VT.isInteger() && "Cannot create FP integer constant!"); 1281 1282 EVT EltVT = VT.getScalarType(); 1283 const ConstantInt *Elt = &Val; 1284 1285 // In some cases the vector type is legal but the element type is illegal and 1286 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 1287 // inserted value (the type does not need to match the vector element type). 1288 // Any extra bits introduced will be truncated away. 1289 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 1290 TargetLowering::TypePromoteInteger) { 1291 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1292 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); 1293 Elt = ConstantInt::get(*getContext(), NewVal); 1294 } 1295 // In other cases the element type is illegal and needs to be expanded, for 1296 // example v2i64 on MIPS32. In this case, find the nearest legal type, split 1297 // the value into n parts and use a vector type with n-times the elements. 1298 // Then bitcast to the type requested. 1299 // Legalizing constants too early makes the DAGCombiner's job harder so we 1300 // only legalize if the DAG tells us we must produce legal types. 1301 else if (NewNodesMustHaveLegalTypes && VT.isVector() && 1302 TLI->getTypeAction(*getContext(), EltVT) == 1303 TargetLowering::TypeExpandInteger) { 1304 const APInt &NewVal = Elt->getValue(); 1305 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1306 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1307 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1308 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1309 1310 // Check the temporary vector is the correct size. If this fails then 1311 // getTypeToTransformTo() probably returned a type whose size (in bits) 1312 // isn't a power-of-2 factor of the requested type size. 1313 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1314 1315 SmallVector<SDValue, 2> EltParts; 1316 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1317 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1318 .zextOrTrunc(ViaEltSizeInBits), DL, 1319 ViaEltVT, isT, isO)); 1320 } 1321 1322 // EltParts is currently in little endian order. If we actually want 1323 // big-endian order then reverse it now. 1324 if (getDataLayout().isBigEndian()) 1325 std::reverse(EltParts.begin(), EltParts.end()); 1326 1327 // The elements must be reversed when the element order is different 1328 // to the endianness of the elements (because the BITCAST is itself a 1329 // vector shuffle in this situation). However, we do not need any code to 1330 // perform this reversal because getConstant() is producing a vector 1331 // splat. 1332 // This situation occurs in MIPS MSA. 1333 1334 SmallVector<SDValue, 8> Ops; 1335 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) 1336 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1337 1338 SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); 1339 return V; 1340 } 1341 1342 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1343 "APInt size does not match type size!"); 1344 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1345 FoldingSetNodeID ID; 1346 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1347 ID.AddPointer(Elt); 1348 ID.AddBoolean(isO); 1349 void *IP = nullptr; 1350 SDNode *N = nullptr; 1351 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1352 if (!VT.isVector()) 1353 return SDValue(N, 0); 1354 1355 if (!N) { 1356 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT); 1357 CSEMap.InsertNode(N, IP); 1358 InsertNode(N); 1359 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this); 1360 } 1361 1362 SDValue Result(N, 0); 1363 if (VT.isScalableVector()) 1364 Result = getSplatVector(VT, DL, Result); 1365 else if (VT.isVector()) 1366 Result = getSplatBuildVector(VT, DL, Result); 1367 1368 return Result; 1369 } 1370 1371 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, 1372 bool isTarget) { 1373 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); 1374 } 1375 1376 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, 1377 const SDLoc &DL, bool LegalTypes) { 1378 assert(VT.isInteger() && "Shift amount is not an integer type!"); 1379 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes); 1380 return getConstant(Val, DL, ShiftVT); 1381 } 1382 1383 SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, 1384 bool isTarget) { 1385 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget); 1386 } 1387 1388 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, 1389 bool isTarget) { 1390 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); 1391 } 1392 1393 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, 1394 EVT VT, bool isTarget) { 1395 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1396 1397 EVT EltVT = VT.getScalarType(); 1398 1399 // Do the map lookup using the actual bit pattern for the floating point 1400 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1401 // we don't have issues with SNANs. 1402 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1403 FoldingSetNodeID ID; 1404 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1405 ID.AddPointer(&V); 1406 void *IP = nullptr; 1407 SDNode *N = nullptr; 1408 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1409 if (!VT.isVector()) 1410 return SDValue(N, 0); 1411 1412 if (!N) { 1413 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT); 1414 CSEMap.InsertNode(N, IP); 1415 InsertNode(N); 1416 } 1417 1418 SDValue Result(N, 0); 1419 if (VT.isScalableVector()) 1420 Result = getSplatVector(VT, DL, Result); 1421 else if (VT.isVector()) 1422 Result = getSplatBuildVector(VT, DL, Result); 1423 NewSDValueDbgMsg(Result, "Creating fp constant: ", this); 1424 return Result; 1425 } 1426 1427 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, 1428 bool isTarget) { 1429 EVT EltVT = VT.getScalarType(); 1430 if (EltVT == MVT::f32) 1431 return getConstantFP(APFloat((float)Val), DL, VT, isTarget); 1432 else if (EltVT == MVT::f64) 1433 return getConstantFP(APFloat(Val), DL, VT, isTarget); 1434 else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || 1435 EltVT == MVT::f16 || EltVT == MVT::bf16) { 1436 bool Ignored; 1437 APFloat APF = APFloat(Val); 1438 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1439 &Ignored); 1440 return getConstantFP(APF, DL, VT, isTarget); 1441 } else 1442 llvm_unreachable("Unsupported type in getConstantFP"); 1443 } 1444 1445 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, 1446 EVT VT, int64_t Offset, bool isTargetGA, 1447 unsigned TargetFlags) { 1448 assert((TargetFlags == 0 || isTargetGA) && 1449 "Cannot set target flags on target-independent globals"); 1450 1451 // Truncate (with sign-extension) the offset value to the pointer size. 1452 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); 1453 if (BitWidth < 64) 1454 Offset = SignExtend64(Offset, BitWidth); 1455 1456 unsigned Opc; 1457 if (GV->isThreadLocal()) 1458 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1459 else 1460 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1461 1462 FoldingSetNodeID ID; 1463 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1464 ID.AddPointer(GV); 1465 ID.AddInteger(Offset); 1466 ID.AddInteger(TargetFlags); 1467 void *IP = nullptr; 1468 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 1469 return SDValue(E, 0); 1470 1471 auto *N = newSDNode<GlobalAddressSDNode>( 1472 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); 1473 CSEMap.InsertNode(N, IP); 1474 InsertNode(N); 1475 return SDValue(N, 0); 1476 } 1477 1478 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1479 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1480 FoldingSetNodeID ID; 1481 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1482 ID.AddInteger(FI); 1483 void *IP = nullptr; 1484 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1485 return SDValue(E, 0); 1486 1487 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget); 1488 CSEMap.InsertNode(N, IP); 1489 InsertNode(N); 1490 return SDValue(N, 0); 1491 } 1492 1493 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1494 unsigned TargetFlags) { 1495 assert((TargetFlags == 0 || isTarget) && 1496 "Cannot set target flags on target-independent jump tables"); 1497 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1498 FoldingSetNodeID ID; 1499 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1500 ID.AddInteger(JTI); 1501 ID.AddInteger(TargetFlags); 1502 void *IP = nullptr; 1503 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1504 return SDValue(E, 0); 1505 1506 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags); 1507 CSEMap.InsertNode(N, IP); 1508 InsertNode(N); 1509 return SDValue(N, 0); 1510 } 1511 1512 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1513 MaybeAlign Alignment, int Offset, 1514 bool isTarget, unsigned TargetFlags) { 1515 assert((TargetFlags == 0 || isTarget) && 1516 "Cannot set target flags on target-independent globals"); 1517 if (!Alignment) 1518 Alignment = shouldOptForSize() 1519 ? getDataLayout().getABITypeAlign(C->getType()) 1520 : getDataLayout().getPrefTypeAlign(C->getType()); 1521 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1522 FoldingSetNodeID ID; 1523 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1524 ID.AddInteger(Alignment->value()); 1525 ID.AddInteger(Offset); 1526 ID.AddPointer(C); 1527 ID.AddInteger(TargetFlags); 1528 void *IP = nullptr; 1529 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1530 return SDValue(E, 0); 1531 1532 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment, 1533 TargetFlags); 1534 CSEMap.InsertNode(N, IP); 1535 InsertNode(N); 1536 SDValue V = SDValue(N, 0); 1537 NewSDValueDbgMsg(V, "Creating new constant pool: ", this); 1538 return V; 1539 } 1540 1541 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1542 MaybeAlign Alignment, int Offset, 1543 bool isTarget, unsigned TargetFlags) { 1544 assert((TargetFlags == 0 || isTarget) && 1545 "Cannot set target flags on target-independent globals"); 1546 if (!Alignment) 1547 Alignment = getDataLayout().getPrefTypeAlign(C->getType()); 1548 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1549 FoldingSetNodeID ID; 1550 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1551 ID.AddInteger(Alignment->value()); 1552 ID.AddInteger(Offset); 1553 C->addSelectionDAGCSEId(ID); 1554 ID.AddInteger(TargetFlags); 1555 void *IP = nullptr; 1556 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1557 return SDValue(E, 0); 1558 1559 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment, 1560 TargetFlags); 1561 CSEMap.InsertNode(N, IP); 1562 InsertNode(N); 1563 return SDValue(N, 0); 1564 } 1565 1566 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1567 unsigned TargetFlags) { 1568 FoldingSetNodeID ID; 1569 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None); 1570 ID.AddInteger(Index); 1571 ID.AddInteger(Offset); 1572 ID.AddInteger(TargetFlags); 1573 void *IP = nullptr; 1574 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1575 return SDValue(E, 0); 1576 1577 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags); 1578 CSEMap.InsertNode(N, IP); 1579 InsertNode(N); 1580 return SDValue(N, 0); 1581 } 1582 1583 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1584 FoldingSetNodeID ID; 1585 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None); 1586 ID.AddPointer(MBB); 1587 void *IP = nullptr; 1588 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1589 return SDValue(E, 0); 1590 1591 auto *N = newSDNode<BasicBlockSDNode>(MBB); 1592 CSEMap.InsertNode(N, IP); 1593 InsertNode(N); 1594 return SDValue(N, 0); 1595 } 1596 1597 SDValue SelectionDAG::getValueType(EVT VT) { 1598 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1599 ValueTypeNodes.size()) 1600 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1601 1602 SDNode *&N = VT.isExtended() ? 1603 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1604 1605 if (N) return SDValue(N, 0); 1606 N = newSDNode<VTSDNode>(VT); 1607 InsertNode(N); 1608 return SDValue(N, 0); 1609 } 1610 1611 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1612 SDNode *&N = ExternalSymbols[Sym]; 1613 if (N) return SDValue(N, 0); 1614 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT); 1615 InsertNode(N); 1616 return SDValue(N, 0); 1617 } 1618 1619 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { 1620 SDNode *&N = MCSymbols[Sym]; 1621 if (N) 1622 return SDValue(N, 0); 1623 N = newSDNode<MCSymbolSDNode>(Sym, VT); 1624 InsertNode(N); 1625 return SDValue(N, 0); 1626 } 1627 1628 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1629 unsigned TargetFlags) { 1630 SDNode *&N = 1631 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)]; 1632 if (N) return SDValue(N, 0); 1633 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT); 1634 InsertNode(N); 1635 return SDValue(N, 0); 1636 } 1637 1638 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1639 if ((unsigned)Cond >= CondCodeNodes.size()) 1640 CondCodeNodes.resize(Cond+1); 1641 1642 if (!CondCodeNodes[Cond]) { 1643 auto *N = newSDNode<CondCodeSDNode>(Cond); 1644 CondCodeNodes[Cond] = N; 1645 InsertNode(N); 1646 } 1647 1648 return SDValue(CondCodeNodes[Cond], 0); 1649 } 1650 1651 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that 1652 /// point at N1 to point at N2 and indices that point at N2 to point at N1. 1653 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { 1654 std::swap(N1, N2); 1655 ShuffleVectorSDNode::commuteMask(M); 1656 } 1657 1658 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, 1659 SDValue N2, ArrayRef<int> Mask) { 1660 assert(VT.getVectorNumElements() == Mask.size() && 1661 "Must have the same number of vector elements as mask elements!"); 1662 assert(VT == N1.getValueType() && VT == N2.getValueType() && 1663 "Invalid VECTOR_SHUFFLE"); 1664 1665 // Canonicalize shuffle undef, undef -> undef 1666 if (N1.isUndef() && N2.isUndef()) 1667 return getUNDEF(VT); 1668 1669 // Validate that all indices in Mask are within the range of the elements 1670 // input to the shuffle. 1671 int NElts = Mask.size(); 1672 assert(llvm::all_of(Mask, 1673 [&](int M) { return M < (NElts * 2) && M >= -1; }) && 1674 "Index out of range"); 1675 1676 // Copy the mask so we can do any needed cleanup. 1677 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end()); 1678 1679 // Canonicalize shuffle v, v -> v, undef 1680 if (N1 == N2) { 1681 N2 = getUNDEF(VT); 1682 for (int i = 0; i != NElts; ++i) 1683 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; 1684 } 1685 1686 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1687 if (N1.isUndef()) 1688 commuteShuffle(N1, N2, MaskVec); 1689 1690 if (TLI->hasVectorBlend()) { 1691 // If shuffling a splat, try to blend the splat instead. We do this here so 1692 // that even when this arises during lowering we don't have to re-handle it. 1693 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { 1694 BitVector UndefElements; 1695 SDValue Splat = BV->getSplatValue(&UndefElements); 1696 if (!Splat) 1697 return; 1698 1699 for (int i = 0; i < NElts; ++i) { 1700 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) 1701 continue; 1702 1703 // If this input comes from undef, mark it as such. 1704 if (UndefElements[MaskVec[i] - Offset]) { 1705 MaskVec[i] = -1; 1706 continue; 1707 } 1708 1709 // If we can blend a non-undef lane, use that instead. 1710 if (!UndefElements[i]) 1711 MaskVec[i] = i + Offset; 1712 } 1713 }; 1714 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1)) 1715 BlendSplat(N1BV, 0); 1716 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2)) 1717 BlendSplat(N2BV, NElts); 1718 } 1719 1720 // Canonicalize all index into lhs, -> shuffle lhs, undef 1721 // Canonicalize all index into rhs, -> shuffle rhs, undef 1722 bool AllLHS = true, AllRHS = true; 1723 bool N2Undef = N2.isUndef(); 1724 for (int i = 0; i != NElts; ++i) { 1725 if (MaskVec[i] >= NElts) { 1726 if (N2Undef) 1727 MaskVec[i] = -1; 1728 else 1729 AllLHS = false; 1730 } else if (MaskVec[i] >= 0) { 1731 AllRHS = false; 1732 } 1733 } 1734 if (AllLHS && AllRHS) 1735 return getUNDEF(VT); 1736 if (AllLHS && !N2Undef) 1737 N2 = getUNDEF(VT); 1738 if (AllRHS) { 1739 N1 = getUNDEF(VT); 1740 commuteShuffle(N1, N2, MaskVec); 1741 } 1742 // Reset our undef status after accounting for the mask. 1743 N2Undef = N2.isUndef(); 1744 // Re-check whether both sides ended up undef. 1745 if (N1.isUndef() && N2Undef) 1746 return getUNDEF(VT); 1747 1748 // If Identity shuffle return that node. 1749 bool Identity = true, AllSame = true; 1750 for (int i = 0; i != NElts; ++i) { 1751 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; 1752 if (MaskVec[i] != MaskVec[0]) AllSame = false; 1753 } 1754 if (Identity && NElts) 1755 return N1; 1756 1757 // Shuffling a constant splat doesn't change the result. 1758 if (N2Undef) { 1759 SDValue V = N1; 1760 1761 // Look through any bitcasts. We check that these don't change the number 1762 // (and size) of elements and just changes their types. 1763 while (V.getOpcode() == ISD::BITCAST) 1764 V = V->getOperand(0); 1765 1766 // A splat should always show up as a build vector node. 1767 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) { 1768 BitVector UndefElements; 1769 SDValue Splat = BV->getSplatValue(&UndefElements); 1770 // If this is a splat of an undef, shuffling it is also undef. 1771 if (Splat && Splat.isUndef()) 1772 return getUNDEF(VT); 1773 1774 bool SameNumElts = 1775 V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); 1776 1777 // We only have a splat which can skip shuffles if there is a splatted 1778 // value and no undef lanes rearranged by the shuffle. 1779 if (Splat && UndefElements.none()) { 1780 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the 1781 // number of elements match or the value splatted is a zero constant. 1782 if (SameNumElts) 1783 return N1; 1784 if (auto *C = dyn_cast<ConstantSDNode>(Splat)) 1785 if (C->isNullValue()) 1786 return N1; 1787 } 1788 1789 // If the shuffle itself creates a splat, build the vector directly. 1790 if (AllSame && SameNumElts) { 1791 EVT BuildVT = BV->getValueType(0); 1792 const SDValue &Splatted = BV->getOperand(MaskVec[0]); 1793 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); 1794 1795 // We may have jumped through bitcasts, so the type of the 1796 // BUILD_VECTOR may not match the type of the shuffle. 1797 if (BuildVT != VT) 1798 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); 1799 return NewBV; 1800 } 1801 } 1802 } 1803 1804 FoldingSetNodeID ID; 1805 SDValue Ops[2] = { N1, N2 }; 1806 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); 1807 for (int i = 0; i != NElts; ++i) 1808 ID.AddInteger(MaskVec[i]); 1809 1810 void* IP = nullptr; 1811 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 1812 return SDValue(E, 0); 1813 1814 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1815 // SDNode doesn't have access to it. This memory will be "leaked" when 1816 // the node is deallocated, but recovered when the NodeAllocator is released. 1817 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1818 llvm::copy(MaskVec, MaskAlloc); 1819 1820 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), 1821 dl.getDebugLoc(), MaskAlloc); 1822 createOperands(N, Ops); 1823 1824 CSEMap.InsertNode(N, IP); 1825 InsertNode(N); 1826 SDValue V = SDValue(N, 0); 1827 NewSDValueDbgMsg(V, "Creating new node: ", this); 1828 return V; 1829 } 1830 1831 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { 1832 EVT VT = SV.getValueType(0); 1833 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end()); 1834 ShuffleVectorSDNode::commuteMask(MaskVec); 1835 1836 SDValue Op0 = SV.getOperand(0); 1837 SDValue Op1 = SV.getOperand(1); 1838 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); 1839 } 1840 1841 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1842 FoldingSetNodeID ID; 1843 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None); 1844 ID.AddInteger(RegNo); 1845 void *IP = nullptr; 1846 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1847 return SDValue(E, 0); 1848 1849 auto *N = newSDNode<RegisterSDNode>(RegNo, VT); 1850 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); 1851 CSEMap.InsertNode(N, IP); 1852 InsertNode(N); 1853 return SDValue(N, 0); 1854 } 1855 1856 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1857 FoldingSetNodeID ID; 1858 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None); 1859 ID.AddPointer(RegMask); 1860 void *IP = nullptr; 1861 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1862 return SDValue(E, 0); 1863 1864 auto *N = newSDNode<RegisterMaskSDNode>(RegMask); 1865 CSEMap.InsertNode(N, IP); 1866 InsertNode(N); 1867 return SDValue(N, 0); 1868 } 1869 1870 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, 1871 MCSymbol *Label) { 1872 return getLabelNode(ISD::EH_LABEL, dl, Root, Label); 1873 } 1874 1875 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, 1876 SDValue Root, MCSymbol *Label) { 1877 FoldingSetNodeID ID; 1878 SDValue Ops[] = { Root }; 1879 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); 1880 ID.AddPointer(Label); 1881 void *IP = nullptr; 1882 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1883 return SDValue(E, 0); 1884 1885 auto *N = 1886 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label); 1887 createOperands(N, Ops); 1888 1889 CSEMap.InsertNode(N, IP); 1890 InsertNode(N); 1891 return SDValue(N, 0); 1892 } 1893 1894 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1895 int64_t Offset, bool isTarget, 1896 unsigned TargetFlags) { 1897 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1898 1899 FoldingSetNodeID ID; 1900 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1901 ID.AddPointer(BA); 1902 ID.AddInteger(Offset); 1903 ID.AddInteger(TargetFlags); 1904 void *IP = nullptr; 1905 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1906 return SDValue(E, 0); 1907 1908 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags); 1909 CSEMap.InsertNode(N, IP); 1910 InsertNode(N); 1911 return SDValue(N, 0); 1912 } 1913 1914 SDValue SelectionDAG::getSrcValue(const Value *V) { 1915 FoldingSetNodeID ID; 1916 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None); 1917 ID.AddPointer(V); 1918 1919 void *IP = nullptr; 1920 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1921 return SDValue(E, 0); 1922 1923 auto *N = newSDNode<SrcValueSDNode>(V); 1924 CSEMap.InsertNode(N, IP); 1925 InsertNode(N); 1926 return SDValue(N, 0); 1927 } 1928 1929 SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1930 FoldingSetNodeID ID; 1931 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None); 1932 ID.AddPointer(MD); 1933 1934 void *IP = nullptr; 1935 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1936 return SDValue(E, 0); 1937 1938 auto *N = newSDNode<MDNodeSDNode>(MD); 1939 CSEMap.InsertNode(N, IP); 1940 InsertNode(N); 1941 return SDValue(N, 0); 1942 } 1943 1944 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { 1945 if (VT == V.getValueType()) 1946 return V; 1947 1948 return getNode(ISD::BITCAST, SDLoc(V), VT, V); 1949 } 1950 1951 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, 1952 unsigned SrcAS, unsigned DestAS) { 1953 SDValue Ops[] = {Ptr}; 1954 FoldingSetNodeID ID; 1955 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); 1956 ID.AddInteger(SrcAS); 1957 ID.AddInteger(DestAS); 1958 1959 void *IP = nullptr; 1960 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 1961 return SDValue(E, 0); 1962 1963 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), 1964 VT, SrcAS, DestAS); 1965 createOperands(N, Ops); 1966 1967 CSEMap.InsertNode(N, IP); 1968 InsertNode(N); 1969 return SDValue(N, 0); 1970 } 1971 1972 SDValue SelectionDAG::getFreeze(SDValue V) { 1973 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); 1974 } 1975 1976 /// getShiftAmountOperand - Return the specified value casted to 1977 /// the target's desired shift amount type. 1978 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1979 EVT OpTy = Op.getValueType(); 1980 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); 1981 if (OpTy == ShTy || OpTy.isVector()) return Op; 1982 1983 return getZExtOrTrunc(Op, SDLoc(Op), ShTy); 1984 } 1985 1986 SDValue SelectionDAG::expandVAArg(SDNode *Node) { 1987 SDLoc dl(Node); 1988 const TargetLowering &TLI = getTargetLoweringInfo(); 1989 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 1990 EVT VT = Node->getValueType(0); 1991 SDValue Tmp1 = Node->getOperand(0); 1992 SDValue Tmp2 = Node->getOperand(1); 1993 const MaybeAlign MA(Node->getConstantOperandVal(3)); 1994 1995 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, 1996 Tmp2, MachinePointerInfo(V)); 1997 SDValue VAList = VAListLoad; 1998 1999 if (MA && *MA > TLI.getMinStackArgumentAlignment()) { 2000 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 2001 getConstant(MA->value() - 1, dl, VAList.getValueType())); 2002 2003 VAList = 2004 getNode(ISD::AND, dl, VAList.getValueType(), VAList, 2005 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType())); 2006 } 2007 2008 // Increment the pointer, VAList, to the next vaarg 2009 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 2010 getConstant(getDataLayout().getTypeAllocSize( 2011 VT.getTypeForEVT(*getContext())), 2012 dl, VAList.getValueType())); 2013 // Store the incremented VAList to the legalized pointer 2014 Tmp1 = 2015 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); 2016 // Load the actual argument out of the pointer VAList 2017 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); 2018 } 2019 2020 SDValue SelectionDAG::expandVACopy(SDNode *Node) { 2021 SDLoc dl(Node); 2022 const TargetLowering &TLI = getTargetLoweringInfo(); 2023 // This defaults to loading a pointer from the input and storing it to the 2024 // output, returning the chain. 2025 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 2026 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 2027 SDValue Tmp1 = 2028 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), 2029 Node->getOperand(2), MachinePointerInfo(VS)); 2030 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 2031 MachinePointerInfo(VD)); 2032 } 2033 2034 Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { 2035 const DataLayout &DL = getDataLayout(); 2036 Type *Ty = VT.getTypeForEVT(*getContext()); 2037 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); 2038 2039 if (TLI->isTypeLegal(VT) || !VT.isVector()) 2040 return RedAlign; 2041 2042 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 2043 const Align StackAlign = TFI->getStackAlign(); 2044 2045 // See if we can choose a smaller ABI alignment in cases where it's an 2046 // illegal vector type that will get broken down. 2047 if (RedAlign > StackAlign) { 2048 EVT IntermediateVT; 2049 MVT RegisterVT; 2050 unsigned NumIntermediates; 2051 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT, 2052 NumIntermediates, RegisterVT); 2053 Ty = IntermediateVT.getTypeForEVT(*getContext()); 2054 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); 2055 if (RedAlign2 < RedAlign) 2056 RedAlign = RedAlign2; 2057 } 2058 2059 return RedAlign; 2060 } 2061 2062 SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { 2063 MachineFrameInfo &MFI = MF->getFrameInfo(); 2064 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 2065 int StackID = 0; 2066 if (Bytes.isScalable()) 2067 StackID = TFI->getStackIDForScalableVectors(); 2068 // The stack id gives an indication of whether the object is scalable or 2069 // not, so it's safe to pass in the minimum size here. 2070 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment, 2071 false, nullptr, StackID); 2072 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); 2073 } 2074 2075 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 2076 Type *Ty = VT.getTypeForEVT(*getContext()); 2077 Align StackAlign = 2078 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign)); 2079 return CreateStackTemporary(VT.getStoreSize(), StackAlign); 2080 } 2081 2082 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 2083 TypeSize VT1Size = VT1.getStoreSize(); 2084 TypeSize VT2Size = VT2.getStoreSize(); 2085 assert(VT1Size.isScalable() == VT2Size.isScalable() && 2086 "Don't know how to choose the maximum size when creating a stack " 2087 "temporary"); 2088 TypeSize Bytes = 2089 VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size; 2090 2091 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 2092 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 2093 const DataLayout &DL = getDataLayout(); 2094 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2)); 2095 return CreateStackTemporary(Bytes, Align); 2096 } 2097 2098 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, 2099 ISD::CondCode Cond, const SDLoc &dl) { 2100 EVT OpVT = N1.getValueType(); 2101 2102 // These setcc operations always fold. 2103 switch (Cond) { 2104 default: break; 2105 case ISD::SETFALSE: 2106 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); 2107 case ISD::SETTRUE: 2108 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); 2109 2110 case ISD::SETOEQ: 2111 case ISD::SETOGT: 2112 case ISD::SETOGE: 2113 case ISD::SETOLT: 2114 case ISD::SETOLE: 2115 case ISD::SETONE: 2116 case ISD::SETO: 2117 case ISD::SETUO: 2118 case ISD::SETUEQ: 2119 case ISD::SETUNE: 2120 assert(!OpVT.isInteger() && "Illegal setcc for integer!"); 2121 break; 2122 } 2123 2124 if (OpVT.isInteger()) { 2125 // For EQ and NE, we can always pick a value for the undef to make the 2126 // predicate pass or fail, so we can return undef. 2127 // Matches behavior in llvm::ConstantFoldCompareInstruction. 2128 // icmp eq/ne X, undef -> undef. 2129 if ((N1.isUndef() || N2.isUndef()) && 2130 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) 2131 return getUNDEF(VT); 2132 2133 // If both operands are undef, we can return undef for int comparison. 2134 // icmp undef, undef -> undef. 2135 if (N1.isUndef() && N2.isUndef()) 2136 return getUNDEF(VT); 2137 2138 // icmp X, X -> true/false 2139 // icmp X, undef -> true/false because undef could be X. 2140 if (N1 == N2) 2141 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); 2142 } 2143 2144 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) { 2145 const APInt &C2 = N2C->getAPIntValue(); 2146 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) { 2147 const APInt &C1 = N1C->getAPIntValue(); 2148 2149 switch (Cond) { 2150 default: llvm_unreachable("Unknown integer setcc!"); 2151 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT); 2152 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT); 2153 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT); 2154 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT); 2155 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT); 2156 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT); 2157 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT); 2158 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT); 2159 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT); 2160 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT); 2161 } 2162 } 2163 } 2164 2165 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 2166 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 2167 2168 if (N1CFP && N2CFP) { 2169 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF()); 2170 switch (Cond) { 2171 default: break; 2172 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 2173 return getUNDEF(VT); 2174 LLVM_FALLTHROUGH; 2175 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, 2176 OpVT); 2177 case ISD::SETNE: if (R==APFloat::cmpUnordered) 2178 return getUNDEF(VT); 2179 LLVM_FALLTHROUGH; 2180 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2181 R==APFloat::cmpLessThan, dl, VT, 2182 OpVT); 2183 case ISD::SETLT: if (R==APFloat::cmpUnordered) 2184 return getUNDEF(VT); 2185 LLVM_FALLTHROUGH; 2186 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, 2187 OpVT); 2188 case ISD::SETGT: if (R==APFloat::cmpUnordered) 2189 return getUNDEF(VT); 2190 LLVM_FALLTHROUGH; 2191 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, 2192 VT, OpVT); 2193 case ISD::SETLE: if (R==APFloat::cmpUnordered) 2194 return getUNDEF(VT); 2195 LLVM_FALLTHROUGH; 2196 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || 2197 R==APFloat::cmpEqual, dl, VT, 2198 OpVT); 2199 case ISD::SETGE: if (R==APFloat::cmpUnordered) 2200 return getUNDEF(VT); 2201 LLVM_FALLTHROUGH; 2202 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2203 R==APFloat::cmpEqual, dl, VT, OpVT); 2204 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, 2205 OpVT); 2206 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, 2207 OpVT); 2208 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || 2209 R==APFloat::cmpEqual, dl, VT, 2210 OpVT); 2211 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, 2212 OpVT); 2213 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || 2214 R==APFloat::cmpLessThan, dl, VT, 2215 OpVT); 2216 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || 2217 R==APFloat::cmpUnordered, dl, VT, 2218 OpVT); 2219 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, 2220 VT, OpVT); 2221 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, 2222 OpVT); 2223 } 2224 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { 2225 // Ensure that the constant occurs on the RHS. 2226 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 2227 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT())) 2228 return SDValue(); 2229 return getSetCC(dl, VT, N2, N1, SwappedCond); 2230 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || 2231 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { 2232 // If an operand is known to be a nan (or undef that could be a nan), we can 2233 // fold it. 2234 // Choosing NaN for the undef will always make unordered comparison succeed 2235 // and ordered comparison fails. 2236 // Matches behavior in llvm::ConstantFoldCompareInstruction. 2237 switch (ISD::getUnorderedFlavor(Cond)) { 2238 default: 2239 llvm_unreachable("Unknown flavor!"); 2240 case 0: // Known false. 2241 return getBoolConstant(false, dl, VT, OpVT); 2242 case 1: // Known true. 2243 return getBoolConstant(true, dl, VT, OpVT); 2244 case 2: // Undefined. 2245 return getUNDEF(VT); 2246 } 2247 } 2248 2249 // Could not fold it. 2250 return SDValue(); 2251 } 2252 2253 /// See if the specified operand can be simplified with the knowledge that only 2254 /// the bits specified by DemandedBits are used. 2255 /// TODO: really we should be making this into the DAG equivalent of 2256 /// SimplifyMultipleUseDemandedBits and not generate any new nodes. 2257 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) { 2258 EVT VT = V.getValueType(); 2259 2260 if (VT.isScalableVector()) 2261 return SDValue(); 2262 2263 APInt DemandedElts = VT.isVector() 2264 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 2265 : APInt(1, 1); 2266 return GetDemandedBits(V, DemandedBits, DemandedElts); 2267 } 2268 2269 /// See if the specified operand can be simplified with the knowledge that only 2270 /// the bits specified by DemandedBits are used in the elements specified by 2271 /// DemandedElts. 2272 /// TODO: really we should be making this into the DAG equivalent of 2273 /// SimplifyMultipleUseDemandedBits and not generate any new nodes. 2274 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, 2275 const APInt &DemandedElts) { 2276 switch (V.getOpcode()) { 2277 default: 2278 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts, 2279 *this, 0); 2280 case ISD::Constant: { 2281 const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue(); 2282 APInt NewVal = CVal & DemandedBits; 2283 if (NewVal != CVal) 2284 return getConstant(NewVal, SDLoc(V), V.getValueType()); 2285 break; 2286 } 2287 case ISD::SRL: 2288 // Only look at single-use SRLs. 2289 if (!V.getNode()->hasOneUse()) 2290 break; 2291 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) { 2292 // See if we can recursively simplify the LHS. 2293 unsigned Amt = RHSC->getZExtValue(); 2294 2295 // Watch out for shift count overflow though. 2296 if (Amt >= DemandedBits.getBitWidth()) 2297 break; 2298 APInt SrcDemandedBits = DemandedBits << Amt; 2299 if (SDValue SimplifyLHS = 2300 GetDemandedBits(V.getOperand(0), SrcDemandedBits)) 2301 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, 2302 V.getOperand(1)); 2303 } 2304 break; 2305 } 2306 return SDValue(); 2307 } 2308 2309 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 2310 /// use this predicate to simplify operations downstream. 2311 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 2312 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2313 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); 2314 } 2315 2316 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 2317 /// this predicate to simplify operations downstream. Mask is known to be zero 2318 /// for bits that V cannot have. 2319 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, 2320 unsigned Depth) const { 2321 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero); 2322 } 2323 2324 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in 2325 /// DemandedElts. We use this predicate to simplify operations downstream. 2326 /// Mask is known to be zero for bits that V cannot have. 2327 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, 2328 const APInt &DemandedElts, 2329 unsigned Depth) const { 2330 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); 2331 } 2332 2333 /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. 2334 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, 2335 unsigned Depth) const { 2336 return Mask.isSubsetOf(computeKnownBits(V, Depth).One); 2337 } 2338 2339 /// isSplatValue - Return true if the vector V has the same value 2340 /// across all DemandedElts. For scalable vectors it does not make 2341 /// sense to specify which elements are demanded or undefined, therefore 2342 /// they are simply ignored. 2343 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, 2344 APInt &UndefElts) { 2345 EVT VT = V.getValueType(); 2346 assert(VT.isVector() && "Vector type expected"); 2347 2348 if (!VT.isScalableVector() && !DemandedElts) 2349 return false; // No demanded elts, better to assume we don't know anything. 2350 2351 // Deal with some common cases here that work for both fixed and scalable 2352 // vector types. 2353 switch (V.getOpcode()) { 2354 case ISD::SPLAT_VECTOR: 2355 UndefElts = V.getOperand(0).isUndef() 2356 ? APInt::getAllOnesValue(DemandedElts.getBitWidth()) 2357 : APInt(DemandedElts.getBitWidth(), 0); 2358 return true; 2359 case ISD::ADD: 2360 case ISD::SUB: 2361 case ISD::AND: { 2362 APInt UndefLHS, UndefRHS; 2363 SDValue LHS = V.getOperand(0); 2364 SDValue RHS = V.getOperand(1); 2365 if (isSplatValue(LHS, DemandedElts, UndefLHS) && 2366 isSplatValue(RHS, DemandedElts, UndefRHS)) { 2367 UndefElts = UndefLHS | UndefRHS; 2368 return true; 2369 } 2370 break; 2371 } 2372 case ISD::TRUNCATE: 2373 case ISD::SIGN_EXTEND: 2374 case ISD::ZERO_EXTEND: 2375 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts); 2376 } 2377 2378 // We don't support other cases than those above for scalable vectors at 2379 // the moment. 2380 if (VT.isScalableVector()) 2381 return false; 2382 2383 unsigned NumElts = VT.getVectorNumElements(); 2384 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch"); 2385 UndefElts = APInt::getNullValue(NumElts); 2386 2387 switch (V.getOpcode()) { 2388 case ISD::BUILD_VECTOR: { 2389 SDValue Scl; 2390 for (unsigned i = 0; i != NumElts; ++i) { 2391 SDValue Op = V.getOperand(i); 2392 if (Op.isUndef()) { 2393 UndefElts.setBit(i); 2394 continue; 2395 } 2396 if (!DemandedElts[i]) 2397 continue; 2398 if (Scl && Scl != Op) 2399 return false; 2400 Scl = Op; 2401 } 2402 return true; 2403 } 2404 case ISD::VECTOR_SHUFFLE: { 2405 // Check if this is a shuffle node doing a splat. 2406 // TODO: Do we need to handle shuffle(splat, undef, mask)? 2407 int SplatIndex = -1; 2408 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask(); 2409 for (int i = 0; i != (int)NumElts; ++i) { 2410 int M = Mask[i]; 2411 if (M < 0) { 2412 UndefElts.setBit(i); 2413 continue; 2414 } 2415 if (!DemandedElts[i]) 2416 continue; 2417 if (0 <= SplatIndex && SplatIndex != M) 2418 return false; 2419 SplatIndex = M; 2420 } 2421 return true; 2422 } 2423 case ISD::EXTRACT_SUBVECTOR: { 2424 // Offset the demanded elts by the subvector index. 2425 SDValue Src = V.getOperand(0); 2426 uint64_t Idx = V.getConstantOperandVal(1); 2427 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 2428 APInt UndefSrcElts; 2429 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 2430 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts)) { 2431 UndefElts = UndefSrcElts.extractBits(NumElts, Idx); 2432 return true; 2433 } 2434 break; 2435 } 2436 } 2437 2438 return false; 2439 } 2440 2441 /// Helper wrapper to main isSplatValue function. 2442 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) { 2443 EVT VT = V.getValueType(); 2444 assert(VT.isVector() && "Vector type expected"); 2445 2446 APInt UndefElts; 2447 APInt DemandedElts; 2448 2449 // For now we don't support this with scalable vectors. 2450 if (!VT.isScalableVector()) 2451 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); 2452 return isSplatValue(V, DemandedElts, UndefElts) && 2453 (AllowUndefs || !UndefElts); 2454 } 2455 2456 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { 2457 V = peekThroughExtractSubvectors(V); 2458 2459 EVT VT = V.getValueType(); 2460 unsigned Opcode = V.getOpcode(); 2461 switch (Opcode) { 2462 default: { 2463 APInt UndefElts; 2464 APInt DemandedElts; 2465 2466 if (!VT.isScalableVector()) 2467 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); 2468 2469 if (isSplatValue(V, DemandedElts, UndefElts)) { 2470 if (VT.isScalableVector()) { 2471 // DemandedElts and UndefElts are ignored for scalable vectors, since 2472 // the only supported cases are SPLAT_VECTOR nodes. 2473 SplatIdx = 0; 2474 } else { 2475 // Handle case where all demanded elements are UNDEF. 2476 if (DemandedElts.isSubsetOf(UndefElts)) { 2477 SplatIdx = 0; 2478 return getUNDEF(VT); 2479 } 2480 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes(); 2481 } 2482 return V; 2483 } 2484 break; 2485 } 2486 case ISD::SPLAT_VECTOR: 2487 SplatIdx = 0; 2488 return V; 2489 case ISD::VECTOR_SHUFFLE: { 2490 if (VT.isScalableVector()) 2491 return SDValue(); 2492 2493 // Check if this is a shuffle node doing a splat. 2494 // TODO - remove this and rely purely on SelectionDAG::isSplatValue, 2495 // getTargetVShiftNode currently struggles without the splat source. 2496 auto *SVN = cast<ShuffleVectorSDNode>(V); 2497 if (!SVN->isSplat()) 2498 break; 2499 int Idx = SVN->getSplatIndex(); 2500 int NumElts = V.getValueType().getVectorNumElements(); 2501 SplatIdx = Idx % NumElts; 2502 return V.getOperand(Idx / NumElts); 2503 } 2504 } 2505 2506 return SDValue(); 2507 } 2508 2509 SDValue SelectionDAG::getSplatValue(SDValue V) { 2510 int SplatIdx; 2511 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) 2512 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), 2513 SrcVector.getValueType().getScalarType(), SrcVector, 2514 getVectorIdxConstant(SplatIdx, SDLoc(V))); 2515 return SDValue(); 2516 } 2517 2518 const APInt * 2519 SelectionDAG::getValidShiftAmountConstant(SDValue V, 2520 const APInt &DemandedElts) const { 2521 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2522 V.getOpcode() == ISD::SRA) && 2523 "Unknown shift node"); 2524 unsigned BitWidth = V.getScalarValueSizeInBits(); 2525 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) { 2526 // Shifting more than the bitwidth is not valid. 2527 const APInt &ShAmt = SA->getAPIntValue(); 2528 if (ShAmt.ult(BitWidth)) 2529 return &ShAmt; 2530 } 2531 return nullptr; 2532 } 2533 2534 const APInt *SelectionDAG::getValidMinimumShiftAmountConstant( 2535 SDValue V, const APInt &DemandedElts) const { 2536 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2537 V.getOpcode() == ISD::SRA) && 2538 "Unknown shift node"); 2539 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) 2540 return ValidAmt; 2541 unsigned BitWidth = V.getScalarValueSizeInBits(); 2542 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); 2543 if (!BV) 2544 return nullptr; 2545 const APInt *MinShAmt = nullptr; 2546 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 2547 if (!DemandedElts[i]) 2548 continue; 2549 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); 2550 if (!SA) 2551 return nullptr; 2552 // Shifting more than the bitwidth is not valid. 2553 const APInt &ShAmt = SA->getAPIntValue(); 2554 if (ShAmt.uge(BitWidth)) 2555 return nullptr; 2556 if (MinShAmt && MinShAmt->ule(ShAmt)) 2557 continue; 2558 MinShAmt = &ShAmt; 2559 } 2560 return MinShAmt; 2561 } 2562 2563 const APInt *SelectionDAG::getValidMaximumShiftAmountConstant( 2564 SDValue V, const APInt &DemandedElts) const { 2565 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2566 V.getOpcode() == ISD::SRA) && 2567 "Unknown shift node"); 2568 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) 2569 return ValidAmt; 2570 unsigned BitWidth = V.getScalarValueSizeInBits(); 2571 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); 2572 if (!BV) 2573 return nullptr; 2574 const APInt *MaxShAmt = nullptr; 2575 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 2576 if (!DemandedElts[i]) 2577 continue; 2578 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); 2579 if (!SA) 2580 return nullptr; 2581 // Shifting more than the bitwidth is not valid. 2582 const APInt &ShAmt = SA->getAPIntValue(); 2583 if (ShAmt.uge(BitWidth)) 2584 return nullptr; 2585 if (MaxShAmt && MaxShAmt->uge(ShAmt)) 2586 continue; 2587 MaxShAmt = &ShAmt; 2588 } 2589 return MaxShAmt; 2590 } 2591 2592 /// Determine which bits of Op are known to be either zero or one and return 2593 /// them in Known. For vectors, the known bits are those that are shared by 2594 /// every vector element. 2595 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { 2596 EVT VT = Op.getValueType(); 2597 2598 // TOOD: Until we have a plan for how to represent demanded elements for 2599 // scalable vectors, we can just bail out for now. 2600 if (Op.getValueType().isScalableVector()) { 2601 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2602 return KnownBits(BitWidth); 2603 } 2604 2605 APInt DemandedElts = VT.isVector() 2606 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 2607 : APInt(1, 1); 2608 return computeKnownBits(Op, DemandedElts, Depth); 2609 } 2610 2611 /// Determine which bits of Op are known to be either zero or one and return 2612 /// them in Known. The DemandedElts argument allows us to only collect the known 2613 /// bits that are shared by the requested vector elements. 2614 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, 2615 unsigned Depth) const { 2616 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2617 2618 KnownBits Known(BitWidth); // Don't know anything. 2619 2620 // TOOD: Until we have a plan for how to represent demanded elements for 2621 // scalable vectors, we can just bail out for now. 2622 if (Op.getValueType().isScalableVector()) 2623 return Known; 2624 2625 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 2626 // We know all of the bits for a constant! 2627 return KnownBits::makeConstant(C->getAPIntValue()); 2628 } 2629 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) { 2630 // We know all of the bits for a constant fp! 2631 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); 2632 } 2633 2634 if (Depth >= MaxRecursionDepth) 2635 return Known; // Limit search depth. 2636 2637 KnownBits Known2; 2638 unsigned NumElts = DemandedElts.getBitWidth(); 2639 assert((!Op.getValueType().isVector() || 2640 NumElts == Op.getValueType().getVectorNumElements()) && 2641 "Unexpected vector size"); 2642 2643 if (!DemandedElts) 2644 return Known; // No demanded elts, better to assume we don't know anything. 2645 2646 unsigned Opcode = Op.getOpcode(); 2647 switch (Opcode) { 2648 case ISD::BUILD_VECTOR: 2649 // Collect the known bits that are shared by every demanded vector element. 2650 Known.Zero.setAllBits(); Known.One.setAllBits(); 2651 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 2652 if (!DemandedElts[i]) 2653 continue; 2654 2655 SDValue SrcOp = Op.getOperand(i); 2656 Known2 = computeKnownBits(SrcOp, Depth + 1); 2657 2658 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 2659 if (SrcOp.getValueSizeInBits() != BitWidth) { 2660 assert(SrcOp.getValueSizeInBits() > BitWidth && 2661 "Expected BUILD_VECTOR implicit truncation"); 2662 Known2 = Known2.trunc(BitWidth); 2663 } 2664 2665 // Known bits are the values that are shared by every demanded element. 2666 Known = KnownBits::commonBits(Known, Known2); 2667 2668 // If we don't know any bits, early out. 2669 if (Known.isUnknown()) 2670 break; 2671 } 2672 break; 2673 case ISD::VECTOR_SHUFFLE: { 2674 // Collect the known bits that are shared by every vector element referenced 2675 // by the shuffle. 2676 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 2677 Known.Zero.setAllBits(); Known.One.setAllBits(); 2678 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 2679 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 2680 for (unsigned i = 0; i != NumElts; ++i) { 2681 if (!DemandedElts[i]) 2682 continue; 2683 2684 int M = SVN->getMaskElt(i); 2685 if (M < 0) { 2686 // For UNDEF elements, we don't know anything about the common state of 2687 // the shuffle result. 2688 Known.resetAll(); 2689 DemandedLHS.clearAllBits(); 2690 DemandedRHS.clearAllBits(); 2691 break; 2692 } 2693 2694 if ((unsigned)M < NumElts) 2695 DemandedLHS.setBit((unsigned)M % NumElts); 2696 else 2697 DemandedRHS.setBit((unsigned)M % NumElts); 2698 } 2699 // Known bits are the values that are shared by every demanded element. 2700 if (!!DemandedLHS) { 2701 SDValue LHS = Op.getOperand(0); 2702 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); 2703 Known = KnownBits::commonBits(Known, Known2); 2704 } 2705 // If we don't know any bits, early out. 2706 if (Known.isUnknown()) 2707 break; 2708 if (!!DemandedRHS) { 2709 SDValue RHS = Op.getOperand(1); 2710 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); 2711 Known = KnownBits::commonBits(Known, Known2); 2712 } 2713 break; 2714 } 2715 case ISD::CONCAT_VECTORS: { 2716 // Split DemandedElts and test each of the demanded subvectors. 2717 Known.Zero.setAllBits(); Known.One.setAllBits(); 2718 EVT SubVectorVT = Op.getOperand(0).getValueType(); 2719 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 2720 unsigned NumSubVectors = Op.getNumOperands(); 2721 for (unsigned i = 0; i != NumSubVectors; ++i) { 2722 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); 2723 DemandedSub = DemandedSub.trunc(NumSubVectorElts); 2724 if (!!DemandedSub) { 2725 SDValue Sub = Op.getOperand(i); 2726 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); 2727 Known = KnownBits::commonBits(Known, Known2); 2728 } 2729 // If we don't know any bits, early out. 2730 if (Known.isUnknown()) 2731 break; 2732 } 2733 break; 2734 } 2735 case ISD::INSERT_SUBVECTOR: { 2736 // Demand any elements from the subvector and the remainder from the src its 2737 // inserted into. 2738 SDValue Src = Op.getOperand(0); 2739 SDValue Sub = Op.getOperand(1); 2740 uint64_t Idx = Op.getConstantOperandVal(2); 2741 unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); 2742 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); 2743 APInt DemandedSrcElts = DemandedElts; 2744 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx); 2745 2746 Known.One.setAllBits(); 2747 Known.Zero.setAllBits(); 2748 if (!!DemandedSubElts) { 2749 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1); 2750 if (Known.isUnknown()) 2751 break; // early-out. 2752 } 2753 if (!!DemandedSrcElts) { 2754 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); 2755 Known = KnownBits::commonBits(Known, Known2); 2756 } 2757 break; 2758 } 2759 case ISD::EXTRACT_SUBVECTOR: { 2760 // Offset the demanded elts by the subvector index. 2761 SDValue Src = Op.getOperand(0); 2762 // Bail until we can represent demanded elements for scalable vectors. 2763 if (Src.getValueType().isScalableVector()) 2764 break; 2765 uint64_t Idx = Op.getConstantOperandVal(1); 2766 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 2767 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 2768 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1); 2769 break; 2770 } 2771 case ISD::SCALAR_TO_VECTOR: { 2772 // We know about scalar_to_vector as much as we know about it source, 2773 // which becomes the first element of otherwise unknown vector. 2774 if (DemandedElts != 1) 2775 break; 2776 2777 SDValue N0 = Op.getOperand(0); 2778 Known = computeKnownBits(N0, Depth + 1); 2779 if (N0.getValueSizeInBits() != BitWidth) 2780 Known = Known.trunc(BitWidth); 2781 2782 break; 2783 } 2784 case ISD::BITCAST: { 2785 SDValue N0 = Op.getOperand(0); 2786 EVT SubVT = N0.getValueType(); 2787 unsigned SubBitWidth = SubVT.getScalarSizeInBits(); 2788 2789 // Ignore bitcasts from unsupported types. 2790 if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) 2791 break; 2792 2793 // Fast handling of 'identity' bitcasts. 2794 if (BitWidth == SubBitWidth) { 2795 Known = computeKnownBits(N0, DemandedElts, Depth + 1); 2796 break; 2797 } 2798 2799 bool IsLE = getDataLayout().isLittleEndian(); 2800 2801 // Bitcast 'small element' vector to 'large element' scalar/vector. 2802 if ((BitWidth % SubBitWidth) == 0) { 2803 assert(N0.getValueType().isVector() && "Expected bitcast from vector"); 2804 2805 // Collect known bits for the (larger) output by collecting the known 2806 // bits from each set of sub elements and shift these into place. 2807 // We need to separately call computeKnownBits for each set of 2808 // sub elements as the knownbits for each is likely to be different. 2809 unsigned SubScale = BitWidth / SubBitWidth; 2810 APInt SubDemandedElts(NumElts * SubScale, 0); 2811 for (unsigned i = 0; i != NumElts; ++i) 2812 if (DemandedElts[i]) 2813 SubDemandedElts.setBit(i * SubScale); 2814 2815 for (unsigned i = 0; i != SubScale; ++i) { 2816 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i), 2817 Depth + 1); 2818 unsigned Shifts = IsLE ? i : SubScale - 1 - i; 2819 Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts); 2820 Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts); 2821 } 2822 } 2823 2824 // Bitcast 'large element' scalar/vector to 'small element' vector. 2825 if ((SubBitWidth % BitWidth) == 0) { 2826 assert(Op.getValueType().isVector() && "Expected bitcast to vector"); 2827 2828 // Collect known bits for the (smaller) output by collecting the known 2829 // bits from the overlapping larger input elements and extracting the 2830 // sub sections we actually care about. 2831 unsigned SubScale = SubBitWidth / BitWidth; 2832 APInt SubDemandedElts(NumElts / SubScale, 0); 2833 for (unsigned i = 0; i != NumElts; ++i) 2834 if (DemandedElts[i]) 2835 SubDemandedElts.setBit(i / SubScale); 2836 2837 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1); 2838 2839 Known.Zero.setAllBits(); Known.One.setAllBits(); 2840 for (unsigned i = 0; i != NumElts; ++i) 2841 if (DemandedElts[i]) { 2842 unsigned Shifts = IsLE ? i : NumElts - 1 - i; 2843 unsigned Offset = (Shifts % SubScale) * BitWidth; 2844 Known.One &= Known2.One.lshr(Offset).trunc(BitWidth); 2845 Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth); 2846 // If we don't know any bits, early out. 2847 if (Known.isUnknown()) 2848 break; 2849 } 2850 } 2851 break; 2852 } 2853 case ISD::AND: 2854 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2855 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2856 2857 Known &= Known2; 2858 break; 2859 case ISD::OR: 2860 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2861 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2862 2863 Known |= Known2; 2864 break; 2865 case ISD::XOR: 2866 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2867 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2868 2869 Known ^= Known2; 2870 break; 2871 case ISD::MUL: { 2872 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2873 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2874 Known = KnownBits::computeForMul(Known, Known2); 2875 break; 2876 } 2877 case ISD::UDIV: { 2878 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2879 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2880 Known = KnownBits::udiv(Known, Known2); 2881 break; 2882 } 2883 case ISD::SELECT: 2884 case ISD::VSELECT: 2885 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); 2886 // If we don't know any bits, early out. 2887 if (Known.isUnknown()) 2888 break; 2889 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); 2890 2891 // Only known if known in both the LHS and RHS. 2892 Known = KnownBits::commonBits(Known, Known2); 2893 break; 2894 case ISD::SELECT_CC: 2895 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); 2896 // If we don't know any bits, early out. 2897 if (Known.isUnknown()) 2898 break; 2899 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); 2900 2901 // Only known if known in both the LHS and RHS. 2902 Known = KnownBits::commonBits(Known, Known2); 2903 break; 2904 case ISD::SMULO: 2905 case ISD::UMULO: 2906 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 2907 if (Op.getResNo() != 1) 2908 break; 2909 // The boolean result conforms to getBooleanContents. 2910 // If we know the result of a setcc has the top bits zero, use this info. 2911 // We know that we have an integer-based boolean since these operations 2912 // are only available for integer. 2913 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == 2914 TargetLowering::ZeroOrOneBooleanContent && 2915 BitWidth > 1) 2916 Known.Zero.setBitsFrom(1); 2917 break; 2918 case ISD::SETCC: 2919 case ISD::STRICT_FSETCC: 2920 case ISD::STRICT_FSETCCS: { 2921 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; 2922 // If we know the result of a setcc has the top bits zero, use this info. 2923 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == 2924 TargetLowering::ZeroOrOneBooleanContent && 2925 BitWidth > 1) 2926 Known.Zero.setBitsFrom(1); 2927 break; 2928 } 2929 case ISD::SHL: 2930 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2931 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2932 Known = KnownBits::shl(Known, Known2); 2933 2934 // Minimum shift low bits are known zero. 2935 if (const APInt *ShMinAmt = 2936 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 2937 Known.Zero.setLowBits(ShMinAmt->getZExtValue()); 2938 break; 2939 case ISD::SRL: 2940 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2941 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2942 Known = KnownBits::lshr(Known, Known2); 2943 2944 // Minimum shift high bits are known zero. 2945 if (const APInt *ShMinAmt = 2946 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 2947 Known.Zero.setHighBits(ShMinAmt->getZExtValue()); 2948 break; 2949 case ISD::SRA: 2950 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2951 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2952 Known = KnownBits::ashr(Known, Known2); 2953 // TODO: Add minimum shift high known sign bits. 2954 break; 2955 case ISD::FSHL: 2956 case ISD::FSHR: 2957 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) { 2958 unsigned Amt = C->getAPIntValue().urem(BitWidth); 2959 2960 // For fshl, 0-shift returns the 1st arg. 2961 // For fshr, 0-shift returns the 2nd arg. 2962 if (Amt == 0) { 2963 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), 2964 DemandedElts, Depth + 1); 2965 break; 2966 } 2967 2968 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) 2969 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) 2970 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2971 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2972 if (Opcode == ISD::FSHL) { 2973 Known.One <<= Amt; 2974 Known.Zero <<= Amt; 2975 Known2.One.lshrInPlace(BitWidth - Amt); 2976 Known2.Zero.lshrInPlace(BitWidth - Amt); 2977 } else { 2978 Known.One <<= BitWidth - Amt; 2979 Known.Zero <<= BitWidth - Amt; 2980 Known2.One.lshrInPlace(Amt); 2981 Known2.Zero.lshrInPlace(Amt); 2982 } 2983 Known.One |= Known2.One; 2984 Known.Zero |= Known2.Zero; 2985 } 2986 break; 2987 case ISD::SIGN_EXTEND_INREG: { 2988 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2989 unsigned EBits = EVT.getScalarSizeInBits(); 2990 2991 // Sign extension. Compute the demanded bits in the result that are not 2992 // present in the input. 2993 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 2994 2995 APInt InSignMask = APInt::getSignMask(EBits); 2996 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 2997 2998 // If the sign extended bits are demanded, we know that the sign 2999 // bit is demanded. 3000 InSignMask = InSignMask.zext(BitWidth); 3001 if (NewBits.getBoolValue()) 3002 InputDemandedBits |= InSignMask; 3003 3004 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3005 Known.One &= InputDemandedBits; 3006 Known.Zero &= InputDemandedBits; 3007 3008 // If the sign bit of the input is known set or clear, then we know the 3009 // top bits of the result. 3010 if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear 3011 Known.Zero |= NewBits; 3012 Known.One &= ~NewBits; 3013 } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set 3014 Known.One |= NewBits; 3015 Known.Zero &= ~NewBits; 3016 } else { // Input sign bit unknown 3017 Known.Zero &= ~NewBits; 3018 Known.One &= ~NewBits; 3019 } 3020 break; 3021 } 3022 case ISD::CTTZ: 3023 case ISD::CTTZ_ZERO_UNDEF: { 3024 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3025 // If we have a known 1, its position is our upper bound. 3026 unsigned PossibleTZ = Known2.countMaxTrailingZeros(); 3027 unsigned LowBits = Log2_32(PossibleTZ) + 1; 3028 Known.Zero.setBitsFrom(LowBits); 3029 break; 3030 } 3031 case ISD::CTLZ: 3032 case ISD::CTLZ_ZERO_UNDEF: { 3033 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3034 // If we have a known 1, its position is our upper bound. 3035 unsigned PossibleLZ = Known2.countMaxLeadingZeros(); 3036 unsigned LowBits = Log2_32(PossibleLZ) + 1; 3037 Known.Zero.setBitsFrom(LowBits); 3038 break; 3039 } 3040 case ISD::CTPOP: { 3041 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3042 // If we know some of the bits are zero, they can't be one. 3043 unsigned PossibleOnes = Known2.countMaxPopulation(); 3044 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); 3045 break; 3046 } 3047 case ISD::PARITY: { 3048 // Parity returns 0 everywhere but the LSB. 3049 Known.Zero.setBitsFrom(1); 3050 break; 3051 } 3052 case ISD::LOAD: { 3053 LoadSDNode *LD = cast<LoadSDNode>(Op); 3054 const Constant *Cst = TLI->getTargetConstantFromLoad(LD); 3055 if (ISD::isNON_EXTLoad(LD) && Cst) { 3056 // Determine any common known bits from the loaded constant pool value. 3057 Type *CstTy = Cst->getType(); 3058 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) { 3059 // If its a vector splat, then we can (quickly) reuse the scalar path. 3060 // NOTE: We assume all elements match and none are UNDEF. 3061 if (CstTy->isVectorTy()) { 3062 if (const Constant *Splat = Cst->getSplatValue()) { 3063 Cst = Splat; 3064 CstTy = Cst->getType(); 3065 } 3066 } 3067 // TODO - do we need to handle different bitwidths? 3068 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { 3069 // Iterate across all vector elements finding common known bits. 3070 Known.One.setAllBits(); 3071 Known.Zero.setAllBits(); 3072 for (unsigned i = 0; i != NumElts; ++i) { 3073 if (!DemandedElts[i]) 3074 continue; 3075 if (Constant *Elt = Cst->getAggregateElement(i)) { 3076 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) { 3077 const APInt &Value = CInt->getValue(); 3078 Known.One &= Value; 3079 Known.Zero &= ~Value; 3080 continue; 3081 } 3082 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) { 3083 APInt Value = CFP->getValueAPF().bitcastToAPInt(); 3084 Known.One &= Value; 3085 Known.Zero &= ~Value; 3086 continue; 3087 } 3088 } 3089 Known.One.clearAllBits(); 3090 Known.Zero.clearAllBits(); 3091 break; 3092 } 3093 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { 3094 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) { 3095 const APInt &Value = CInt->getValue(); 3096 Known.One = Value; 3097 Known.Zero = ~Value; 3098 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) { 3099 APInt Value = CFP->getValueAPF().bitcastToAPInt(); 3100 Known.One = Value; 3101 Known.Zero = ~Value; 3102 } 3103 } 3104 } 3105 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 3106 // If this is a ZEXTLoad and we are looking at the loaded value. 3107 EVT VT = LD->getMemoryVT(); 3108 unsigned MemBits = VT.getScalarSizeInBits(); 3109 Known.Zero.setBitsFrom(MemBits); 3110 } else if (const MDNode *Ranges = LD->getRanges()) { 3111 if (LD->getExtensionType() == ISD::NON_EXTLOAD) 3112 computeKnownBitsFromRangeMetadata(*Ranges, Known); 3113 } 3114 break; 3115 } 3116 case ISD::ZERO_EXTEND_VECTOR_INREG: { 3117 EVT InVT = Op.getOperand(0).getValueType(); 3118 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3119 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3120 Known = Known.zext(BitWidth); 3121 break; 3122 } 3123 case ISD::ZERO_EXTEND: { 3124 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3125 Known = Known.zext(BitWidth); 3126 break; 3127 } 3128 case ISD::SIGN_EXTEND_VECTOR_INREG: { 3129 EVT InVT = Op.getOperand(0).getValueType(); 3130 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3131 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3132 // If the sign bit is known to be zero or one, then sext will extend 3133 // it to the top bits, else it will just zext. 3134 Known = Known.sext(BitWidth); 3135 break; 3136 } 3137 case ISD::SIGN_EXTEND: { 3138 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3139 // If the sign bit is known to be zero or one, then sext will extend 3140 // it to the top bits, else it will just zext. 3141 Known = Known.sext(BitWidth); 3142 break; 3143 } 3144 case ISD::ANY_EXTEND_VECTOR_INREG: { 3145 EVT InVT = Op.getOperand(0).getValueType(); 3146 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3147 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3148 Known = Known.anyext(BitWidth); 3149 break; 3150 } 3151 case ISD::ANY_EXTEND: { 3152 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3153 Known = Known.anyext(BitWidth); 3154 break; 3155 } 3156 case ISD::TRUNCATE: { 3157 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3158 Known = Known.trunc(BitWidth); 3159 break; 3160 } 3161 case ISD::AssertZext: { 3162 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 3163 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 3164 Known = computeKnownBits(Op.getOperand(0), Depth+1); 3165 Known.Zero |= (~InMask); 3166 Known.One &= (~Known.Zero); 3167 break; 3168 } 3169 case ISD::AssertAlign: { 3170 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign()); 3171 assert(LogOfAlign != 0); 3172 // If a node is guaranteed to be aligned, set low zero bits accordingly as 3173 // well as clearing one bits. 3174 Known.Zero.setLowBits(LogOfAlign); 3175 Known.One.clearLowBits(LogOfAlign); 3176 break; 3177 } 3178 case ISD::FGETSIGN: 3179 // All bits are zero except the low bit. 3180 Known.Zero.setBitsFrom(1); 3181 break; 3182 case ISD::USUBO: 3183 case ISD::SSUBO: 3184 if (Op.getResNo() == 1) { 3185 // If we know the result of a setcc has the top bits zero, use this info. 3186 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 3187 TargetLowering::ZeroOrOneBooleanContent && 3188 BitWidth > 1) 3189 Known.Zero.setBitsFrom(1); 3190 break; 3191 } 3192 LLVM_FALLTHROUGH; 3193 case ISD::SUB: 3194 case ISD::SUBC: { 3195 assert(Op.getResNo() == 0 && 3196 "We only compute knownbits for the difference here."); 3197 3198 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3199 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3200 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false, 3201 Known, Known2); 3202 break; 3203 } 3204 case ISD::UADDO: 3205 case ISD::SADDO: 3206 case ISD::ADDCARRY: 3207 if (Op.getResNo() == 1) { 3208 // If we know the result of a setcc has the top bits zero, use this info. 3209 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 3210 TargetLowering::ZeroOrOneBooleanContent && 3211 BitWidth > 1) 3212 Known.Zero.setBitsFrom(1); 3213 break; 3214 } 3215 LLVM_FALLTHROUGH; 3216 case ISD::ADD: 3217 case ISD::ADDC: 3218 case ISD::ADDE: { 3219 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here."); 3220 3221 // With ADDE and ADDCARRY, a carry bit may be added in. 3222 KnownBits Carry(1); 3223 if (Opcode == ISD::ADDE) 3224 // Can't track carry from glue, set carry to unknown. 3225 Carry.resetAll(); 3226 else if (Opcode == ISD::ADDCARRY) 3227 // TODO: Compute known bits for the carry operand. Not sure if it is worth 3228 // the trouble (how often will we find a known carry bit). And I haven't 3229 // tested this very much yet, but something like this might work: 3230 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); 3231 // Carry = Carry.zextOrTrunc(1, false); 3232 Carry.resetAll(); 3233 else 3234 Carry.setAllZero(); 3235 3236 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3237 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3238 Known = KnownBits::computeForAddCarry(Known, Known2, Carry); 3239 break; 3240 } 3241 case ISD::SREM: { 3242 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3243 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3244 Known = KnownBits::srem(Known, Known2); 3245 break; 3246 } 3247 case ISD::UREM: { 3248 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3249 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3250 Known = KnownBits::urem(Known, Known2); 3251 break; 3252 } 3253 case ISD::EXTRACT_ELEMENT: { 3254 Known = computeKnownBits(Op.getOperand(0), Depth+1); 3255 const unsigned Index = Op.getConstantOperandVal(1); 3256 const unsigned EltBitWidth = Op.getValueSizeInBits(); 3257 3258 // Remove low part of known bits mask 3259 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth); 3260 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth); 3261 3262 // Remove high part of known bit mask 3263 Known = Known.trunc(EltBitWidth); 3264 break; 3265 } 3266 case ISD::EXTRACT_VECTOR_ELT: { 3267 SDValue InVec = Op.getOperand(0); 3268 SDValue EltNo = Op.getOperand(1); 3269 EVT VecVT = InVec.getValueType(); 3270 // computeKnownBits not yet implemented for scalable vectors. 3271 if (VecVT.isScalableVector()) 3272 break; 3273 const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); 3274 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 3275 3276 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know 3277 // anything about the extended bits. 3278 if (BitWidth > EltBitWidth) 3279 Known = Known.trunc(EltBitWidth); 3280 3281 // If we know the element index, just demand that vector element, else for 3282 // an unknown element index, ignore DemandedElts and demand them all. 3283 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); 3284 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 3285 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) 3286 DemandedSrcElts = 3287 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); 3288 3289 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1); 3290 if (BitWidth > EltBitWidth) 3291 Known = Known.anyext(BitWidth); 3292 break; 3293 } 3294 case ISD::INSERT_VECTOR_ELT: { 3295 // If we know the element index, split the demand between the 3296 // source vector and the inserted element, otherwise assume we need 3297 // the original demanded vector elements and the value. 3298 SDValue InVec = Op.getOperand(0); 3299 SDValue InVal = Op.getOperand(1); 3300 SDValue EltNo = Op.getOperand(2); 3301 bool DemandedVal = true; 3302 APInt DemandedVecElts = DemandedElts; 3303 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 3304 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 3305 unsigned EltIdx = CEltNo->getZExtValue(); 3306 DemandedVal = !!DemandedElts[EltIdx]; 3307 DemandedVecElts.clearBit(EltIdx); 3308 } 3309 Known.One.setAllBits(); 3310 Known.Zero.setAllBits(); 3311 if (DemandedVal) { 3312 Known2 = computeKnownBits(InVal, Depth + 1); 3313 Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth)); 3314 } 3315 if (!!DemandedVecElts) { 3316 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); 3317 Known = KnownBits::commonBits(Known, Known2); 3318 } 3319 break; 3320 } 3321 case ISD::BITREVERSE: { 3322 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3323 Known = Known2.reverseBits(); 3324 break; 3325 } 3326 case ISD::BSWAP: { 3327 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3328 Known = Known2.byteSwap(); 3329 break; 3330 } 3331 case ISD::ABS: { 3332 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3333 Known = Known2.abs(); 3334 break; 3335 } 3336 case ISD::UMIN: { 3337 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3338 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3339 Known = KnownBits::umin(Known, Known2); 3340 break; 3341 } 3342 case ISD::UMAX: { 3343 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3344 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3345 Known = KnownBits::umax(Known, Known2); 3346 break; 3347 } 3348 case ISD::SMIN: 3349 case ISD::SMAX: { 3350 // If we have a clamp pattern, we know that the number of sign bits will be 3351 // the minimum of the clamp min/max range. 3352 bool IsMax = (Opcode == ISD::SMAX); 3353 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3354 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) 3355 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3356 CstHigh = 3357 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); 3358 if (CstLow && CstHigh) { 3359 if (!IsMax) 3360 std::swap(CstLow, CstHigh); 3361 3362 const APInt &ValueLow = CstLow->getAPIntValue(); 3363 const APInt &ValueHigh = CstHigh->getAPIntValue(); 3364 if (ValueLow.sle(ValueHigh)) { 3365 unsigned LowSignBits = ValueLow.getNumSignBits(); 3366 unsigned HighSignBits = ValueHigh.getNumSignBits(); 3367 unsigned MinSignBits = std::min(LowSignBits, HighSignBits); 3368 if (ValueLow.isNegative() && ValueHigh.isNegative()) { 3369 Known.One.setHighBits(MinSignBits); 3370 break; 3371 } 3372 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { 3373 Known.Zero.setHighBits(MinSignBits); 3374 break; 3375 } 3376 } 3377 } 3378 3379 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3380 if (Known.isUnknown()) break; // Early-out 3381 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3382 if (IsMax) 3383 Known = KnownBits::smax(Known, Known2); 3384 else 3385 Known = KnownBits::smin(Known, Known2); 3386 break; 3387 } 3388 case ISD::FrameIndex: 3389 case ISD::TargetFrameIndex: 3390 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(), 3391 Known, getMachineFunction()); 3392 break; 3393 3394 default: 3395 if (Opcode < ISD::BUILTIN_OP_END) 3396 break; 3397 LLVM_FALLTHROUGH; 3398 case ISD::INTRINSIC_WO_CHAIN: 3399 case ISD::INTRINSIC_W_CHAIN: 3400 case ISD::INTRINSIC_VOID: 3401 // Allow the target to implement this method for its nodes. 3402 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); 3403 break; 3404 } 3405 3406 assert(!Known.hasConflict() && "Bits known to be one AND zero?"); 3407 return Known; 3408 } 3409 3410 SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0, 3411 SDValue N1) const { 3412 // X + 0 never overflow 3413 if (isNullConstant(N1)) 3414 return OFK_Never; 3415 3416 KnownBits N1Known = computeKnownBits(N1); 3417 if (N1Known.Zero.getBoolValue()) { 3418 KnownBits N0Known = computeKnownBits(N0); 3419 3420 bool overflow; 3421 (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow); 3422 if (!overflow) 3423 return OFK_Never; 3424 } 3425 3426 // mulhi + 1 never overflow 3427 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && 3428 (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue()) 3429 return OFK_Never; 3430 3431 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) { 3432 KnownBits N0Known = computeKnownBits(N0); 3433 3434 if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue()) 3435 return OFK_Never; 3436 } 3437 3438 return OFK_Sometime; 3439 } 3440 3441 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const { 3442 EVT OpVT = Val.getValueType(); 3443 unsigned BitWidth = OpVT.getScalarSizeInBits(); 3444 3445 // Is the constant a known power of 2? 3446 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val)) 3447 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3448 3449 // A left-shift of a constant one will have exactly one bit set because 3450 // shifting the bit off the end is undefined. 3451 if (Val.getOpcode() == ISD::SHL) { 3452 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3453 if (C && C->getAPIntValue() == 1) 3454 return true; 3455 } 3456 3457 // Similarly, a logical right-shift of a constant sign-bit will have exactly 3458 // one bit set. 3459 if (Val.getOpcode() == ISD::SRL) { 3460 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3461 if (C && C->getAPIntValue().isSignMask()) 3462 return true; 3463 } 3464 3465 // Are all operands of a build vector constant powers of two? 3466 if (Val.getOpcode() == ISD::BUILD_VECTOR) 3467 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) { 3468 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E)) 3469 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3470 return false; 3471 })) 3472 return true; 3473 3474 // More could be done here, though the above checks are enough 3475 // to handle some common cases. 3476 3477 // Fall back to computeKnownBits to catch other known cases. 3478 KnownBits Known = computeKnownBits(Val); 3479 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1); 3480 } 3481 3482 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { 3483 EVT VT = Op.getValueType(); 3484 3485 // TODO: Assume we don't know anything for now. 3486 if (VT.isScalableVector()) 3487 return 1; 3488 3489 APInt DemandedElts = VT.isVector() 3490 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 3491 : APInt(1, 1); 3492 return ComputeNumSignBits(Op, DemandedElts, Depth); 3493 } 3494 3495 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 3496 unsigned Depth) const { 3497 EVT VT = Op.getValueType(); 3498 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"); 3499 unsigned VTBits = VT.getScalarSizeInBits(); 3500 unsigned NumElts = DemandedElts.getBitWidth(); 3501 unsigned Tmp, Tmp2; 3502 unsigned FirstAnswer = 1; 3503 3504 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 3505 const APInt &Val = C->getAPIntValue(); 3506 return Val.getNumSignBits(); 3507 } 3508 3509 if (Depth >= MaxRecursionDepth) 3510 return 1; // Limit search depth. 3511 3512 if (!DemandedElts || VT.isScalableVector()) 3513 return 1; // No demanded elts, better to assume we don't know anything. 3514 3515 unsigned Opcode = Op.getOpcode(); 3516 switch (Opcode) { 3517 default: break; 3518 case ISD::AssertSext: 3519 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3520 return VTBits-Tmp+1; 3521 case ISD::AssertZext: 3522 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3523 return VTBits-Tmp; 3524 3525 case ISD::BUILD_VECTOR: 3526 Tmp = VTBits; 3527 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { 3528 if (!DemandedElts[i]) 3529 continue; 3530 3531 SDValue SrcOp = Op.getOperand(i); 3532 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1); 3533 3534 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 3535 if (SrcOp.getValueSizeInBits() != VTBits) { 3536 assert(SrcOp.getValueSizeInBits() > VTBits && 3537 "Expected BUILD_VECTOR implicit truncation"); 3538 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; 3539 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); 3540 } 3541 Tmp = std::min(Tmp, Tmp2); 3542 } 3543 return Tmp; 3544 3545 case ISD::VECTOR_SHUFFLE: { 3546 // Collect the minimum number of sign bits that are shared by every vector 3547 // element referenced by the shuffle. 3548 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 3549 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 3550 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 3551 for (unsigned i = 0; i != NumElts; ++i) { 3552 int M = SVN->getMaskElt(i); 3553 if (!DemandedElts[i]) 3554 continue; 3555 // For UNDEF elements, we don't know anything about the common state of 3556 // the shuffle result. 3557 if (M < 0) 3558 return 1; 3559 if ((unsigned)M < NumElts) 3560 DemandedLHS.setBit((unsigned)M % NumElts); 3561 else 3562 DemandedRHS.setBit((unsigned)M % NumElts); 3563 } 3564 Tmp = std::numeric_limits<unsigned>::max(); 3565 if (!!DemandedLHS) 3566 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); 3567 if (!!DemandedRHS) { 3568 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); 3569 Tmp = std::min(Tmp, Tmp2); 3570 } 3571 // If we don't know anything, early out and try computeKnownBits fall-back. 3572 if (Tmp == 1) 3573 break; 3574 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3575 return Tmp; 3576 } 3577 3578 case ISD::BITCAST: { 3579 SDValue N0 = Op.getOperand(0); 3580 EVT SrcVT = N0.getValueType(); 3581 unsigned SrcBits = SrcVT.getScalarSizeInBits(); 3582 3583 // Ignore bitcasts from unsupported types.. 3584 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) 3585 break; 3586 3587 // Fast handling of 'identity' bitcasts. 3588 if (VTBits == SrcBits) 3589 return ComputeNumSignBits(N0, DemandedElts, Depth + 1); 3590 3591 bool IsLE = getDataLayout().isLittleEndian(); 3592 3593 // Bitcast 'large element' scalar/vector to 'small element' vector. 3594 if ((SrcBits % VTBits) == 0) { 3595 assert(VT.isVector() && "Expected bitcast to vector"); 3596 3597 unsigned Scale = SrcBits / VTBits; 3598 APInt SrcDemandedElts(NumElts / Scale, 0); 3599 for (unsigned i = 0; i != NumElts; ++i) 3600 if (DemandedElts[i]) 3601 SrcDemandedElts.setBit(i / Scale); 3602 3603 // Fast case - sign splat can be simply split across the small elements. 3604 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1); 3605 if (Tmp == SrcBits) 3606 return VTBits; 3607 3608 // Slow case - determine how far the sign extends into each sub-element. 3609 Tmp2 = VTBits; 3610 for (unsigned i = 0; i != NumElts; ++i) 3611 if (DemandedElts[i]) { 3612 unsigned SubOffset = i % Scale; 3613 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset); 3614 SubOffset = SubOffset * VTBits; 3615 if (Tmp <= SubOffset) 3616 return 1; 3617 Tmp2 = std::min(Tmp2, Tmp - SubOffset); 3618 } 3619 return Tmp2; 3620 } 3621 break; 3622 } 3623 3624 case ISD::SIGN_EXTEND: 3625 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); 3626 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp; 3627 case ISD::SIGN_EXTEND_INREG: 3628 // Max of the input and what this extends. 3629 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits(); 3630 Tmp = VTBits-Tmp+1; 3631 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3632 return std::max(Tmp, Tmp2); 3633 case ISD::SIGN_EXTEND_VECTOR_INREG: { 3634 SDValue Src = Op.getOperand(0); 3635 EVT SrcVT = Src.getValueType(); 3636 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements()); 3637 Tmp = VTBits - SrcVT.getScalarSizeInBits(); 3638 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; 3639 } 3640 case ISD::SRA: 3641 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3642 // SRA X, C -> adds C sign bits. 3643 if (const APInt *ShAmt = 3644 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 3645 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits); 3646 return Tmp; 3647 case ISD::SHL: 3648 if (const APInt *ShAmt = 3649 getValidMaximumShiftAmountConstant(Op, DemandedElts)) { 3650 // shl destroys sign bits, ensure it doesn't shift out all sign bits. 3651 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3652 if (ShAmt->ult(Tmp)) 3653 return Tmp - ShAmt->getZExtValue(); 3654 } 3655 break; 3656 case ISD::AND: 3657 case ISD::OR: 3658 case ISD::XOR: // NOT is handled here. 3659 // Logical binary ops preserve the number of sign bits at the worst. 3660 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3661 if (Tmp != 1) { 3662 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3663 FirstAnswer = std::min(Tmp, Tmp2); 3664 // We computed what we know about the sign bits as our first 3665 // answer. Now proceed to the generic code that uses 3666 // computeKnownBits, and pick whichever answer is better. 3667 } 3668 break; 3669 3670 case ISD::SELECT: 3671 case ISD::VSELECT: 3672 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3673 if (Tmp == 1) return 1; // Early out. 3674 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3675 return std::min(Tmp, Tmp2); 3676 case ISD::SELECT_CC: 3677 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3678 if (Tmp == 1) return 1; // Early out. 3679 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1); 3680 return std::min(Tmp, Tmp2); 3681 3682 case ISD::SMIN: 3683 case ISD::SMAX: { 3684 // If we have a clamp pattern, we know that the number of sign bits will be 3685 // the minimum of the clamp min/max range. 3686 bool IsMax = (Opcode == ISD::SMAX); 3687 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3688 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) 3689 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3690 CstHigh = 3691 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); 3692 if (CstLow && CstHigh) { 3693 if (!IsMax) 3694 std::swap(CstLow, CstHigh); 3695 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) { 3696 Tmp = CstLow->getAPIntValue().getNumSignBits(); 3697 Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); 3698 return std::min(Tmp, Tmp2); 3699 } 3700 } 3701 3702 // Fallback - just get the minimum number of sign bits of the operands. 3703 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3704 if (Tmp == 1) 3705 return 1; // Early out. 3706 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3707 return std::min(Tmp, Tmp2); 3708 } 3709 case ISD::UMIN: 3710 case ISD::UMAX: 3711 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3712 if (Tmp == 1) 3713 return 1; // Early out. 3714 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3715 return std::min(Tmp, Tmp2); 3716 case ISD::SADDO: 3717 case ISD::UADDO: 3718 case ISD::SSUBO: 3719 case ISD::USUBO: 3720 case ISD::SMULO: 3721 case ISD::UMULO: 3722 if (Op.getResNo() != 1) 3723 break; 3724 // The boolean result conforms to getBooleanContents. Fall through. 3725 // If setcc returns 0/-1, all bits are sign bits. 3726 // We know that we have an integer-based boolean since these operations 3727 // are only available for integer. 3728 if (TLI->getBooleanContents(VT.isVector(), false) == 3729 TargetLowering::ZeroOrNegativeOneBooleanContent) 3730 return VTBits; 3731 break; 3732 case ISD::SETCC: 3733 case ISD::STRICT_FSETCC: 3734 case ISD::STRICT_FSETCCS: { 3735 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; 3736 // If setcc returns 0/-1, all bits are sign bits. 3737 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == 3738 TargetLowering::ZeroOrNegativeOneBooleanContent) 3739 return VTBits; 3740 break; 3741 } 3742 case ISD::ROTL: 3743 case ISD::ROTR: 3744 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3745 3746 // If we're rotating an 0/-1 value, then it stays an 0/-1 value. 3747 if (Tmp == VTBits) 3748 return VTBits; 3749 3750 if (ConstantSDNode *C = 3751 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { 3752 unsigned RotAmt = C->getAPIntValue().urem(VTBits); 3753 3754 // Handle rotate right by N like a rotate left by 32-N. 3755 if (Opcode == ISD::ROTR) 3756 RotAmt = (VTBits - RotAmt) % VTBits; 3757 3758 // If we aren't rotating out all of the known-in sign bits, return the 3759 // number that are left. This handles rotl(sext(x), 1) for example. 3760 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); 3761 } 3762 break; 3763 case ISD::ADD: 3764 case ISD::ADDC: 3765 // Add can have at most one carry bit. Thus we know that the output 3766 // is, at worst, one more bit than the inputs. 3767 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3768 if (Tmp == 1) return 1; // Early out. 3769 3770 // Special case decrementing a value (ADD X, -1): 3771 if (ConstantSDNode *CRHS = 3772 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) 3773 if (CRHS->isAllOnesValue()) { 3774 KnownBits Known = 3775 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3776 3777 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3778 // sign bits set. 3779 if ((Known.Zero | 1).isAllOnesValue()) 3780 return VTBits; 3781 3782 // If we are subtracting one from a positive number, there is no carry 3783 // out of the result. 3784 if (Known.isNonNegative()) 3785 return Tmp; 3786 } 3787 3788 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3789 if (Tmp2 == 1) return 1; // Early out. 3790 return std::min(Tmp, Tmp2) - 1; 3791 case ISD::SUB: 3792 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3793 if (Tmp2 == 1) return 1; // Early out. 3794 3795 // Handle NEG. 3796 if (ConstantSDNode *CLHS = 3797 isConstOrConstSplat(Op.getOperand(0), DemandedElts)) 3798 if (CLHS->isNullValue()) { 3799 KnownBits Known = 3800 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3801 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3802 // sign bits set. 3803 if ((Known.Zero | 1).isAllOnesValue()) 3804 return VTBits; 3805 3806 // If the input is known to be positive (the sign bit is known clear), 3807 // the output of the NEG has the same number of sign bits as the input. 3808 if (Known.isNonNegative()) 3809 return Tmp2; 3810 3811 // Otherwise, we treat this like a SUB. 3812 } 3813 3814 // Sub can have at most one carry bit. Thus we know that the output 3815 // is, at worst, one more bit than the inputs. 3816 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3817 if (Tmp == 1) return 1; // Early out. 3818 return std::min(Tmp, Tmp2) - 1; 3819 case ISD::MUL: { 3820 // The output of the Mul can be at most twice the valid bits in the inputs. 3821 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3822 if (SignBitsOp0 == 1) 3823 break; 3824 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); 3825 if (SignBitsOp1 == 1) 3826 break; 3827 unsigned OutValidBits = 3828 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1); 3829 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1; 3830 } 3831 case ISD::TRUNCATE: { 3832 // Check if the sign bits of source go down as far as the truncated value. 3833 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); 3834 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3835 if (NumSrcSignBits > (NumSrcBits - VTBits)) 3836 return NumSrcSignBits - (NumSrcBits - VTBits); 3837 break; 3838 } 3839 case ISD::EXTRACT_ELEMENT: { 3840 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); 3841 const int BitWidth = Op.getValueSizeInBits(); 3842 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; 3843 3844 // Get reverse index (starting from 1), Op1 value indexes elements from 3845 // little end. Sign starts at big end. 3846 const int rIndex = Items - 1 - Op.getConstantOperandVal(1); 3847 3848 // If the sign portion ends in our element the subtraction gives correct 3849 // result. Otherwise it gives either negative or > bitwidth result 3850 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0); 3851 } 3852 case ISD::INSERT_VECTOR_ELT: { 3853 // If we know the element index, split the demand between the 3854 // source vector and the inserted element, otherwise assume we need 3855 // the original demanded vector elements and the value. 3856 SDValue InVec = Op.getOperand(0); 3857 SDValue InVal = Op.getOperand(1); 3858 SDValue EltNo = Op.getOperand(2); 3859 bool DemandedVal = true; 3860 APInt DemandedVecElts = DemandedElts; 3861 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 3862 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 3863 unsigned EltIdx = CEltNo->getZExtValue(); 3864 DemandedVal = !!DemandedElts[EltIdx]; 3865 DemandedVecElts.clearBit(EltIdx); 3866 } 3867 Tmp = std::numeric_limits<unsigned>::max(); 3868 if (DemandedVal) { 3869 // TODO - handle implicit truncation of inserted elements. 3870 if (InVal.getScalarValueSizeInBits() != VTBits) 3871 break; 3872 Tmp2 = ComputeNumSignBits(InVal, Depth + 1); 3873 Tmp = std::min(Tmp, Tmp2); 3874 } 3875 if (!!DemandedVecElts) { 3876 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1); 3877 Tmp = std::min(Tmp, Tmp2); 3878 } 3879 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3880 return Tmp; 3881 } 3882 case ISD::EXTRACT_VECTOR_ELT: { 3883 SDValue InVec = Op.getOperand(0); 3884 SDValue EltNo = Op.getOperand(1); 3885 EVT VecVT = InVec.getValueType(); 3886 const unsigned BitWidth = Op.getValueSizeInBits(); 3887 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); 3888 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 3889 3890 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know 3891 // anything about sign bits. But if the sizes match we can derive knowledge 3892 // about sign bits from the vector operand. 3893 if (BitWidth != EltBitWidth) 3894 break; 3895 3896 // If we know the element index, just demand that vector element, else for 3897 // an unknown element index, ignore DemandedElts and demand them all. 3898 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); 3899 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 3900 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) 3901 DemandedSrcElts = 3902 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); 3903 3904 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); 3905 } 3906 case ISD::EXTRACT_SUBVECTOR: { 3907 // Offset the demanded elts by the subvector index. 3908 SDValue Src = Op.getOperand(0); 3909 // Bail until we can represent demanded elements for scalable vectors. 3910 if (Src.getValueType().isScalableVector()) 3911 break; 3912 uint64_t Idx = Op.getConstantOperandVal(1); 3913 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 3914 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 3915 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); 3916 } 3917 case ISD::CONCAT_VECTORS: { 3918 // Determine the minimum number of sign bits across all demanded 3919 // elts of the input vectors. Early out if the result is already 1. 3920 Tmp = std::numeric_limits<unsigned>::max(); 3921 EVT SubVectorVT = Op.getOperand(0).getValueType(); 3922 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 3923 unsigned NumSubVectors = Op.getNumOperands(); 3924 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { 3925 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); 3926 DemandedSub = DemandedSub.trunc(NumSubVectorElts); 3927 if (!DemandedSub) 3928 continue; 3929 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); 3930 Tmp = std::min(Tmp, Tmp2); 3931 } 3932 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3933 return Tmp; 3934 } 3935 case ISD::INSERT_SUBVECTOR: { 3936 // Demand any elements from the subvector and the remainder from the src its 3937 // inserted into. 3938 SDValue Src = Op.getOperand(0); 3939 SDValue Sub = Op.getOperand(1); 3940 uint64_t Idx = Op.getConstantOperandVal(2); 3941 unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); 3942 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); 3943 APInt DemandedSrcElts = DemandedElts; 3944 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx); 3945 3946 Tmp = std::numeric_limits<unsigned>::max(); 3947 if (!!DemandedSubElts) { 3948 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1); 3949 if (Tmp == 1) 3950 return 1; // early-out 3951 } 3952 if (!!DemandedSrcElts) { 3953 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); 3954 Tmp = std::min(Tmp, Tmp2); 3955 } 3956 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3957 return Tmp; 3958 } 3959 } 3960 3961 // If we are looking at the loaded value of the SDNode. 3962 if (Op.getResNo() == 0) { 3963 // Handle LOADX separately here. EXTLOAD case will fallthrough. 3964 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 3965 unsigned ExtType = LD->getExtensionType(); 3966 switch (ExtType) { 3967 default: break; 3968 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. 3969 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 3970 return VTBits - Tmp + 1; 3971 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. 3972 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 3973 return VTBits - Tmp; 3974 case ISD::NON_EXTLOAD: 3975 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { 3976 // We only need to handle vectors - computeKnownBits should handle 3977 // scalar cases. 3978 Type *CstTy = Cst->getType(); 3979 if (CstTy->isVectorTy() && 3980 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) { 3981 Tmp = VTBits; 3982 for (unsigned i = 0; i != NumElts; ++i) { 3983 if (!DemandedElts[i]) 3984 continue; 3985 if (Constant *Elt = Cst->getAggregateElement(i)) { 3986 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) { 3987 const APInt &Value = CInt->getValue(); 3988 Tmp = std::min(Tmp, Value.getNumSignBits()); 3989 continue; 3990 } 3991 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) { 3992 APInt Value = CFP->getValueAPF().bitcastToAPInt(); 3993 Tmp = std::min(Tmp, Value.getNumSignBits()); 3994 continue; 3995 } 3996 } 3997 // Unknown type. Conservatively assume no bits match sign bit. 3998 return 1; 3999 } 4000 return Tmp; 4001 } 4002 } 4003 break; 4004 } 4005 } 4006 } 4007 4008 // Allow the target to implement this method for its nodes. 4009 if (Opcode >= ISD::BUILTIN_OP_END || 4010 Opcode == ISD::INTRINSIC_WO_CHAIN || 4011 Opcode == ISD::INTRINSIC_W_CHAIN || 4012 Opcode == ISD::INTRINSIC_VOID) { 4013 unsigned NumBits = 4014 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); 4015 if (NumBits > 1) 4016 FirstAnswer = std::max(FirstAnswer, NumBits); 4017 } 4018 4019 // Finally, if we can prove that the top bits of the result are 0's or 1's, 4020 // use this information. 4021 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth); 4022 4023 APInt Mask; 4024 if (Known.isNonNegative()) { // sign bit is 0 4025 Mask = Known.Zero; 4026 } else if (Known.isNegative()) { // sign bit is 1; 4027 Mask = Known.One; 4028 } else { 4029 // Nothing known. 4030 return FirstAnswer; 4031 } 4032 4033 // Okay, we know that the sign bit in Mask is set. Use CLO to determine 4034 // the number of identical bits in the top of the input value. 4035 Mask <<= Mask.getBitWidth()-VTBits; 4036 return std::max(FirstAnswer, Mask.countLeadingOnes()); 4037 } 4038 4039 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 4040 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 4041 !isa<ConstantSDNode>(Op.getOperand(1))) 4042 return false; 4043 4044 if (Op.getOpcode() == ISD::OR && 4045 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))) 4046 return false; 4047 4048 return true; 4049 } 4050 4051 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { 4052 // If we're told that NaNs won't happen, assume they won't. 4053 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs()) 4054 return true; 4055 4056 if (Depth >= MaxRecursionDepth) 4057 return false; // Limit search depth. 4058 4059 // TODO: Handle vectors. 4060 // If the value is a constant, we can obviously see if it is a NaN or not. 4061 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) { 4062 return !C->getValueAPF().isNaN() || 4063 (SNaN && !C->getValueAPF().isSignaling()); 4064 } 4065 4066 unsigned Opcode = Op.getOpcode(); 4067 switch (Opcode) { 4068 case ISD::FADD: 4069 case ISD::FSUB: 4070 case ISD::FMUL: 4071 case ISD::FDIV: 4072 case ISD::FREM: 4073 case ISD::FSIN: 4074 case ISD::FCOS: { 4075 if (SNaN) 4076 return true; 4077 // TODO: Need isKnownNeverInfinity 4078 return false; 4079 } 4080 case ISD::FCANONICALIZE: 4081 case ISD::FEXP: 4082 case ISD::FEXP2: 4083 case ISD::FTRUNC: 4084 case ISD::FFLOOR: 4085 case ISD::FCEIL: 4086 case ISD::FROUND: 4087 case ISD::FROUNDEVEN: 4088 case ISD::FRINT: 4089 case ISD::FNEARBYINT: { 4090 if (SNaN) 4091 return true; 4092 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4093 } 4094 case ISD::FABS: 4095 case ISD::FNEG: 4096 case ISD::FCOPYSIGN: { 4097 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4098 } 4099 case ISD::SELECT: 4100 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 4101 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 4102 case ISD::FP_EXTEND: 4103 case ISD::FP_ROUND: { 4104 if (SNaN) 4105 return true; 4106 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4107 } 4108 case ISD::SINT_TO_FP: 4109 case ISD::UINT_TO_FP: 4110 return true; 4111 case ISD::FMA: 4112 case ISD::FMAD: { 4113 if (SNaN) 4114 return true; 4115 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 4116 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 4117 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 4118 } 4119 case ISD::FSQRT: // Need is known positive 4120 case ISD::FLOG: 4121 case ISD::FLOG2: 4122 case ISD::FLOG10: 4123 case ISD::FPOWI: 4124 case ISD::FPOW: { 4125 if (SNaN) 4126 return true; 4127 // TODO: Refine on operand 4128 return false; 4129 } 4130 case ISD::FMINNUM: 4131 case ISD::FMAXNUM: { 4132 // Only one needs to be known not-nan, since it will be returned if the 4133 // other ends up being one. 4134 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) || 4135 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 4136 } 4137 case ISD::FMINNUM_IEEE: 4138 case ISD::FMAXNUM_IEEE: { 4139 if (SNaN) 4140 return true; 4141 // This can return a NaN if either operand is an sNaN, or if both operands 4142 // are NaN. 4143 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) && 4144 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) || 4145 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) && 4146 isKnownNeverSNaN(Op.getOperand(0), Depth + 1)); 4147 } 4148 case ISD::FMINIMUM: 4149 case ISD::FMAXIMUM: { 4150 // TODO: Does this quiet or return the origina NaN as-is? 4151 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 4152 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 4153 } 4154 case ISD::EXTRACT_VECTOR_ELT: { 4155 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4156 } 4157 default: 4158 if (Opcode >= ISD::BUILTIN_OP_END || 4159 Opcode == ISD::INTRINSIC_WO_CHAIN || 4160 Opcode == ISD::INTRINSIC_W_CHAIN || 4161 Opcode == ISD::INTRINSIC_VOID) { 4162 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth); 4163 } 4164 4165 return false; 4166 } 4167 } 4168 4169 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { 4170 assert(Op.getValueType().isFloatingPoint() && 4171 "Floating point type expected"); 4172 4173 // If the value is a constant, we can obviously see if it is a zero or not. 4174 // TODO: Add BuildVector support. 4175 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 4176 return !C->isZero(); 4177 return false; 4178 } 4179 4180 bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 4181 assert(!Op.getValueType().isFloatingPoint() && 4182 "Floating point types unsupported - use isKnownNeverZeroFloat"); 4183 4184 // If the value is a constant, we can obviously see if it is a zero or not. 4185 if (ISD::matchUnaryPredicate( 4186 Op, [](ConstantSDNode *C) { return !C->isNullValue(); })) 4187 return true; 4188 4189 // TODO: Recognize more cases here. 4190 switch (Op.getOpcode()) { 4191 default: break; 4192 case ISD::OR: 4193 if (isKnownNeverZero(Op.getOperand(1)) || 4194 isKnownNeverZero(Op.getOperand(0))) 4195 return true; 4196 break; 4197 } 4198 4199 return false; 4200 } 4201 4202 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 4203 // Check the obvious case. 4204 if (A == B) return true; 4205 4206 // For for negative and positive zero. 4207 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 4208 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 4209 if (CA->isZero() && CB->isZero()) return true; 4210 4211 // Otherwise they may not be equal. 4212 return false; 4213 } 4214 4215 // FIXME: unify with llvm::haveNoCommonBitsSet. 4216 // FIXME: could also handle masked merge pattern (X & ~M) op (Y & M) 4217 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { 4218 assert(A.getValueType() == B.getValueType() && 4219 "Values must have the same type"); 4220 return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue(); 4221 } 4222 4223 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, 4224 ArrayRef<SDValue> Ops, 4225 SelectionDAG &DAG) { 4226 int NumOps = Ops.size(); 4227 assert(NumOps != 0 && "Can't build an empty vector!"); 4228 assert(!VT.isScalableVector() && 4229 "BUILD_VECTOR cannot be used with scalable types"); 4230 assert(VT.getVectorNumElements() == (unsigned)NumOps && 4231 "Incorrect element count in BUILD_VECTOR!"); 4232 4233 // BUILD_VECTOR of UNDEFs is UNDEF. 4234 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) 4235 return DAG.getUNDEF(VT); 4236 4237 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. 4238 SDValue IdentitySrc; 4239 bool IsIdentity = true; 4240 for (int i = 0; i != NumOps; ++i) { 4241 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || 4242 Ops[i].getOperand(0).getValueType() != VT || 4243 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) || 4244 !isa<ConstantSDNode>(Ops[i].getOperand(1)) || 4245 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) { 4246 IsIdentity = false; 4247 break; 4248 } 4249 IdentitySrc = Ops[i].getOperand(0); 4250 } 4251 if (IsIdentity) 4252 return IdentitySrc; 4253 4254 return SDValue(); 4255 } 4256 4257 /// Try to simplify vector concatenation to an input value, undef, or build 4258 /// vector. 4259 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, 4260 ArrayRef<SDValue> Ops, 4261 SelectionDAG &DAG) { 4262 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); 4263 assert(llvm::all_of(Ops, 4264 [Ops](SDValue Op) { 4265 return Ops[0].getValueType() == Op.getValueType(); 4266 }) && 4267 "Concatenation of vectors with inconsistent value types!"); 4268 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) == 4269 VT.getVectorElementCount() && 4270 "Incorrect element count in vector concatenation!"); 4271 4272 if (Ops.size() == 1) 4273 return Ops[0]; 4274 4275 // Concat of UNDEFs is UNDEF. 4276 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) 4277 return DAG.getUNDEF(VT); 4278 4279 // Scan the operands and look for extract operations from a single source 4280 // that correspond to insertion at the same location via this concatenation: 4281 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... 4282 SDValue IdentitySrc; 4283 bool IsIdentity = true; 4284 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 4285 SDValue Op = Ops[i]; 4286 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements(); 4287 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || 4288 Op.getOperand(0).getValueType() != VT || 4289 (IdentitySrc && Op.getOperand(0) != IdentitySrc) || 4290 Op.getConstantOperandVal(1) != IdentityIndex) { 4291 IsIdentity = false; 4292 break; 4293 } 4294 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && 4295 "Unexpected identity source vector for concat of extracts"); 4296 IdentitySrc = Op.getOperand(0); 4297 } 4298 if (IsIdentity) { 4299 assert(IdentitySrc && "Failed to set source vector of extracts"); 4300 return IdentitySrc; 4301 } 4302 4303 // The code below this point is only designed to work for fixed width 4304 // vectors, so we bail out for now. 4305 if (VT.isScalableVector()) 4306 return SDValue(); 4307 4308 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be 4309 // simplified to one big BUILD_VECTOR. 4310 // FIXME: Add support for SCALAR_TO_VECTOR as well. 4311 EVT SVT = VT.getScalarType(); 4312 SmallVector<SDValue, 16> Elts; 4313 for (SDValue Op : Ops) { 4314 EVT OpVT = Op.getValueType(); 4315 if (Op.isUndef()) 4316 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); 4317 else if (Op.getOpcode() == ISD::BUILD_VECTOR) 4318 Elts.append(Op->op_begin(), Op->op_end()); 4319 else 4320 return SDValue(); 4321 } 4322 4323 // BUILD_VECTOR requires all inputs to be of the same type, find the 4324 // maximum type and extend them all. 4325 for (SDValue Op : Elts) 4326 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); 4327 4328 if (SVT.bitsGT(VT.getScalarType())) { 4329 for (SDValue &Op : Elts) { 4330 if (Op.isUndef()) 4331 Op = DAG.getUNDEF(SVT); 4332 else 4333 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) 4334 ? DAG.getZExtOrTrunc(Op, DL, SVT) 4335 : DAG.getSExtOrTrunc(Op, DL, SVT); 4336 } 4337 } 4338 4339 SDValue V = DAG.getBuildVector(VT, DL, Elts); 4340 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); 4341 return V; 4342 } 4343 4344 /// Gets or creates the specified node. 4345 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { 4346 FoldingSetNodeID ID; 4347 AddNodeIDNode(ID, Opcode, getVTList(VT), None); 4348 void *IP = nullptr; 4349 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 4350 return SDValue(E, 0); 4351 4352 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4353 getVTList(VT)); 4354 CSEMap.InsertNode(N, IP); 4355 4356 InsertNode(N); 4357 SDValue V = SDValue(N, 0); 4358 NewSDValueDbgMsg(V, "Creating new node: ", this); 4359 return V; 4360 } 4361 4362 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4363 SDValue Operand) { 4364 SDNodeFlags Flags; 4365 if (Inserter) 4366 Flags = Inserter->getFlags(); 4367 return getNode(Opcode, DL, VT, Operand, Flags); 4368 } 4369 4370 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4371 SDValue Operand, const SDNodeFlags Flags) { 4372 // Constant fold unary operations with an integer constant operand. Even 4373 // opaque constant will be folded, because the folding of unary operations 4374 // doesn't create new constants with different values. Nevertheless, the 4375 // opaque flag is preserved during folding to prevent future folding with 4376 // other constants. 4377 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) { 4378 const APInt &Val = C->getAPIntValue(); 4379 switch (Opcode) { 4380 default: break; 4381 case ISD::SIGN_EXTEND: 4382 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, 4383 C->isTargetOpcode(), C->isOpaque()); 4384 case ISD::TRUNCATE: 4385 if (C->isOpaque()) 4386 break; 4387 LLVM_FALLTHROUGH; 4388 case ISD::ANY_EXTEND: 4389 case ISD::ZERO_EXTEND: 4390 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, 4391 C->isTargetOpcode(), C->isOpaque()); 4392 case ISD::UINT_TO_FP: 4393 case ISD::SINT_TO_FP: { 4394 APFloat apf(EVTToAPFloatSemantics(VT), 4395 APInt::getNullValue(VT.getSizeInBits())); 4396 (void)apf.convertFromAPInt(Val, 4397 Opcode==ISD::SINT_TO_FP, 4398 APFloat::rmNearestTiesToEven); 4399 return getConstantFP(apf, DL, VT); 4400 } 4401 case ISD::BITCAST: 4402 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16) 4403 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); 4404 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 4405 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); 4406 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 4407 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); 4408 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128) 4409 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); 4410 break; 4411 case ISD::ABS: 4412 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), 4413 C->isOpaque()); 4414 case ISD::BITREVERSE: 4415 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), 4416 C->isOpaque()); 4417 case ISD::BSWAP: 4418 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), 4419 C->isOpaque()); 4420 case ISD::CTPOP: 4421 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(), 4422 C->isOpaque()); 4423 case ISD::CTLZ: 4424 case ISD::CTLZ_ZERO_UNDEF: 4425 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(), 4426 C->isOpaque()); 4427 case ISD::CTTZ: 4428 case ISD::CTTZ_ZERO_UNDEF: 4429 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(), 4430 C->isOpaque()); 4431 case ISD::FP16_TO_FP: { 4432 bool Ignored; 4433 APFloat FPV(APFloat::IEEEhalf(), 4434 (Val.getBitWidth() == 16) ? Val : Val.trunc(16)); 4435 4436 // This can return overflow, underflow, or inexact; we don't care. 4437 // FIXME need to be more flexible about rounding mode. 4438 (void)FPV.convert(EVTToAPFloatSemantics(VT), 4439 APFloat::rmNearestTiesToEven, &Ignored); 4440 return getConstantFP(FPV, DL, VT); 4441 } 4442 } 4443 } 4444 4445 // Constant fold unary operations with a floating point constant operand. 4446 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) { 4447 APFloat V = C->getValueAPF(); // make copy 4448 switch (Opcode) { 4449 case ISD::FNEG: 4450 V.changeSign(); 4451 return getConstantFP(V, DL, VT); 4452 case ISD::FABS: 4453 V.clearSign(); 4454 return getConstantFP(V, DL, VT); 4455 case ISD::FCEIL: { 4456 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 4457 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4458 return getConstantFP(V, DL, VT); 4459 break; 4460 } 4461 case ISD::FTRUNC: { 4462 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 4463 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4464 return getConstantFP(V, DL, VT); 4465 break; 4466 } 4467 case ISD::FFLOOR: { 4468 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 4469 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4470 return getConstantFP(V, DL, VT); 4471 break; 4472 } 4473 case ISD::FP_EXTEND: { 4474 bool ignored; 4475 // This can return overflow, underflow, or inexact; we don't care. 4476 // FIXME need to be more flexible about rounding mode. 4477 (void)V.convert(EVTToAPFloatSemantics(VT), 4478 APFloat::rmNearestTiesToEven, &ignored); 4479 return getConstantFP(V, DL, VT); 4480 } 4481 case ISD::FP_TO_SINT: 4482 case ISD::FP_TO_UINT: { 4483 bool ignored; 4484 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); 4485 // FIXME need to be more flexible about rounding mode. 4486 APFloat::opStatus s = 4487 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); 4488 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual 4489 break; 4490 return getConstant(IntVal, DL, VT); 4491 } 4492 case ISD::BITCAST: 4493 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16) 4494 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 4495 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 4496 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 4497 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 4498 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); 4499 break; 4500 case ISD::FP_TO_FP16: { 4501 bool Ignored; 4502 // This can return overflow, underflow, or inexact; we don't care. 4503 // FIXME need to be more flexible about rounding mode. 4504 (void)V.convert(APFloat::IEEEhalf(), 4505 APFloat::rmNearestTiesToEven, &Ignored); 4506 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); 4507 } 4508 } 4509 } 4510 4511 // Constant fold unary operations with a vector integer or float operand. 4512 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) { 4513 if (BV->isConstant()) { 4514 switch (Opcode) { 4515 default: 4516 // FIXME: Entirely reasonable to perform folding of other unary 4517 // operations here as the need arises. 4518 break; 4519 case ISD::FNEG: 4520 case ISD::FABS: 4521 case ISD::FCEIL: 4522 case ISD::FTRUNC: 4523 case ISD::FFLOOR: 4524 case ISD::FP_EXTEND: 4525 case ISD::FP_TO_SINT: 4526 case ISD::FP_TO_UINT: 4527 case ISD::TRUNCATE: 4528 case ISD::ANY_EXTEND: 4529 case ISD::ZERO_EXTEND: 4530 case ISD::SIGN_EXTEND: 4531 case ISD::UINT_TO_FP: 4532 case ISD::SINT_TO_FP: 4533 case ISD::ABS: 4534 case ISD::BITREVERSE: 4535 case ISD::BSWAP: 4536 case ISD::CTLZ: 4537 case ISD::CTLZ_ZERO_UNDEF: 4538 case ISD::CTTZ: 4539 case ISD::CTTZ_ZERO_UNDEF: 4540 case ISD::CTPOP: { 4541 SDValue Ops = { Operand }; 4542 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) 4543 return Fold; 4544 } 4545 } 4546 } 4547 } 4548 4549 unsigned OpOpcode = Operand.getNode()->getOpcode(); 4550 switch (Opcode) { 4551 case ISD::FREEZE: 4552 assert(VT == Operand.getValueType() && "Unexpected VT!"); 4553 break; 4554 case ISD::TokenFactor: 4555 case ISD::MERGE_VALUES: 4556 case ISD::CONCAT_VECTORS: 4557 return Operand; // Factor, merge or concat of one node? No need. 4558 case ISD::BUILD_VECTOR: { 4559 // Attempt to simplify BUILD_VECTOR. 4560 SDValue Ops[] = {Operand}; 4561 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 4562 return V; 4563 break; 4564 } 4565 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 4566 case ISD::FP_EXTEND: 4567 assert(VT.isFloatingPoint() && 4568 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 4569 if (Operand.getValueType() == VT) return Operand; // noop conversion. 4570 assert((!VT.isVector() || 4571 VT.getVectorElementCount() == 4572 Operand.getValueType().getVectorElementCount()) && 4573 "Vector element count mismatch!"); 4574 assert(Operand.getValueType().bitsLT(VT) && 4575 "Invalid fpext node, dst < src!"); 4576 if (Operand.isUndef()) 4577 return getUNDEF(VT); 4578 break; 4579 case ISD::FP_TO_SINT: 4580 case ISD::FP_TO_UINT: 4581 if (Operand.isUndef()) 4582 return getUNDEF(VT); 4583 break; 4584 case ISD::SINT_TO_FP: 4585 case ISD::UINT_TO_FP: 4586 // [us]itofp(undef) = 0, because the result value is bounded. 4587 if (Operand.isUndef()) 4588 return getConstantFP(0.0, DL, VT); 4589 break; 4590 case ISD::SIGN_EXTEND: 4591 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4592 "Invalid SIGN_EXTEND!"); 4593 assert(VT.isVector() == Operand.getValueType().isVector() && 4594 "SIGN_EXTEND result type type should be vector iff the operand " 4595 "type is vector!"); 4596 if (Operand.getValueType() == VT) return Operand; // noop extension 4597 assert((!VT.isVector() || 4598 VT.getVectorElementCount() == 4599 Operand.getValueType().getVectorElementCount()) && 4600 "Vector element count mismatch!"); 4601 assert(Operand.getValueType().bitsLT(VT) && 4602 "Invalid sext node, dst < src!"); 4603 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 4604 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4605 else if (OpOpcode == ISD::UNDEF) 4606 // sext(undef) = 0, because the top bits will all be the same. 4607 return getConstant(0, DL, VT); 4608 break; 4609 case ISD::ZERO_EXTEND: 4610 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4611 "Invalid ZERO_EXTEND!"); 4612 assert(VT.isVector() == Operand.getValueType().isVector() && 4613 "ZERO_EXTEND result type type should be vector iff the operand " 4614 "type is vector!"); 4615 if (Operand.getValueType() == VT) return Operand; // noop extension 4616 assert((!VT.isVector() || 4617 VT.getVectorElementCount() == 4618 Operand.getValueType().getVectorElementCount()) && 4619 "Vector element count mismatch!"); 4620 assert(Operand.getValueType().bitsLT(VT) && 4621 "Invalid zext node, dst < src!"); 4622 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 4623 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0)); 4624 else if (OpOpcode == ISD::UNDEF) 4625 // zext(undef) = 0, because the top bits will be zero. 4626 return getConstant(0, DL, VT); 4627 break; 4628 case ISD::ANY_EXTEND: 4629 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4630 "Invalid ANY_EXTEND!"); 4631 assert(VT.isVector() == Operand.getValueType().isVector() && 4632 "ANY_EXTEND result type type should be vector iff the operand " 4633 "type is vector!"); 4634 if (Operand.getValueType() == VT) return Operand; // noop extension 4635 assert((!VT.isVector() || 4636 VT.getVectorElementCount() == 4637 Operand.getValueType().getVectorElementCount()) && 4638 "Vector element count mismatch!"); 4639 assert(Operand.getValueType().bitsLT(VT) && 4640 "Invalid anyext node, dst < src!"); 4641 4642 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4643 OpOpcode == ISD::ANY_EXTEND) 4644 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 4645 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4646 else if (OpOpcode == ISD::UNDEF) 4647 return getUNDEF(VT); 4648 4649 // (ext (trunc x)) -> x 4650 if (OpOpcode == ISD::TRUNCATE) { 4651 SDValue OpOp = Operand.getOperand(0); 4652 if (OpOp.getValueType() == VT) { 4653 transferDbgValues(Operand, OpOp); 4654 return OpOp; 4655 } 4656 } 4657 break; 4658 case ISD::TRUNCATE: 4659 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4660 "Invalid TRUNCATE!"); 4661 assert(VT.isVector() == Operand.getValueType().isVector() && 4662 "TRUNCATE result type type should be vector iff the operand " 4663 "type is vector!"); 4664 if (Operand.getValueType() == VT) return Operand; // noop truncate 4665 assert((!VT.isVector() || 4666 VT.getVectorElementCount() == 4667 Operand.getValueType().getVectorElementCount()) && 4668 "Vector element count mismatch!"); 4669 assert(Operand.getValueType().bitsGT(VT) && 4670 "Invalid truncate node, src < dst!"); 4671 if (OpOpcode == ISD::TRUNCATE) 4672 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4673 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4674 OpOpcode == ISD::ANY_EXTEND) { 4675 // If the source is smaller than the dest, we still need an extend. 4676 if (Operand.getOperand(0).getValueType().getScalarType() 4677 .bitsLT(VT.getScalarType())) 4678 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4679 if (Operand.getOperand(0).getValueType().bitsGT(VT)) 4680 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4681 return Operand.getOperand(0); 4682 } 4683 if (OpOpcode == ISD::UNDEF) 4684 return getUNDEF(VT); 4685 break; 4686 case ISD::ANY_EXTEND_VECTOR_INREG: 4687 case ISD::ZERO_EXTEND_VECTOR_INREG: 4688 case ISD::SIGN_EXTEND_VECTOR_INREG: 4689 assert(VT.isVector() && "This DAG node is restricted to vector types."); 4690 assert(Operand.getValueType().bitsLE(VT) && 4691 "The input must be the same size or smaller than the result."); 4692 assert(VT.getVectorNumElements() < 4693 Operand.getValueType().getVectorNumElements() && 4694 "The destination vector type must have fewer lanes than the input."); 4695 break; 4696 case ISD::ABS: 4697 assert(VT.isInteger() && VT == Operand.getValueType() && 4698 "Invalid ABS!"); 4699 if (OpOpcode == ISD::UNDEF) 4700 return getUNDEF(VT); 4701 break; 4702 case ISD::BSWAP: 4703 assert(VT.isInteger() && VT == Operand.getValueType() && 4704 "Invalid BSWAP!"); 4705 assert((VT.getScalarSizeInBits() % 16 == 0) && 4706 "BSWAP types must be a multiple of 16 bits!"); 4707 if (OpOpcode == ISD::UNDEF) 4708 return getUNDEF(VT); 4709 break; 4710 case ISD::BITREVERSE: 4711 assert(VT.isInteger() && VT == Operand.getValueType() && 4712 "Invalid BITREVERSE!"); 4713 if (OpOpcode == ISD::UNDEF) 4714 return getUNDEF(VT); 4715 break; 4716 case ISD::BITCAST: 4717 // Basic sanity checking. 4718 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() && 4719 "Cannot BITCAST between types of different sizes!"); 4720 if (VT == Operand.getValueType()) return Operand; // noop conversion. 4721 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 4722 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 4723 if (OpOpcode == ISD::UNDEF) 4724 return getUNDEF(VT); 4725 break; 4726 case ISD::SCALAR_TO_VECTOR: 4727 assert(VT.isVector() && !Operand.getValueType().isVector() && 4728 (VT.getVectorElementType() == Operand.getValueType() || 4729 (VT.getVectorElementType().isInteger() && 4730 Operand.getValueType().isInteger() && 4731 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 4732 "Illegal SCALAR_TO_VECTOR node!"); 4733 if (OpOpcode == ISD::UNDEF) 4734 return getUNDEF(VT); 4735 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 4736 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 4737 isa<ConstantSDNode>(Operand.getOperand(1)) && 4738 Operand.getConstantOperandVal(1) == 0 && 4739 Operand.getOperand(0).getValueType() == VT) 4740 return Operand.getOperand(0); 4741 break; 4742 case ISD::FNEG: 4743 // Negation of an unknown bag of bits is still completely undefined. 4744 if (OpOpcode == ISD::UNDEF) 4745 return getUNDEF(VT); 4746 4747 if (OpOpcode == ISD::FNEG) // --X -> X 4748 return Operand.getOperand(0); 4749 break; 4750 case ISD::FABS: 4751 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 4752 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0)); 4753 break; 4754 case ISD::VSCALE: 4755 assert(VT == Operand.getValueType() && "Unexpected VT!"); 4756 break; 4757 case ISD::VECREDUCE_SMIN: 4758 case ISD::VECREDUCE_UMAX: 4759 if (Operand.getValueType().getScalarType() == MVT::i1) 4760 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); 4761 break; 4762 case ISD::VECREDUCE_SMAX: 4763 case ISD::VECREDUCE_UMIN: 4764 if (Operand.getValueType().getScalarType() == MVT::i1) 4765 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); 4766 break; 4767 } 4768 4769 SDNode *N; 4770 SDVTList VTs = getVTList(VT); 4771 SDValue Ops[] = {Operand}; 4772 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 4773 FoldingSetNodeID ID; 4774 AddNodeIDNode(ID, Opcode, VTs, Ops); 4775 void *IP = nullptr; 4776 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 4777 E->intersectFlagsWith(Flags); 4778 return SDValue(E, 0); 4779 } 4780 4781 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4782 N->setFlags(Flags); 4783 createOperands(N, Ops); 4784 CSEMap.InsertNode(N, IP); 4785 } else { 4786 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4787 createOperands(N, Ops); 4788 } 4789 4790 InsertNode(N); 4791 SDValue V = SDValue(N, 0); 4792 NewSDValueDbgMsg(V, "Creating new node: ", this); 4793 return V; 4794 } 4795 4796 static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1, 4797 const APInt &C2) { 4798 switch (Opcode) { 4799 case ISD::ADD: return C1 + C2; 4800 case ISD::SUB: return C1 - C2; 4801 case ISD::MUL: return C1 * C2; 4802 case ISD::AND: return C1 & C2; 4803 case ISD::OR: return C1 | C2; 4804 case ISD::XOR: return C1 ^ C2; 4805 case ISD::SHL: return C1 << C2; 4806 case ISD::SRL: return C1.lshr(C2); 4807 case ISD::SRA: return C1.ashr(C2); 4808 case ISD::ROTL: return C1.rotl(C2); 4809 case ISD::ROTR: return C1.rotr(C2); 4810 case ISD::SMIN: return C1.sle(C2) ? C1 : C2; 4811 case ISD::SMAX: return C1.sge(C2) ? C1 : C2; 4812 case ISD::UMIN: return C1.ule(C2) ? C1 : C2; 4813 case ISD::UMAX: return C1.uge(C2) ? C1 : C2; 4814 case ISD::SADDSAT: return C1.sadd_sat(C2); 4815 case ISD::UADDSAT: return C1.uadd_sat(C2); 4816 case ISD::SSUBSAT: return C1.ssub_sat(C2); 4817 case ISD::USUBSAT: return C1.usub_sat(C2); 4818 case ISD::UDIV: 4819 if (!C2.getBoolValue()) 4820 break; 4821 return C1.udiv(C2); 4822 case ISD::UREM: 4823 if (!C2.getBoolValue()) 4824 break; 4825 return C1.urem(C2); 4826 case ISD::SDIV: 4827 if (!C2.getBoolValue()) 4828 break; 4829 return C1.sdiv(C2); 4830 case ISD::SREM: 4831 if (!C2.getBoolValue()) 4832 break; 4833 return C1.srem(C2); 4834 } 4835 return llvm::None; 4836 } 4837 4838 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, 4839 const GlobalAddressSDNode *GA, 4840 const SDNode *N2) { 4841 if (GA->getOpcode() != ISD::GlobalAddress) 4842 return SDValue(); 4843 if (!TLI->isOffsetFoldingLegal(GA)) 4844 return SDValue(); 4845 auto *C2 = dyn_cast<ConstantSDNode>(N2); 4846 if (!C2) 4847 return SDValue(); 4848 int64_t Offset = C2->getSExtValue(); 4849 switch (Opcode) { 4850 case ISD::ADD: break; 4851 case ISD::SUB: Offset = -uint64_t(Offset); break; 4852 default: return SDValue(); 4853 } 4854 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT, 4855 GA->getOffset() + uint64_t(Offset)); 4856 } 4857 4858 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { 4859 switch (Opcode) { 4860 case ISD::SDIV: 4861 case ISD::UDIV: 4862 case ISD::SREM: 4863 case ISD::UREM: { 4864 // If a divisor is zero/undef or any element of a divisor vector is 4865 // zero/undef, the whole op is undef. 4866 assert(Ops.size() == 2 && "Div/rem should have 2 operands"); 4867 SDValue Divisor = Ops[1]; 4868 if (Divisor.isUndef() || isNullConstant(Divisor)) 4869 return true; 4870 4871 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && 4872 llvm::any_of(Divisor->op_values(), 4873 [](SDValue V) { return V.isUndef() || 4874 isNullConstant(V); }); 4875 // TODO: Handle signed overflow. 4876 } 4877 // TODO: Handle oversized shifts. 4878 default: 4879 return false; 4880 } 4881 } 4882 4883 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, 4884 EVT VT, ArrayRef<SDValue> Ops) { 4885 // If the opcode is a target-specific ISD node, there's nothing we can 4886 // do here and the operand rules may not line up with the below, so 4887 // bail early. 4888 if (Opcode >= ISD::BUILTIN_OP_END) 4889 return SDValue(); 4890 4891 // For now, the array Ops should only contain two values. 4892 // This enforcement will be removed once this function is merged with 4893 // FoldConstantVectorArithmetic 4894 if (Ops.size() != 2) 4895 return SDValue(); 4896 4897 if (isUndef(Opcode, Ops)) 4898 return getUNDEF(VT); 4899 4900 SDNode *N1 = Ops[0].getNode(); 4901 SDNode *N2 = Ops[1].getNode(); 4902 4903 // Handle the case of two scalars. 4904 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) { 4905 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) { 4906 if (C1->isOpaque() || C2->isOpaque()) 4907 return SDValue(); 4908 4909 Optional<APInt> FoldAttempt = 4910 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue()); 4911 if (!FoldAttempt) 4912 return SDValue(); 4913 4914 SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT); 4915 assert((!Folded || !VT.isVector()) && 4916 "Can't fold vectors ops with scalar operands"); 4917 return Folded; 4918 } 4919 } 4920 4921 // fold (add Sym, c) -> Sym+c 4922 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1)) 4923 return FoldSymbolOffset(Opcode, VT, GA, N2); 4924 if (TLI->isCommutativeBinOp(Opcode)) 4925 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2)) 4926 return FoldSymbolOffset(Opcode, VT, GA, N1); 4927 4928 // TODO: All the folds below are performed lane-by-lane and assume a fixed 4929 // vector width, however we should be able to do constant folds involving 4930 // splat vector nodes too. 4931 if (VT.isScalableVector()) 4932 return SDValue(); 4933 4934 // For fixed width vectors, extract each constant element and fold them 4935 // individually. Either input may be an undef value. 4936 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1); 4937 if (!BV1 && !N1->isUndef()) 4938 return SDValue(); 4939 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2); 4940 if (!BV2 && !N2->isUndef()) 4941 return SDValue(); 4942 // If both operands are undef, that's handled the same way as scalars. 4943 if (!BV1 && !BV2) 4944 return SDValue(); 4945 4946 assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) && 4947 "Vector binop with different number of elements in operands?"); 4948 4949 EVT SVT = VT.getScalarType(); 4950 EVT LegalSVT = SVT; 4951 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 4952 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 4953 if (LegalSVT.bitsLT(SVT)) 4954 return SDValue(); 4955 } 4956 SmallVector<SDValue, 4> Outputs; 4957 unsigned NumOps = BV1 ? BV1->getNumOperands() : BV2->getNumOperands(); 4958 for (unsigned I = 0; I != NumOps; ++I) { 4959 SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT); 4960 SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT); 4961 if (SVT.isInteger()) { 4962 if (V1->getValueType(0).bitsGT(SVT)) 4963 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1); 4964 if (V2->getValueType(0).bitsGT(SVT)) 4965 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2); 4966 } 4967 4968 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 4969 return SDValue(); 4970 4971 // Fold one vector element. 4972 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2); 4973 if (LegalSVT != SVT) 4974 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 4975 4976 // Scalar folding only succeeded if the result is a constant or UNDEF. 4977 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 4978 ScalarResult.getOpcode() != ISD::ConstantFP) 4979 return SDValue(); 4980 Outputs.push_back(ScalarResult); 4981 } 4982 4983 assert(VT.getVectorNumElements() == Outputs.size() && 4984 "Vector size mismatch!"); 4985 4986 // We may have a vector type but a scalar result. Create a splat. 4987 Outputs.resize(VT.getVectorNumElements(), Outputs.back()); 4988 4989 // Build a big vector out of the scalar elements we generated. 4990 return getBuildVector(VT, SDLoc(), Outputs); 4991 } 4992 4993 // TODO: Merge with FoldConstantArithmetic 4994 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, 4995 const SDLoc &DL, EVT VT, 4996 ArrayRef<SDValue> Ops, 4997 const SDNodeFlags Flags) { 4998 // If the opcode is a target-specific ISD node, there's nothing we can 4999 // do here and the operand rules may not line up with the below, so 5000 // bail early. 5001 if (Opcode >= ISD::BUILTIN_OP_END) 5002 return SDValue(); 5003 5004 if (isUndef(Opcode, Ops)) 5005 return getUNDEF(VT); 5006 5007 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday? 5008 if (!VT.isVector()) 5009 return SDValue(); 5010 5011 // TODO: All the folds below are performed lane-by-lane and assume a fixed 5012 // vector width, however we should be able to do constant folds involving 5013 // splat vector nodes too. 5014 if (VT.isScalableVector()) 5015 return SDValue(); 5016 5017 // From this point onwards all vectors are assumed to be fixed width. 5018 unsigned NumElts = VT.getVectorNumElements(); 5019 5020 auto IsScalarOrSameVectorSize = [&](const SDValue &Op) { 5021 return !Op.getValueType().isVector() || 5022 Op.getValueType().getVectorNumElements() == NumElts; 5023 }; 5024 5025 auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) { 5026 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op); 5027 return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) || 5028 (BV && BV->isConstant()); 5029 }; 5030 5031 // All operands must be vector types with the same number of elements as 5032 // the result type and must be either UNDEF or a build vector of constant 5033 // or UNDEF scalars. 5034 if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) || 5035 !llvm::all_of(Ops, IsScalarOrSameVectorSize)) 5036 return SDValue(); 5037 5038 // If we are comparing vectors, then the result needs to be a i1 boolean 5039 // that is then sign-extended back to the legal result type. 5040 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); 5041 5042 // Find legal integer scalar type for constant promotion and 5043 // ensure that its scalar size is at least as large as source. 5044 EVT LegalSVT = VT.getScalarType(); 5045 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 5046 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 5047 if (LegalSVT.bitsLT(VT.getScalarType())) 5048 return SDValue(); 5049 } 5050 5051 // Constant fold each scalar lane separately. 5052 SmallVector<SDValue, 4> ScalarResults; 5053 for (unsigned i = 0; i != NumElts; i++) { 5054 SmallVector<SDValue, 4> ScalarOps; 5055 for (SDValue Op : Ops) { 5056 EVT InSVT = Op.getValueType().getScalarType(); 5057 BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op); 5058 if (!InBV) { 5059 // We've checked that this is UNDEF or a constant of some kind. 5060 if (Op.isUndef()) 5061 ScalarOps.push_back(getUNDEF(InSVT)); 5062 else 5063 ScalarOps.push_back(Op); 5064 continue; 5065 } 5066 5067 SDValue ScalarOp = InBV->getOperand(i); 5068 EVT ScalarVT = ScalarOp.getValueType(); 5069 5070 // Build vector (integer) scalar operands may need implicit 5071 // truncation - do this before constant folding. 5072 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) 5073 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); 5074 5075 ScalarOps.push_back(ScalarOp); 5076 } 5077 5078 // Constant fold the scalar operands. 5079 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags); 5080 5081 // Legalize the (integer) scalar constant if necessary. 5082 if (LegalSVT != SVT) 5083 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 5084 5085 // Scalar folding only succeeded if the result is a constant or UNDEF. 5086 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 5087 ScalarResult.getOpcode() != ISD::ConstantFP) 5088 return SDValue(); 5089 ScalarResults.push_back(ScalarResult); 5090 } 5091 5092 SDValue V = getBuildVector(VT, DL, ScalarResults); 5093 NewSDValueDbgMsg(V, "New node fold constant vector: ", this); 5094 return V; 5095 } 5096 5097 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, 5098 EVT VT, SDValue N1, SDValue N2) { 5099 // TODO: We don't do any constant folding for strict FP opcodes here, but we 5100 // should. That will require dealing with a potentially non-default 5101 // rounding mode, checking the "opStatus" return value from the APFloat 5102 // math calculations, and possibly other variations. 5103 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 5104 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 5105 if (N1CFP && N2CFP) { 5106 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF(); 5107 switch (Opcode) { 5108 case ISD::FADD: 5109 C1.add(C2, APFloat::rmNearestTiesToEven); 5110 return getConstantFP(C1, DL, VT); 5111 case ISD::FSUB: 5112 C1.subtract(C2, APFloat::rmNearestTiesToEven); 5113 return getConstantFP(C1, DL, VT); 5114 case ISD::FMUL: 5115 C1.multiply(C2, APFloat::rmNearestTiesToEven); 5116 return getConstantFP(C1, DL, VT); 5117 case ISD::FDIV: 5118 C1.divide(C2, APFloat::rmNearestTiesToEven); 5119 return getConstantFP(C1, DL, VT); 5120 case ISD::FREM: 5121 C1.mod(C2); 5122 return getConstantFP(C1, DL, VT); 5123 case ISD::FCOPYSIGN: 5124 C1.copySign(C2); 5125 return getConstantFP(C1, DL, VT); 5126 default: break; 5127 } 5128 } 5129 if (N1CFP && Opcode == ISD::FP_ROUND) { 5130 APFloat C1 = N1CFP->getValueAPF(); // make copy 5131 bool Unused; 5132 // This can return overflow, underflow, or inexact; we don't care. 5133 // FIXME need to be more flexible about rounding mode. 5134 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, 5135 &Unused); 5136 return getConstantFP(C1, DL, VT); 5137 } 5138 5139 switch (Opcode) { 5140 case ISD::FSUB: 5141 // -0.0 - undef --> undef (consistent with "fneg undef") 5142 if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef()) 5143 return getUNDEF(VT); 5144 LLVM_FALLTHROUGH; 5145 5146 case ISD::FADD: 5147 case ISD::FMUL: 5148 case ISD::FDIV: 5149 case ISD::FREM: 5150 // If both operands are undef, the result is undef. If 1 operand is undef, 5151 // the result is NaN. This should match the behavior of the IR optimizer. 5152 if (N1.isUndef() && N2.isUndef()) 5153 return getUNDEF(VT); 5154 if (N1.isUndef() || N2.isUndef()) 5155 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); 5156 } 5157 return SDValue(); 5158 } 5159 5160 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { 5161 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!"); 5162 5163 // There's no need to assert on a byte-aligned pointer. All pointers are at 5164 // least byte aligned. 5165 if (A == Align(1)) 5166 return Val; 5167 5168 FoldingSetNodeID ID; 5169 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val}); 5170 ID.AddInteger(A.value()); 5171 5172 void *IP = nullptr; 5173 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 5174 return SDValue(E, 0); 5175 5176 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(), 5177 Val.getValueType(), A); 5178 createOperands(N, {Val}); 5179 5180 CSEMap.InsertNode(N, IP); 5181 InsertNode(N); 5182 5183 SDValue V(N, 0); 5184 NewSDValueDbgMsg(V, "Creating new node: ", this); 5185 return V; 5186 } 5187 5188 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5189 SDValue N1, SDValue N2) { 5190 SDNodeFlags Flags; 5191 if (Inserter) 5192 Flags = Inserter->getFlags(); 5193 return getNode(Opcode, DL, VT, N1, N2, Flags); 5194 } 5195 5196 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5197 SDValue N1, SDValue N2, const SDNodeFlags Flags) { 5198 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 5199 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 5200 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 5201 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 5202 5203 // Canonicalize constant to RHS if commutative. 5204 if (TLI->isCommutativeBinOp(Opcode)) { 5205 if (N1C && !N2C) { 5206 std::swap(N1C, N2C); 5207 std::swap(N1, N2); 5208 } else if (N1CFP && !N2CFP) { 5209 std::swap(N1CFP, N2CFP); 5210 std::swap(N1, N2); 5211 } 5212 } 5213 5214 switch (Opcode) { 5215 default: break; 5216 case ISD::TokenFactor: 5217 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 5218 N2.getValueType() == MVT::Other && "Invalid token factor!"); 5219 // Fold trivial token factors. 5220 if (N1.getOpcode() == ISD::EntryToken) return N2; 5221 if (N2.getOpcode() == ISD::EntryToken) return N1; 5222 if (N1 == N2) return N1; 5223 break; 5224 case ISD::BUILD_VECTOR: { 5225 // Attempt to simplify BUILD_VECTOR. 5226 SDValue Ops[] = {N1, N2}; 5227 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 5228 return V; 5229 break; 5230 } 5231 case ISD::CONCAT_VECTORS: { 5232 SDValue Ops[] = {N1, N2}; 5233 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 5234 return V; 5235 break; 5236 } 5237 case ISD::AND: 5238 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5239 assert(N1.getValueType() == N2.getValueType() && 5240 N1.getValueType() == VT && "Binary operator types must match!"); 5241 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 5242 // worth handling here. 5243 if (N2C && N2C->isNullValue()) 5244 return N2; 5245 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 5246 return N1; 5247 break; 5248 case ISD::OR: 5249 case ISD::XOR: 5250 case ISD::ADD: 5251 case ISD::SUB: 5252 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5253 assert(N1.getValueType() == N2.getValueType() && 5254 N1.getValueType() == VT && "Binary operator types must match!"); 5255 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 5256 // it's worth handling here. 5257 if (N2C && N2C->isNullValue()) 5258 return N1; 5259 break; 5260 case ISD::MUL: 5261 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5262 assert(N1.getValueType() == N2.getValueType() && 5263 N1.getValueType() == VT && "Binary operator types must match!"); 5264 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { 5265 APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue(); 5266 APInt N2CImm = N2C->getAPIntValue(); 5267 return getVScale(DL, VT, MulImm * N2CImm); 5268 } 5269 break; 5270 case ISD::UDIV: 5271 case ISD::UREM: 5272 case ISD::MULHU: 5273 case ISD::MULHS: 5274 case ISD::SDIV: 5275 case ISD::SREM: 5276 case ISD::SADDSAT: 5277 case ISD::SSUBSAT: 5278 case ISD::UADDSAT: 5279 case ISD::USUBSAT: 5280 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5281 assert(N1.getValueType() == N2.getValueType() && 5282 N1.getValueType() == VT && "Binary operator types must match!"); 5283 break; 5284 case ISD::SMIN: 5285 case ISD::UMAX: 5286 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5287 assert(N1.getValueType() == N2.getValueType() && 5288 N1.getValueType() == VT && "Binary operator types must match!"); 5289 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) 5290 return getNode(ISD::OR, DL, VT, N1, N2); 5291 break; 5292 case ISD::SMAX: 5293 case ISD::UMIN: 5294 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5295 assert(N1.getValueType() == N2.getValueType() && 5296 N1.getValueType() == VT && "Binary operator types must match!"); 5297 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) 5298 return getNode(ISD::AND, DL, VT, N1, N2); 5299 break; 5300 case ISD::FADD: 5301 case ISD::FSUB: 5302 case ISD::FMUL: 5303 case ISD::FDIV: 5304 case ISD::FREM: 5305 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 5306 assert(N1.getValueType() == N2.getValueType() && 5307 N1.getValueType() == VT && "Binary operator types must match!"); 5308 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags)) 5309 return V; 5310 break; 5311 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 5312 assert(N1.getValueType() == VT && 5313 N1.getValueType().isFloatingPoint() && 5314 N2.getValueType().isFloatingPoint() && 5315 "Invalid FCOPYSIGN!"); 5316 break; 5317 case ISD::SHL: 5318 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { 5319 APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue(); 5320 APInt ShiftImm = N2C->getAPIntValue(); 5321 return getVScale(DL, VT, MulImm << ShiftImm); 5322 } 5323 LLVM_FALLTHROUGH; 5324 case ISD::SRA: 5325 case ISD::SRL: 5326 if (SDValue V = simplifyShift(N1, N2)) 5327 return V; 5328 LLVM_FALLTHROUGH; 5329 case ISD::ROTL: 5330 case ISD::ROTR: 5331 assert(VT == N1.getValueType() && 5332 "Shift operators return type must be the same as their first arg"); 5333 assert(VT.isInteger() && N2.getValueType().isInteger() && 5334 "Shifts only work on integers"); 5335 assert((!VT.isVector() || VT == N2.getValueType()) && 5336 "Vector shift amounts must be in the same as their first arg"); 5337 // Verify that the shift amount VT is big enough to hold valid shift 5338 // amounts. This catches things like trying to shift an i1024 value by an 5339 // i8, which is easy to fall into in generic code that uses 5340 // TLI.getShiftAmount(). 5341 assert(N2.getValueType().getScalarSizeInBits() >= 5342 Log2_32_Ceil(VT.getScalarSizeInBits()) && 5343 "Invalid use of small shift amount with oversized value!"); 5344 5345 // Always fold shifts of i1 values so the code generator doesn't need to 5346 // handle them. Since we know the size of the shift has to be less than the 5347 // size of the value, the shift/rotate count is guaranteed to be zero. 5348 if (VT == MVT::i1) 5349 return N1; 5350 if (N2C && N2C->isNullValue()) 5351 return N1; 5352 break; 5353 case ISD::FP_ROUND: 5354 assert(VT.isFloatingPoint() && 5355 N1.getValueType().isFloatingPoint() && 5356 VT.bitsLE(N1.getValueType()) && 5357 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && 5358 "Invalid FP_ROUND!"); 5359 if (N1.getValueType() == VT) return N1; // noop conversion. 5360 break; 5361 case ISD::AssertSext: 5362 case ISD::AssertZext: { 5363 EVT EVT = cast<VTSDNode>(N2)->getVT(); 5364 assert(VT == N1.getValueType() && "Not an inreg extend!"); 5365 assert(VT.isInteger() && EVT.isInteger() && 5366 "Cannot *_EXTEND_INREG FP types"); 5367 assert(!EVT.isVector() && 5368 "AssertSExt/AssertZExt type should be the vector element type " 5369 "rather than the vector type!"); 5370 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!"); 5371 if (VT.getScalarType() == EVT) return N1; // noop assertion. 5372 break; 5373 } 5374 case ISD::SIGN_EXTEND_INREG: { 5375 EVT EVT = cast<VTSDNode>(N2)->getVT(); 5376 assert(VT == N1.getValueType() && "Not an inreg extend!"); 5377 assert(VT.isInteger() && EVT.isInteger() && 5378 "Cannot *_EXTEND_INREG FP types"); 5379 assert(EVT.isVector() == VT.isVector() && 5380 "SIGN_EXTEND_INREG type should be vector iff the operand " 5381 "type is vector!"); 5382 assert((!EVT.isVector() || 5383 EVT.getVectorElementCount() == VT.getVectorElementCount()) && 5384 "Vector element counts must match in SIGN_EXTEND_INREG"); 5385 assert(EVT.bitsLE(VT) && "Not extending!"); 5386 if (EVT == VT) return N1; // Not actually extending 5387 5388 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { 5389 unsigned FromBits = EVT.getScalarSizeInBits(); 5390 Val <<= Val.getBitWidth() - FromBits; 5391 Val.ashrInPlace(Val.getBitWidth() - FromBits); 5392 return getConstant(Val, DL, ConstantVT); 5393 }; 5394 5395 if (N1C) { 5396 const APInt &Val = N1C->getAPIntValue(); 5397 return SignExtendInReg(Val, VT); 5398 } 5399 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { 5400 SmallVector<SDValue, 8> Ops; 5401 llvm::EVT OpVT = N1.getOperand(0).getValueType(); 5402 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 5403 SDValue Op = N1.getOperand(i); 5404 if (Op.isUndef()) { 5405 Ops.push_back(getUNDEF(OpVT)); 5406 continue; 5407 } 5408 ConstantSDNode *C = cast<ConstantSDNode>(Op); 5409 APInt Val = C->getAPIntValue(); 5410 Ops.push_back(SignExtendInReg(Val, OpVT)); 5411 } 5412 return getBuildVector(VT, DL, Ops); 5413 } 5414 break; 5415 } 5416 case ISD::EXTRACT_VECTOR_ELT: 5417 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && 5418 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ 5419 element type of the vector."); 5420 5421 // Extract from an undefined value or using an undefined index is undefined. 5422 if (N1.isUndef() || N2.isUndef()) 5423 return getUNDEF(VT); 5424 5425 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length 5426 // vectors. For scalable vectors we will provide appropriate support for 5427 // dealing with arbitrary indices. 5428 if (N2C && N1.getValueType().isFixedLengthVector() && 5429 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements())) 5430 return getUNDEF(VT); 5431 5432 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 5433 // expanding copies of large vectors from registers. This only works for 5434 // fixed length vectors, since we need to know the exact number of 5435 // elements. 5436 if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() && 5437 N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { 5438 unsigned Factor = 5439 N1.getOperand(0).getValueType().getVectorNumElements(); 5440 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 5441 N1.getOperand(N2C->getZExtValue() / Factor), 5442 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL)); 5443 } 5444 5445 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while 5446 // lowering is expanding large vector constants. 5447 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || 5448 N1.getOpcode() == ISD::SPLAT_VECTOR)) { 5449 assert((N1.getOpcode() != ISD::BUILD_VECTOR || 5450 N1.getValueType().isFixedLengthVector()) && 5451 "BUILD_VECTOR used for scalable vectors"); 5452 unsigned Index = 5453 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; 5454 SDValue Elt = N1.getOperand(Index); 5455 5456 if (VT != Elt.getValueType()) 5457 // If the vector element type is not legal, the BUILD_VECTOR operands 5458 // are promoted and implicitly truncated, and the result implicitly 5459 // extended. Make that explicit here. 5460 Elt = getAnyExtOrTrunc(Elt, DL, VT); 5461 5462 return Elt; 5463 } 5464 5465 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 5466 // operations are lowered to scalars. 5467 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 5468 // If the indices are the same, return the inserted element else 5469 // if the indices are known different, extract the element from 5470 // the original vector. 5471 SDValue N1Op2 = N1.getOperand(2); 5472 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2); 5473 5474 if (N1Op2C && N2C) { 5475 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 5476 if (VT == N1.getOperand(1).getValueType()) 5477 return N1.getOperand(1); 5478 else 5479 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 5480 } 5481 5482 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 5483 } 5484 } 5485 5486 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed 5487 // when vector types are scalarized and v1iX is legal. 5488 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). 5489 // Here we are completely ignoring the extract element index (N2), 5490 // which is fine for fixed width vectors, since any index other than 0 5491 // is undefined anyway. However, this cannot be ignored for scalable 5492 // vectors - in theory we could support this, but we don't want to do this 5493 // without a profitability check. 5494 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && 5495 N1.getValueType().isFixedLengthVector() && 5496 N1.getValueType().getVectorNumElements() == 1) { 5497 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), 5498 N1.getOperand(1)); 5499 } 5500 break; 5501 case ISD::EXTRACT_ELEMENT: 5502 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 5503 assert(!N1.getValueType().isVector() && !VT.isVector() && 5504 (N1.getValueType().isInteger() == VT.isInteger()) && 5505 N1.getValueType() != VT && 5506 "Wrong types for EXTRACT_ELEMENT!"); 5507 5508 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 5509 // 64-bit integers into 32-bit parts. Instead of building the extract of 5510 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 5511 if (N1.getOpcode() == ISD::BUILD_PAIR) 5512 return N1.getOperand(N2C->getZExtValue()); 5513 5514 // EXTRACT_ELEMENT of a constant int is also very common. 5515 if (N1C) { 5516 unsigned ElementSize = VT.getSizeInBits(); 5517 unsigned Shift = ElementSize * N2C->getZExtValue(); 5518 APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift); 5519 return getConstant(ShiftedVal.trunc(ElementSize), DL, VT); 5520 } 5521 break; 5522 case ISD::EXTRACT_SUBVECTOR: 5523 EVT N1VT = N1.getValueType(); 5524 assert(VT.isVector() && N1VT.isVector() && 5525 "Extract subvector VTs must be vectors!"); 5526 assert(VT.getVectorElementType() == N1VT.getVectorElementType() && 5527 "Extract subvector VTs must have the same element type!"); 5528 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) && 5529 "Cannot extract a scalable vector from a fixed length vector!"); 5530 assert((VT.isScalableVector() != N1VT.isScalableVector() || 5531 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && 5532 "Extract subvector must be from larger vector to smaller vector!"); 5533 assert(N2C && "Extract subvector index must be a constant"); 5534 assert((VT.isScalableVector() != N1VT.isScalableVector() || 5535 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= 5536 N1VT.getVectorMinNumElements()) && 5537 "Extract subvector overflow!"); 5538 assert(N2C->getAPIntValue().getBitWidth() == 5539 TLI->getVectorIdxTy(getDataLayout()) 5540 .getSizeInBits() 5541 .getFixedSize() && 5542 "Constant index for EXTRACT_SUBVECTOR has an invalid size"); 5543 5544 // Trivial extraction. 5545 if (VT == N1VT) 5546 return N1; 5547 5548 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. 5549 if (N1.isUndef()) 5550 return getUNDEF(VT); 5551 5552 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of 5553 // the concat have the same type as the extract. 5554 if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && 5555 VT == N1.getOperand(0).getValueType()) { 5556 unsigned Factor = VT.getVectorMinNumElements(); 5557 return N1.getOperand(N2C->getZExtValue() / Factor); 5558 } 5559 5560 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created 5561 // during shuffle legalization. 5562 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && 5563 VT == N1.getOperand(1).getValueType()) 5564 return N1.getOperand(1); 5565 break; 5566 } 5567 5568 // Perform trivial constant folding. 5569 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2})) 5570 return SV; 5571 5572 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2)) 5573 return V; 5574 5575 // Canonicalize an UNDEF to the RHS, even over a constant. 5576 if (N1.isUndef()) { 5577 if (TLI->isCommutativeBinOp(Opcode)) { 5578 std::swap(N1, N2); 5579 } else { 5580 switch (Opcode) { 5581 case ISD::SIGN_EXTEND_INREG: 5582 case ISD::SUB: 5583 return getUNDEF(VT); // fold op(undef, arg2) -> undef 5584 case ISD::UDIV: 5585 case ISD::SDIV: 5586 case ISD::UREM: 5587 case ISD::SREM: 5588 case ISD::SSUBSAT: 5589 case ISD::USUBSAT: 5590 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 5591 } 5592 } 5593 } 5594 5595 // Fold a bunch of operators when the RHS is undef. 5596 if (N2.isUndef()) { 5597 switch (Opcode) { 5598 case ISD::XOR: 5599 if (N1.isUndef()) 5600 // Handle undef ^ undef -> 0 special case. This is a common 5601 // idiom (misuse). 5602 return getConstant(0, DL, VT); 5603 LLVM_FALLTHROUGH; 5604 case ISD::ADD: 5605 case ISD::SUB: 5606 case ISD::UDIV: 5607 case ISD::SDIV: 5608 case ISD::UREM: 5609 case ISD::SREM: 5610 return getUNDEF(VT); // fold op(arg1, undef) -> undef 5611 case ISD::MUL: 5612 case ISD::AND: 5613 case ISD::SSUBSAT: 5614 case ISD::USUBSAT: 5615 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 5616 case ISD::OR: 5617 case ISD::SADDSAT: 5618 case ISD::UADDSAT: 5619 return getAllOnesConstant(DL, VT); 5620 } 5621 } 5622 5623 // Memoize this node if possible. 5624 SDNode *N; 5625 SDVTList VTs = getVTList(VT); 5626 SDValue Ops[] = {N1, N2}; 5627 if (VT != MVT::Glue) { 5628 FoldingSetNodeID ID; 5629 AddNodeIDNode(ID, Opcode, VTs, Ops); 5630 void *IP = nullptr; 5631 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 5632 E->intersectFlagsWith(Flags); 5633 return SDValue(E, 0); 5634 } 5635 5636 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5637 N->setFlags(Flags); 5638 createOperands(N, Ops); 5639 CSEMap.InsertNode(N, IP); 5640 } else { 5641 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5642 createOperands(N, Ops); 5643 } 5644 5645 InsertNode(N); 5646 SDValue V = SDValue(N, 0); 5647 NewSDValueDbgMsg(V, "Creating new node: ", this); 5648 return V; 5649 } 5650 5651 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5652 SDValue N1, SDValue N2, SDValue N3) { 5653 SDNodeFlags Flags; 5654 if (Inserter) 5655 Flags = Inserter->getFlags(); 5656 return getNode(Opcode, DL, VT, N1, N2, N3, Flags); 5657 } 5658 5659 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5660 SDValue N1, SDValue N2, SDValue N3, 5661 const SDNodeFlags Flags) { 5662 // Perform various simplifications. 5663 switch (Opcode) { 5664 case ISD::FMA: { 5665 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 5666 assert(N1.getValueType() == VT && N2.getValueType() == VT && 5667 N3.getValueType() == VT && "FMA types must match!"); 5668 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 5669 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 5670 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 5671 if (N1CFP && N2CFP && N3CFP) { 5672 APFloat V1 = N1CFP->getValueAPF(); 5673 const APFloat &V2 = N2CFP->getValueAPF(); 5674 const APFloat &V3 = N3CFP->getValueAPF(); 5675 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 5676 return getConstantFP(V1, DL, VT); 5677 } 5678 break; 5679 } 5680 case ISD::BUILD_VECTOR: { 5681 // Attempt to simplify BUILD_VECTOR. 5682 SDValue Ops[] = {N1, N2, N3}; 5683 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 5684 return V; 5685 break; 5686 } 5687 case ISD::CONCAT_VECTORS: { 5688 SDValue Ops[] = {N1, N2, N3}; 5689 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 5690 return V; 5691 break; 5692 } 5693 case ISD::SETCC: { 5694 assert(VT.isInteger() && "SETCC result type must be an integer!"); 5695 assert(N1.getValueType() == N2.getValueType() && 5696 "SETCC operands must have the same type!"); 5697 assert(VT.isVector() == N1.getValueType().isVector() && 5698 "SETCC type should be vector iff the operand type is vector!"); 5699 assert((!VT.isVector() || VT.getVectorElementCount() == 5700 N1.getValueType().getVectorElementCount()) && 5701 "SETCC vector element counts must match!"); 5702 // Use FoldSetCC to simplify SETCC's. 5703 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL)) 5704 return V; 5705 // Vector constant folding. 5706 SDValue Ops[] = {N1, N2, N3}; 5707 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) { 5708 NewSDValueDbgMsg(V, "New node vector constant folding: ", this); 5709 return V; 5710 } 5711 break; 5712 } 5713 case ISD::SELECT: 5714 case ISD::VSELECT: 5715 if (SDValue V = simplifySelect(N1, N2, N3)) 5716 return V; 5717 break; 5718 case ISD::VECTOR_SHUFFLE: 5719 llvm_unreachable("should use getVectorShuffle constructor!"); 5720 case ISD::INSERT_VECTOR_ELT: { 5721 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3); 5722 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except 5723 // for scalable vectors where we will generate appropriate code to 5724 // deal with out-of-bounds cases correctly. 5725 if (N3C && N1.getValueType().isFixedLengthVector() && 5726 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) 5727 return getUNDEF(VT); 5728 5729 // Undefined index can be assumed out-of-bounds, so that's UNDEF too. 5730 if (N3.isUndef()) 5731 return getUNDEF(VT); 5732 5733 // If the inserted element is an UNDEF, just use the input vector. 5734 if (N2.isUndef()) 5735 return N1; 5736 5737 break; 5738 } 5739 case ISD::INSERT_SUBVECTOR: { 5740 // Inserting undef into undef is still undef. 5741 if (N1.isUndef() && N2.isUndef()) 5742 return getUNDEF(VT); 5743 5744 EVT N2VT = N2.getValueType(); 5745 assert(VT == N1.getValueType() && 5746 "Dest and insert subvector source types must match!"); 5747 assert(VT.isVector() && N2VT.isVector() && 5748 "Insert subvector VTs must be vectors!"); 5749 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) && 5750 "Cannot insert a scalable vector into a fixed length vector!"); 5751 assert((VT.isScalableVector() != N2VT.isScalableVector() || 5752 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && 5753 "Insert subvector must be from smaller vector to larger vector!"); 5754 assert(isa<ConstantSDNode>(N3) && 5755 "Insert subvector index must be constant"); 5756 assert((VT.isScalableVector() != N2VT.isScalableVector() || 5757 (N2VT.getVectorMinNumElements() + 5758 cast<ConstantSDNode>(N3)->getZExtValue()) <= 5759 VT.getVectorMinNumElements()) && 5760 "Insert subvector overflow!"); 5761 5762 // Trivial insertion. 5763 if (VT == N2VT) 5764 return N2; 5765 5766 // If this is an insert of an extracted vector into an undef vector, we 5767 // can just use the input to the extract. 5768 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && 5769 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) 5770 return N2.getOperand(0); 5771 break; 5772 } 5773 case ISD::BITCAST: 5774 // Fold bit_convert nodes from a type to themselves. 5775 if (N1.getValueType() == VT) 5776 return N1; 5777 break; 5778 } 5779 5780 // Memoize node if it doesn't produce a flag. 5781 SDNode *N; 5782 SDVTList VTs = getVTList(VT); 5783 SDValue Ops[] = {N1, N2, N3}; 5784 if (VT != MVT::Glue) { 5785 FoldingSetNodeID ID; 5786 AddNodeIDNode(ID, Opcode, VTs, Ops); 5787 void *IP = nullptr; 5788 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 5789 E->intersectFlagsWith(Flags); 5790 return SDValue(E, 0); 5791 } 5792 5793 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5794 N->setFlags(Flags); 5795 createOperands(N, Ops); 5796 CSEMap.InsertNode(N, IP); 5797 } else { 5798 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5799 createOperands(N, Ops); 5800 } 5801 5802 InsertNode(N); 5803 SDValue V = SDValue(N, 0); 5804 NewSDValueDbgMsg(V, "Creating new node: ", this); 5805 return V; 5806 } 5807 5808 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5809 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 5810 SDValue Ops[] = { N1, N2, N3, N4 }; 5811 return getNode(Opcode, DL, VT, Ops); 5812 } 5813 5814 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5815 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 5816 SDValue N5) { 5817 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 5818 return getNode(Opcode, DL, VT, Ops); 5819 } 5820 5821 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all 5822 /// the incoming stack arguments to be loaded from the stack. 5823 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 5824 SmallVector<SDValue, 8> ArgChains; 5825 5826 // Include the original chain at the beginning of the list. When this is 5827 // used by target LowerCall hooks, this helps legalize find the 5828 // CALLSEQ_BEGIN node. 5829 ArgChains.push_back(Chain); 5830 5831 // Add a chain value for each stack argument. 5832 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 5833 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 5834 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 5835 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 5836 if (FI->getIndex() < 0) 5837 ArgChains.push_back(SDValue(L, 1)); 5838 5839 // Build a tokenfactor for all the chains. 5840 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); 5841 } 5842 5843 /// getMemsetValue - Vectorized representation of the memset value 5844 /// operand. 5845 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 5846 const SDLoc &dl) { 5847 assert(!Value.isUndef()); 5848 5849 unsigned NumBits = VT.getScalarSizeInBits(); 5850 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 5851 assert(C->getAPIntValue().getBitWidth() == 8); 5852 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 5853 if (VT.isInteger()) { 5854 bool IsOpaque = VT.getSizeInBits() > 64 || 5855 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue()); 5856 return DAG.getConstant(Val, dl, VT, false, IsOpaque); 5857 } 5858 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, 5859 VT); 5860 } 5861 5862 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); 5863 EVT IntVT = VT.getScalarType(); 5864 if (!IntVT.isInteger()) 5865 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); 5866 5867 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); 5868 if (NumBits > 8) { 5869 // Use a multiplication with 0x010101... to extend the input to the 5870 // required length. 5871 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 5872 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, 5873 DAG.getConstant(Magic, dl, IntVT)); 5874 } 5875 5876 if (VT != Value.getValueType() && !VT.isInteger()) 5877 Value = DAG.getBitcast(VT.getScalarType(), Value); 5878 if (VT != Value.getValueType()) 5879 Value = DAG.getSplatBuildVector(VT, dl, Value); 5880 5881 return Value; 5882 } 5883 5884 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only 5885 /// used when a memcpy is turned into a memset when the source is a constant 5886 /// string ptr. 5887 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, 5888 const TargetLowering &TLI, 5889 const ConstantDataArraySlice &Slice) { 5890 // Handle vector with all elements zero. 5891 if (Slice.Array == nullptr) { 5892 if (VT.isInteger()) 5893 return DAG.getConstant(0, dl, VT); 5894 else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) 5895 return DAG.getConstantFP(0.0, dl, VT); 5896 else if (VT.isVector()) { 5897 unsigned NumElts = VT.getVectorNumElements(); 5898 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 5899 return DAG.getNode(ISD::BITCAST, dl, VT, 5900 DAG.getConstant(0, dl, 5901 EVT::getVectorVT(*DAG.getContext(), 5902 EltVT, NumElts))); 5903 } else 5904 llvm_unreachable("Expected type!"); 5905 } 5906 5907 assert(!VT.isVector() && "Can't handle vector type here!"); 5908 unsigned NumVTBits = VT.getSizeInBits(); 5909 unsigned NumVTBytes = NumVTBits / 8; 5910 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); 5911 5912 APInt Val(NumVTBits, 0); 5913 if (DAG.getDataLayout().isLittleEndian()) { 5914 for (unsigned i = 0; i != NumBytes; ++i) 5915 Val |= (uint64_t)(unsigned char)Slice[i] << i*8; 5916 } else { 5917 for (unsigned i = 0; i != NumBytes; ++i) 5918 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; 5919 } 5920 5921 // If the "cost" of materializing the integer immediate is less than the cost 5922 // of a load, then it is cost effective to turn the load into the immediate. 5923 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 5924 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) 5925 return DAG.getConstant(Val, dl, VT); 5926 return SDValue(nullptr, 0); 5927 } 5928 5929 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, 5930 const SDLoc &DL, 5931 const SDNodeFlags Flags) { 5932 EVT VT = Base.getValueType(); 5933 SDValue Index; 5934 5935 if (Offset.isScalable()) 5936 Index = getVScale(DL, Base.getValueType(), 5937 APInt(Base.getValueSizeInBits().getFixedSize(), 5938 Offset.getKnownMinSize())); 5939 else 5940 Index = getConstant(Offset.getFixedSize(), DL, VT); 5941 5942 return getMemBasePlusOffset(Base, Index, DL, Flags); 5943 } 5944 5945 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, 5946 const SDLoc &DL, 5947 const SDNodeFlags Flags) { 5948 assert(Offset.getValueType().isInteger()); 5949 EVT BasePtrVT = Ptr.getValueType(); 5950 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); 5951 } 5952 5953 /// Returns true if memcpy source is constant data. 5954 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { 5955 uint64_t SrcDelta = 0; 5956 GlobalAddressSDNode *G = nullptr; 5957 if (Src.getOpcode() == ISD::GlobalAddress) 5958 G = cast<GlobalAddressSDNode>(Src); 5959 else if (Src.getOpcode() == ISD::ADD && 5960 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 5961 Src.getOperand(1).getOpcode() == ISD::Constant) { 5962 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 5963 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 5964 } 5965 if (!G) 5966 return false; 5967 5968 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, 5969 SrcDelta + G->getOffset()); 5970 } 5971 5972 static bool shouldLowerMemFuncForSize(const MachineFunction &MF, 5973 SelectionDAG &DAG) { 5974 // On Darwin, -Os means optimize for size without hurting performance, so 5975 // only really optimize for size when -Oz (MinSize) is used. 5976 if (MF.getTarget().getTargetTriple().isOSDarwin()) 5977 return MF.getFunction().hasMinSize(); 5978 return DAG.shouldOptForSize(); 5979 } 5980 5981 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, 5982 SmallVector<SDValue, 32> &OutChains, unsigned From, 5983 unsigned To, SmallVector<SDValue, 16> &OutLoadChains, 5984 SmallVector<SDValue, 16> &OutStoreChains) { 5985 assert(OutLoadChains.size() && "Missing loads in memcpy inlining"); 5986 assert(OutStoreChains.size() && "Missing stores in memcpy inlining"); 5987 SmallVector<SDValue, 16> GluedLoadChains; 5988 for (unsigned i = From; i < To; ++i) { 5989 OutChains.push_back(OutLoadChains[i]); 5990 GluedLoadChains.push_back(OutLoadChains[i]); 5991 } 5992 5993 // Chain for all loads. 5994 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 5995 GluedLoadChains); 5996 5997 for (unsigned i = From; i < To; ++i) { 5998 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]); 5999 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(), 6000 ST->getBasePtr(), ST->getMemoryVT(), 6001 ST->getMemOperand()); 6002 OutChains.push_back(NewStore); 6003 } 6004 } 6005 6006 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 6007 SDValue Chain, SDValue Dst, SDValue Src, 6008 uint64_t Size, Align Alignment, 6009 bool isVol, bool AlwaysInline, 6010 MachinePointerInfo DstPtrInfo, 6011 MachinePointerInfo SrcPtrInfo) { 6012 // Turn a memcpy of undef to nop. 6013 // FIXME: We need to honor volatile even is Src is undef. 6014 if (Src.isUndef()) 6015 return Chain; 6016 6017 // Expand memcpy to a series of load and store ops if the size operand falls 6018 // below a certain threshold. 6019 // TODO: In the AlwaysInline case, if the size is big then generate a loop 6020 // rather than maybe a humongous number of loads and stores. 6021 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6022 const DataLayout &DL = DAG.getDataLayout(); 6023 LLVMContext &C = *DAG.getContext(); 6024 std::vector<EVT> MemOps; 6025 bool DstAlignCanChange = false; 6026 MachineFunction &MF = DAG.getMachineFunction(); 6027 MachineFrameInfo &MFI = MF.getFrameInfo(); 6028 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6029 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6030 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6031 DstAlignCanChange = true; 6032 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); 6033 if (!SrcAlign || Alignment > *SrcAlign) 6034 SrcAlign = Alignment; 6035 assert(SrcAlign && "SrcAlign must be set"); 6036 ConstantDataArraySlice Slice; 6037 // If marked as volatile, perform a copy even when marked as constant. 6038 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); 6039 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; 6040 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 6041 const MemOp Op = isZeroConstant 6042 ? MemOp::Set(Size, DstAlignCanChange, Alignment, 6043 /*IsZeroMemset*/ true, isVol) 6044 : MemOp::Copy(Size, DstAlignCanChange, Alignment, 6045 *SrcAlign, isVol, CopyFromConstant); 6046 if (!TLI.findOptimalMemOpLowering( 6047 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(), 6048 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes())) 6049 return SDValue(); 6050 6051 if (DstAlignCanChange) { 6052 Type *Ty = MemOps[0].getTypeForEVT(C); 6053 Align NewAlign = DL.getABITypeAlign(Ty); 6054 6055 // Don't promote to an alignment that would require dynamic stack 6056 // realignment. 6057 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 6058 if (!TRI->needsStackRealignment(MF)) 6059 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) 6060 NewAlign = NewAlign / 2; 6061 6062 if (NewAlign > Alignment) { 6063 // Give the stack frame object a larger alignment if needed. 6064 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6065 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6066 Alignment = NewAlign; 6067 } 6068 } 6069 6070 MachineMemOperand::Flags MMOFlags = 6071 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 6072 SmallVector<SDValue, 16> OutLoadChains; 6073 SmallVector<SDValue, 16> OutStoreChains; 6074 SmallVector<SDValue, 32> OutChains; 6075 unsigned NumMemOps = MemOps.size(); 6076 uint64_t SrcOff = 0, DstOff = 0; 6077 for (unsigned i = 0; i != NumMemOps; ++i) { 6078 EVT VT = MemOps[i]; 6079 unsigned VTSize = VT.getSizeInBits() / 8; 6080 SDValue Value, Store; 6081 6082 if (VTSize > Size) { 6083 // Issuing an unaligned load / store pair that overlaps with the previous 6084 // pair. Adjust the offset accordingly. 6085 assert(i == NumMemOps-1 && i != 0); 6086 SrcOff -= VTSize - Size; 6087 DstOff -= VTSize - Size; 6088 } 6089 6090 if (CopyFromConstant && 6091 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) { 6092 // It's unlikely a store of a vector immediate can be done in a single 6093 // instruction. It would require a load from a constantpool first. 6094 // We only handle zero vectors here. 6095 // FIXME: Handle other cases where store of vector immediate is done in 6096 // a single instruction. 6097 ConstantDataArraySlice SubSlice; 6098 if (SrcOff < Slice.Length) { 6099 SubSlice = Slice; 6100 SubSlice.move(SrcOff); 6101 } else { 6102 // This is an out-of-bounds access and hence UB. Pretend we read zero. 6103 SubSlice.Array = nullptr; 6104 SubSlice.Offset = 0; 6105 SubSlice.Length = VTSize; 6106 } 6107 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); 6108 if (Value.getNode()) { 6109 Store = DAG.getStore( 6110 Chain, dl, Value, 6111 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6112 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); 6113 OutChains.push_back(Store); 6114 } 6115 } 6116 6117 if (!Store.getNode()) { 6118 // The type might not be legal for the target. This should only happen 6119 // if the type is smaller than a legal type, as on PPC, so the right 6120 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 6121 // to Load/Store if NVT==VT. 6122 // FIXME does the case above also need this? 6123 EVT NVT = TLI.getTypeToTransformTo(C, VT); 6124 assert(NVT.bitsGE(VT)); 6125 6126 bool isDereferenceable = 6127 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 6128 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 6129 if (isDereferenceable) 6130 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 6131 6132 Value = DAG.getExtLoad( 6133 ISD::EXTLOAD, dl, NVT, Chain, 6134 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), 6135 SrcPtrInfo.getWithOffset(SrcOff), VT, 6136 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); 6137 OutLoadChains.push_back(Value.getValue(1)); 6138 6139 Store = DAG.getTruncStore( 6140 Chain, dl, Value, 6141 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6142 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); 6143 OutStoreChains.push_back(Store); 6144 } 6145 SrcOff += VTSize; 6146 DstOff += VTSize; 6147 Size -= VTSize; 6148 } 6149 6150 unsigned GluedLdStLimit = MaxLdStGlue == 0 ? 6151 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; 6152 unsigned NumLdStInMemcpy = OutStoreChains.size(); 6153 6154 if (NumLdStInMemcpy) { 6155 // It may be that memcpy might be converted to memset if it's memcpy 6156 // of constants. In such a case, we won't have loads and stores, but 6157 // just stores. In the absence of loads, there is nothing to gang up. 6158 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { 6159 // If target does not care, just leave as it. 6160 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { 6161 OutChains.push_back(OutLoadChains[i]); 6162 OutChains.push_back(OutStoreChains[i]); 6163 } 6164 } else { 6165 // Ld/St less than/equal limit set by target. 6166 if (NumLdStInMemcpy <= GluedLdStLimit) { 6167 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 6168 NumLdStInMemcpy, OutLoadChains, 6169 OutStoreChains); 6170 } else { 6171 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; 6172 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; 6173 unsigned GlueIter = 0; 6174 6175 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { 6176 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; 6177 unsigned IndexTo = NumLdStInMemcpy - GlueIter; 6178 6179 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo, 6180 OutLoadChains, OutStoreChains); 6181 GlueIter += GluedLdStLimit; 6182 } 6183 6184 // Residual ld/st. 6185 if (RemainingLdStInMemcpy) { 6186 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 6187 RemainingLdStInMemcpy, OutLoadChains, 6188 OutStoreChains); 6189 } 6190 } 6191 } 6192 } 6193 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6194 } 6195 6196 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 6197 SDValue Chain, SDValue Dst, SDValue Src, 6198 uint64_t Size, Align Alignment, 6199 bool isVol, bool AlwaysInline, 6200 MachinePointerInfo DstPtrInfo, 6201 MachinePointerInfo SrcPtrInfo) { 6202 // Turn a memmove of undef to nop. 6203 // FIXME: We need to honor volatile even is Src is undef. 6204 if (Src.isUndef()) 6205 return Chain; 6206 6207 // Expand memmove to a series of load and store ops if the size operand falls 6208 // below a certain threshold. 6209 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6210 const DataLayout &DL = DAG.getDataLayout(); 6211 LLVMContext &C = *DAG.getContext(); 6212 std::vector<EVT> MemOps; 6213 bool DstAlignCanChange = false; 6214 MachineFunction &MF = DAG.getMachineFunction(); 6215 MachineFrameInfo &MFI = MF.getFrameInfo(); 6216 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6217 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6218 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6219 DstAlignCanChange = true; 6220 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); 6221 if (!SrcAlign || Alignment > *SrcAlign) 6222 SrcAlign = Alignment; 6223 assert(SrcAlign && "SrcAlign must be set"); 6224 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 6225 if (!TLI.findOptimalMemOpLowering( 6226 MemOps, Limit, 6227 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, 6228 /*IsVolatile*/ true), 6229 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), 6230 MF.getFunction().getAttributes())) 6231 return SDValue(); 6232 6233 if (DstAlignCanChange) { 6234 Type *Ty = MemOps[0].getTypeForEVT(C); 6235 Align NewAlign = DL.getABITypeAlign(Ty); 6236 if (NewAlign > Alignment) { 6237 // Give the stack frame object a larger alignment if needed. 6238 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6239 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6240 Alignment = NewAlign; 6241 } 6242 } 6243 6244 MachineMemOperand::Flags MMOFlags = 6245 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 6246 uint64_t SrcOff = 0, DstOff = 0; 6247 SmallVector<SDValue, 8> LoadValues; 6248 SmallVector<SDValue, 8> LoadChains; 6249 SmallVector<SDValue, 8> OutChains; 6250 unsigned NumMemOps = MemOps.size(); 6251 for (unsigned i = 0; i < NumMemOps; i++) { 6252 EVT VT = MemOps[i]; 6253 unsigned VTSize = VT.getSizeInBits() / 8; 6254 SDValue Value; 6255 6256 bool isDereferenceable = 6257 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 6258 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 6259 if (isDereferenceable) 6260 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 6261 6262 Value = 6263 DAG.getLoad(VT, dl, Chain, 6264 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), 6265 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); 6266 LoadValues.push_back(Value); 6267 LoadChains.push_back(Value.getValue(1)); 6268 SrcOff += VTSize; 6269 } 6270 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 6271 OutChains.clear(); 6272 for (unsigned i = 0; i < NumMemOps; i++) { 6273 EVT VT = MemOps[i]; 6274 unsigned VTSize = VT.getSizeInBits() / 8; 6275 SDValue Store; 6276 6277 Store = 6278 DAG.getStore(Chain, dl, LoadValues[i], 6279 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6280 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); 6281 OutChains.push_back(Store); 6282 DstOff += VTSize; 6283 } 6284 6285 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6286 } 6287 6288 /// Lower the call to 'memset' intrinsic function into a series of store 6289 /// operations. 6290 /// 6291 /// \param DAG Selection DAG where lowered code is placed. 6292 /// \param dl Link to corresponding IR location. 6293 /// \param Chain Control flow dependency. 6294 /// \param Dst Pointer to destination memory location. 6295 /// \param Src Value of byte to write into the memory. 6296 /// \param Size Number of bytes to write. 6297 /// \param Alignment Alignment of the destination in bytes. 6298 /// \param isVol True if destination is volatile. 6299 /// \param DstPtrInfo IR information on the memory pointer. 6300 /// \returns New head in the control flow, if lowering was successful, empty 6301 /// SDValue otherwise. 6302 /// 6303 /// The function tries to replace 'llvm.memset' intrinsic with several store 6304 /// operations and value calculation code. This is usually profitable for small 6305 /// memory size. 6306 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, 6307 SDValue Chain, SDValue Dst, SDValue Src, 6308 uint64_t Size, Align Alignment, bool isVol, 6309 MachinePointerInfo DstPtrInfo) { 6310 // Turn a memset of undef to nop. 6311 // FIXME: We need to honor volatile even is Src is undef. 6312 if (Src.isUndef()) 6313 return Chain; 6314 6315 // Expand memset to a series of load/store ops if the size operand 6316 // falls below a certain threshold. 6317 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6318 std::vector<EVT> MemOps; 6319 bool DstAlignCanChange = false; 6320 MachineFunction &MF = DAG.getMachineFunction(); 6321 MachineFrameInfo &MFI = MF.getFrameInfo(); 6322 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6323 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6324 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6325 DstAlignCanChange = true; 6326 bool IsZeroVal = 6327 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 6328 if (!TLI.findOptimalMemOpLowering( 6329 MemOps, TLI.getMaxStoresPerMemset(OptSize), 6330 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol), 6331 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes())) 6332 return SDValue(); 6333 6334 if (DstAlignCanChange) { 6335 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 6336 Align NewAlign = DAG.getDataLayout().getABITypeAlign(Ty); 6337 if (NewAlign > Alignment) { 6338 // Give the stack frame object a larger alignment if needed. 6339 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6340 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6341 Alignment = NewAlign; 6342 } 6343 } 6344 6345 SmallVector<SDValue, 8> OutChains; 6346 uint64_t DstOff = 0; 6347 unsigned NumMemOps = MemOps.size(); 6348 6349 // Find the largest store and generate the bit pattern for it. 6350 EVT LargestVT = MemOps[0]; 6351 for (unsigned i = 1; i < NumMemOps; i++) 6352 if (MemOps[i].bitsGT(LargestVT)) 6353 LargestVT = MemOps[i]; 6354 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 6355 6356 for (unsigned i = 0; i < NumMemOps; i++) { 6357 EVT VT = MemOps[i]; 6358 unsigned VTSize = VT.getSizeInBits() / 8; 6359 if (VTSize > Size) { 6360 // Issuing an unaligned load / store pair that overlaps with the previous 6361 // pair. Adjust the offset accordingly. 6362 assert(i == NumMemOps-1 && i != 0); 6363 DstOff -= VTSize - Size; 6364 } 6365 6366 // If this store is smaller than the largest store see whether we can get 6367 // the smaller value for free with a truncate. 6368 SDValue Value = MemSetValue; 6369 if (VT.bitsLT(LargestVT)) { 6370 if (!LargestVT.isVector() && !VT.isVector() && 6371 TLI.isTruncateFree(LargestVT, VT)) 6372 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 6373 else 6374 Value = getMemsetValue(Src, VT, DAG, dl); 6375 } 6376 assert(Value.getValueType() == VT && "Value with wrong type."); 6377 SDValue Store = DAG.getStore( 6378 Chain, dl, Value, 6379 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6380 DstPtrInfo.getWithOffset(DstOff), Alignment, 6381 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); 6382 OutChains.push_back(Store); 6383 DstOff += VT.getSizeInBits() / 8; 6384 Size -= VTSize; 6385 } 6386 6387 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6388 } 6389 6390 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, 6391 unsigned AS) { 6392 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all 6393 // pointer operands can be losslessly bitcasted to pointers of address space 0 6394 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { 6395 report_fatal_error("cannot lower memory intrinsic in address space " + 6396 Twine(AS)); 6397 } 6398 } 6399 6400 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 6401 SDValue Src, SDValue Size, Align Alignment, 6402 bool isVol, bool AlwaysInline, bool isTailCall, 6403 MachinePointerInfo DstPtrInfo, 6404 MachinePointerInfo SrcPtrInfo) { 6405 // Check to see if we should lower the memcpy to loads and stores first. 6406 // For cases within the target-specified limits, this is the best choice. 6407 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 6408 if (ConstantSize) { 6409 // Memcpy with size zero? Just return the original chain. 6410 if (ConstantSize->isNullValue()) 6411 return Chain; 6412 6413 SDValue Result = getMemcpyLoadsAndStores( 6414 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, 6415 isVol, false, DstPtrInfo, SrcPtrInfo); 6416 if (Result.getNode()) 6417 return Result; 6418 } 6419 6420 // Then check to see if we should lower the memcpy with target-specific 6421 // code. If the target chooses to do this, this is the next best. 6422 if (TSI) { 6423 SDValue Result = TSI->EmitTargetCodeForMemcpy( 6424 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, 6425 DstPtrInfo, SrcPtrInfo); 6426 if (Result.getNode()) 6427 return Result; 6428 } 6429 6430 // If we really need inline code and the target declined to provide it, 6431 // use a (potentially long) sequence of loads and stores. 6432 if (AlwaysInline) { 6433 assert(ConstantSize && "AlwaysInline requires a constant size!"); 6434 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 6435 ConstantSize->getZExtValue(), Alignment, 6436 isVol, true, DstPtrInfo, SrcPtrInfo); 6437 } 6438 6439 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 6440 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 6441 6442 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 6443 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 6444 // respect volatile, so they may do things like read or write memory 6445 // beyond the given memory regions. But fixing this isn't easy, and most 6446 // people don't care. 6447 6448 // Emit a library call. 6449 TargetLowering::ArgListTy Args; 6450 TargetLowering::ArgListEntry Entry; 6451 Entry.Ty = Type::getInt8PtrTy(*getContext()); 6452 Entry.Node = Dst; Args.push_back(Entry); 6453 Entry.Node = Src; Args.push_back(Entry); 6454 6455 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6456 Entry.Node = Size; Args.push_back(Entry); 6457 // FIXME: pass in SDLoc 6458 TargetLowering::CallLoweringInfo CLI(*this); 6459 CLI.setDebugLoc(dl) 6460 .setChain(Chain) 6461 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), 6462 Dst.getValueType().getTypeForEVT(*getContext()), 6463 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 6464 TLI->getPointerTy(getDataLayout())), 6465 std::move(Args)) 6466 .setDiscardResult() 6467 .setTailCall(isTailCall); 6468 6469 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 6470 return CallResult.second; 6471 } 6472 6473 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, 6474 SDValue Dst, unsigned DstAlign, 6475 SDValue Src, unsigned SrcAlign, 6476 SDValue Size, Type *SizeTy, 6477 unsigned ElemSz, bool isTailCall, 6478 MachinePointerInfo DstPtrInfo, 6479 MachinePointerInfo SrcPtrInfo) { 6480 // Emit a library call. 6481 TargetLowering::ArgListTy Args; 6482 TargetLowering::ArgListEntry Entry; 6483 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6484 Entry.Node = Dst; 6485 Args.push_back(Entry); 6486 6487 Entry.Node = Src; 6488 Args.push_back(Entry); 6489 6490 Entry.Ty = SizeTy; 6491 Entry.Node = Size; 6492 Args.push_back(Entry); 6493 6494 RTLIB::Libcall LibraryCall = 6495 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz); 6496 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 6497 report_fatal_error("Unsupported element size"); 6498 6499 TargetLowering::CallLoweringInfo CLI(*this); 6500 CLI.setDebugLoc(dl) 6501 .setChain(Chain) 6502 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 6503 Type::getVoidTy(*getContext()), 6504 getExternalSymbol(TLI->getLibcallName(LibraryCall), 6505 TLI->getPointerTy(getDataLayout())), 6506 std::move(Args)) 6507 .setDiscardResult() 6508 .setTailCall(isTailCall); 6509 6510 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 6511 return CallResult.second; 6512 } 6513 6514 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 6515 SDValue Src, SDValue Size, Align Alignment, 6516 bool isVol, bool isTailCall, 6517 MachinePointerInfo DstPtrInfo, 6518 MachinePointerInfo SrcPtrInfo) { 6519 // Check to see if we should lower the memmove to loads and stores first. 6520 // For cases within the target-specified limits, this is the best choice. 6521 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 6522 if (ConstantSize) { 6523 // Memmove with size zero? Just return the original chain. 6524 if (ConstantSize->isNullValue()) 6525 return Chain; 6526 6527 SDValue Result = getMemmoveLoadsAndStores( 6528 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, 6529 isVol, false, DstPtrInfo, SrcPtrInfo); 6530 if (Result.getNode()) 6531 return Result; 6532 } 6533 6534 // Then check to see if we should lower the memmove with target-specific 6535 // code. If the target chooses to do this, this is the next best. 6536 if (TSI) { 6537 SDValue Result = 6538 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, 6539 Alignment, isVol, DstPtrInfo, SrcPtrInfo); 6540 if (Result.getNode()) 6541 return Result; 6542 } 6543 6544 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 6545 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 6546 6547 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 6548 // not be safe. See memcpy above for more details. 6549 6550 // Emit a library call. 6551 TargetLowering::ArgListTy Args; 6552 TargetLowering::ArgListEntry Entry; 6553 Entry.Ty = Type::getInt8PtrTy(*getContext()); 6554 Entry.Node = Dst; Args.push_back(Entry); 6555 Entry.Node = Src; Args.push_back(Entry); 6556 6557 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6558 Entry.Node = Size; Args.push_back(Entry); 6559 // FIXME: pass in SDLoc 6560 TargetLowering::CallLoweringInfo CLI(*this); 6561 CLI.setDebugLoc(dl) 6562 .setChain(Chain) 6563 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 6564 Dst.getValueType().getTypeForEVT(*getContext()), 6565 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 6566 TLI->getPointerTy(getDataLayout())), 6567 std::move(Args)) 6568 .setDiscardResult() 6569 .setTailCall(isTailCall); 6570 6571 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 6572 return CallResult.second; 6573 } 6574 6575 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, 6576 SDValue Dst, unsigned DstAlign, 6577 SDValue Src, unsigned SrcAlign, 6578 SDValue Size, Type *SizeTy, 6579 unsigned ElemSz, bool isTailCall, 6580 MachinePointerInfo DstPtrInfo, 6581 MachinePointerInfo SrcPtrInfo) { 6582 // Emit a library call. 6583 TargetLowering::ArgListTy Args; 6584 TargetLowering::ArgListEntry Entry; 6585 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6586 Entry.Node = Dst; 6587 Args.push_back(Entry); 6588 6589 Entry.Node = Src; 6590 Args.push_back(Entry); 6591 6592 Entry.Ty = SizeTy; 6593 Entry.Node = Size; 6594 Args.push_back(Entry); 6595 6596 RTLIB::Libcall LibraryCall = 6597 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz); 6598 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 6599 report_fatal_error("Unsupported element size"); 6600 6601 TargetLowering::CallLoweringInfo CLI(*this); 6602 CLI.setDebugLoc(dl) 6603 .setChain(Chain) 6604 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 6605 Type::getVoidTy(*getContext()), 6606 getExternalSymbol(TLI->getLibcallName(LibraryCall), 6607 TLI->getPointerTy(getDataLayout())), 6608 std::move(Args)) 6609 .setDiscardResult() 6610 .setTailCall(isTailCall); 6611 6612 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 6613 return CallResult.second; 6614 } 6615 6616 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 6617 SDValue Src, SDValue Size, Align Alignment, 6618 bool isVol, bool isTailCall, 6619 MachinePointerInfo DstPtrInfo) { 6620 // Check to see if we should lower the memset to stores first. 6621 // For cases within the target-specified limits, this is the best choice. 6622 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 6623 if (ConstantSize) { 6624 // Memset with size zero? Just return the original chain. 6625 if (ConstantSize->isNullValue()) 6626 return Chain; 6627 6628 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, 6629 ConstantSize->getZExtValue(), Alignment, 6630 isVol, DstPtrInfo); 6631 6632 if (Result.getNode()) 6633 return Result; 6634 } 6635 6636 // Then check to see if we should lower the memset with target-specific 6637 // code. If the target chooses to do this, this is the next best. 6638 if (TSI) { 6639 SDValue Result = TSI->EmitTargetCodeForMemset( 6640 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo); 6641 if (Result.getNode()) 6642 return Result; 6643 } 6644 6645 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 6646 6647 // Emit a library call. 6648 TargetLowering::ArgListTy Args; 6649 TargetLowering::ArgListEntry Entry; 6650 Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext()); 6651 Args.push_back(Entry); 6652 Entry.Node = Src; 6653 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext()); 6654 Args.push_back(Entry); 6655 Entry.Node = Size; 6656 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6657 Args.push_back(Entry); 6658 6659 // FIXME: pass in SDLoc 6660 TargetLowering::CallLoweringInfo CLI(*this); 6661 CLI.setDebugLoc(dl) 6662 .setChain(Chain) 6663 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), 6664 Dst.getValueType().getTypeForEVT(*getContext()), 6665 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 6666 TLI->getPointerTy(getDataLayout())), 6667 std::move(Args)) 6668 .setDiscardResult() 6669 .setTailCall(isTailCall); 6670 6671 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 6672 return CallResult.second; 6673 } 6674 6675 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, 6676 SDValue Dst, unsigned DstAlign, 6677 SDValue Value, SDValue Size, Type *SizeTy, 6678 unsigned ElemSz, bool isTailCall, 6679 MachinePointerInfo DstPtrInfo) { 6680 // Emit a library call. 6681 TargetLowering::ArgListTy Args; 6682 TargetLowering::ArgListEntry Entry; 6683 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6684 Entry.Node = Dst; 6685 Args.push_back(Entry); 6686 6687 Entry.Ty = Type::getInt8Ty(*getContext()); 6688 Entry.Node = Value; 6689 Args.push_back(Entry); 6690 6691 Entry.Ty = SizeTy; 6692 Entry.Node = Size; 6693 Args.push_back(Entry); 6694 6695 RTLIB::Libcall LibraryCall = 6696 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz); 6697 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 6698 report_fatal_error("Unsupported element size"); 6699 6700 TargetLowering::CallLoweringInfo CLI(*this); 6701 CLI.setDebugLoc(dl) 6702 .setChain(Chain) 6703 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 6704 Type::getVoidTy(*getContext()), 6705 getExternalSymbol(TLI->getLibcallName(LibraryCall), 6706 TLI->getPointerTy(getDataLayout())), 6707 std::move(Args)) 6708 .setDiscardResult() 6709 .setTailCall(isTailCall); 6710 6711 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 6712 return CallResult.second; 6713 } 6714 6715 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6716 SDVTList VTList, ArrayRef<SDValue> Ops, 6717 MachineMemOperand *MMO) { 6718 FoldingSetNodeID ID; 6719 ID.AddInteger(MemVT.getRawBits()); 6720 AddNodeIDNode(ID, Opcode, VTList, Ops); 6721 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6722 void* IP = nullptr; 6723 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6724 cast<AtomicSDNode>(E)->refineAlignment(MMO); 6725 return SDValue(E, 0); 6726 } 6727 6728 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6729 VTList, MemVT, MMO); 6730 createOperands(N, Ops); 6731 6732 CSEMap.InsertNode(N, IP); 6733 InsertNode(N); 6734 return SDValue(N, 0); 6735 } 6736 6737 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, 6738 EVT MemVT, SDVTList VTs, SDValue Chain, 6739 SDValue Ptr, SDValue Cmp, SDValue Swp, 6740 MachineMemOperand *MMO) { 6741 assert(Opcode == ISD::ATOMIC_CMP_SWAP || 6742 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 6743 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 6744 6745 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 6746 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6747 } 6748 6749 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6750 SDValue Chain, SDValue Ptr, SDValue Val, 6751 MachineMemOperand *MMO) { 6752 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 6753 Opcode == ISD::ATOMIC_LOAD_SUB || 6754 Opcode == ISD::ATOMIC_LOAD_AND || 6755 Opcode == ISD::ATOMIC_LOAD_CLR || 6756 Opcode == ISD::ATOMIC_LOAD_OR || 6757 Opcode == ISD::ATOMIC_LOAD_XOR || 6758 Opcode == ISD::ATOMIC_LOAD_NAND || 6759 Opcode == ISD::ATOMIC_LOAD_MIN || 6760 Opcode == ISD::ATOMIC_LOAD_MAX || 6761 Opcode == ISD::ATOMIC_LOAD_UMIN || 6762 Opcode == ISD::ATOMIC_LOAD_UMAX || 6763 Opcode == ISD::ATOMIC_LOAD_FADD || 6764 Opcode == ISD::ATOMIC_LOAD_FSUB || 6765 Opcode == ISD::ATOMIC_SWAP || 6766 Opcode == ISD::ATOMIC_STORE) && 6767 "Invalid Atomic Op"); 6768 6769 EVT VT = Val.getValueType(); 6770 6771 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 6772 getVTList(VT, MVT::Other); 6773 SDValue Ops[] = {Chain, Ptr, Val}; 6774 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6775 } 6776 6777 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6778 EVT VT, SDValue Chain, SDValue Ptr, 6779 MachineMemOperand *MMO) { 6780 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 6781 6782 SDVTList VTs = getVTList(VT, MVT::Other); 6783 SDValue Ops[] = {Chain, Ptr}; 6784 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6785 } 6786 6787 /// getMergeValues - Create a MERGE_VALUES node from the given operands. 6788 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { 6789 if (Ops.size() == 1) 6790 return Ops[0]; 6791 6792 SmallVector<EVT, 4> VTs; 6793 VTs.reserve(Ops.size()); 6794 for (unsigned i = 0; i < Ops.size(); ++i) 6795 VTs.push_back(Ops[i].getValueType()); 6796 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); 6797 } 6798 6799 SDValue SelectionDAG::getMemIntrinsicNode( 6800 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 6801 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, 6802 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) { 6803 if (!Size && MemVT.isScalableVector()) 6804 Size = MemoryLocation::UnknownSize; 6805 else if (!Size) 6806 Size = MemVT.getStoreSize(); 6807 6808 MachineFunction &MF = getMachineFunction(); 6809 MachineMemOperand *MMO = 6810 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo); 6811 6812 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); 6813 } 6814 6815 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, 6816 SDVTList VTList, 6817 ArrayRef<SDValue> Ops, EVT MemVT, 6818 MachineMemOperand *MMO) { 6819 assert((Opcode == ISD::INTRINSIC_VOID || 6820 Opcode == ISD::INTRINSIC_W_CHAIN || 6821 Opcode == ISD::PREFETCH || 6822 ((int)Opcode <= std::numeric_limits<int>::max() && 6823 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 6824 "Opcode is not a memory-accessing opcode!"); 6825 6826 // Memoize the node unless it returns a flag. 6827 MemIntrinsicSDNode *N; 6828 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 6829 FoldingSetNodeID ID; 6830 AddNodeIDNode(ID, Opcode, VTList, Ops); 6831 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>( 6832 Opcode, dl.getIROrder(), VTList, MemVT, MMO)); 6833 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6834 void *IP = nullptr; 6835 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6836 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 6837 return SDValue(E, 0); 6838 } 6839 6840 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6841 VTList, MemVT, MMO); 6842 createOperands(N, Ops); 6843 6844 CSEMap.InsertNode(N, IP); 6845 } else { 6846 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6847 VTList, MemVT, MMO); 6848 createOperands(N, Ops); 6849 } 6850 InsertNode(N); 6851 SDValue V(N, 0); 6852 NewSDValueDbgMsg(V, "Creating new node: ", this); 6853 return V; 6854 } 6855 6856 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, 6857 SDValue Chain, int FrameIndex, 6858 int64_t Size, int64_t Offset) { 6859 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; 6860 const auto VTs = getVTList(MVT::Other); 6861 SDValue Ops[2] = { 6862 Chain, 6863 getFrameIndex(FrameIndex, 6864 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), 6865 true)}; 6866 6867 FoldingSetNodeID ID; 6868 AddNodeIDNode(ID, Opcode, VTs, Ops); 6869 ID.AddInteger(FrameIndex); 6870 ID.AddInteger(Size); 6871 ID.AddInteger(Offset); 6872 void *IP = nullptr; 6873 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 6874 return SDValue(E, 0); 6875 6876 LifetimeSDNode *N = newSDNode<LifetimeSDNode>( 6877 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); 6878 createOperands(N, Ops); 6879 CSEMap.InsertNode(N, IP); 6880 InsertNode(N); 6881 SDValue V(N, 0); 6882 NewSDValueDbgMsg(V, "Creating new node: ", this); 6883 return V; 6884 } 6885 6886 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 6887 /// MachinePointerInfo record from it. This is particularly useful because the 6888 /// code generator has many cases where it doesn't bother passing in a 6889 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 6890 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 6891 SelectionDAG &DAG, SDValue Ptr, 6892 int64_t Offset = 0) { 6893 // If this is FI+Offset, we can model it. 6894 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 6895 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), 6896 FI->getIndex(), Offset); 6897 6898 // If this is (FI+Offset1)+Offset2, we can model it. 6899 if (Ptr.getOpcode() != ISD::ADD || 6900 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 6901 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 6902 return Info; 6903 6904 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6905 return MachinePointerInfo::getFixedStack( 6906 DAG.getMachineFunction(), FI, 6907 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 6908 } 6909 6910 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 6911 /// MachinePointerInfo record from it. This is particularly useful because the 6912 /// code generator has many cases where it doesn't bother passing in a 6913 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 6914 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 6915 SelectionDAG &DAG, SDValue Ptr, 6916 SDValue OffsetOp) { 6917 // If the 'Offset' value isn't a constant, we can't handle this. 6918 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 6919 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue()); 6920 if (OffsetOp.isUndef()) 6921 return InferPointerInfo(Info, DAG, Ptr); 6922 return Info; 6923 } 6924 6925 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 6926 EVT VT, const SDLoc &dl, SDValue Chain, 6927 SDValue Ptr, SDValue Offset, 6928 MachinePointerInfo PtrInfo, EVT MemVT, 6929 Align Alignment, 6930 MachineMemOperand::Flags MMOFlags, 6931 const AAMDNodes &AAInfo, const MDNode *Ranges) { 6932 assert(Chain.getValueType() == MVT::Other && 6933 "Invalid chain type"); 6934 6935 MMOFlags |= MachineMemOperand::MOLoad; 6936 assert((MMOFlags & MachineMemOperand::MOStore) == 0); 6937 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 6938 // clients. 6939 if (PtrInfo.V.isNull()) 6940 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); 6941 6942 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); 6943 MachineFunction &MF = getMachineFunction(); 6944 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, 6945 Alignment, AAInfo, Ranges); 6946 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 6947 } 6948 6949 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 6950 EVT VT, const SDLoc &dl, SDValue Chain, 6951 SDValue Ptr, SDValue Offset, EVT MemVT, 6952 MachineMemOperand *MMO) { 6953 if (VT == MemVT) { 6954 ExtType = ISD::NON_EXTLOAD; 6955 } else if (ExtType == ISD::NON_EXTLOAD) { 6956 assert(VT == MemVT && "Non-extending load from different memory type!"); 6957 } else { 6958 // Extending load. 6959 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 6960 "Should only be an extending load, not truncating!"); 6961 assert(VT.isInteger() == MemVT.isInteger() && 6962 "Cannot convert from FP to Int or Int -> FP!"); 6963 assert(VT.isVector() == MemVT.isVector() && 6964 "Cannot use an ext load to convert to or from a vector!"); 6965 assert((!VT.isVector() || 6966 VT.getVectorElementCount() == MemVT.getVectorElementCount()) && 6967 "Cannot use an ext load to change the number of vector elements!"); 6968 } 6969 6970 bool Indexed = AM != ISD::UNINDEXED; 6971 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); 6972 6973 SDVTList VTs = Indexed ? 6974 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 6975 SDValue Ops[] = { Chain, Ptr, Offset }; 6976 FoldingSetNodeID ID; 6977 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); 6978 ID.AddInteger(MemVT.getRawBits()); 6979 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>( 6980 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); 6981 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6982 void *IP = nullptr; 6983 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6984 cast<LoadSDNode>(E)->refineAlignment(MMO); 6985 return SDValue(E, 0); 6986 } 6987 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 6988 ExtType, MemVT, MMO); 6989 createOperands(N, Ops); 6990 6991 CSEMap.InsertNode(N, IP); 6992 InsertNode(N); 6993 SDValue V(N, 0); 6994 NewSDValueDbgMsg(V, "Creating new node: ", this); 6995 return V; 6996 } 6997 6998 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 6999 SDValue Ptr, MachinePointerInfo PtrInfo, 7000 MaybeAlign Alignment, 7001 MachineMemOperand::Flags MMOFlags, 7002 const AAMDNodes &AAInfo, const MDNode *Ranges) { 7003 SDValue Undef = getUNDEF(Ptr.getValueType()); 7004 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 7005 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); 7006 } 7007 7008 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 7009 SDValue Ptr, MachineMemOperand *MMO) { 7010 SDValue Undef = getUNDEF(Ptr.getValueType()); 7011 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 7012 VT, MMO); 7013 } 7014 7015 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 7016 EVT VT, SDValue Chain, SDValue Ptr, 7017 MachinePointerInfo PtrInfo, EVT MemVT, 7018 MaybeAlign Alignment, 7019 MachineMemOperand::Flags MMOFlags, 7020 const AAMDNodes &AAInfo) { 7021 SDValue Undef = getUNDEF(Ptr.getValueType()); 7022 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, 7023 MemVT, Alignment, MMOFlags, AAInfo); 7024 } 7025 7026 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 7027 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, 7028 MachineMemOperand *MMO) { 7029 SDValue Undef = getUNDEF(Ptr.getValueType()); 7030 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 7031 MemVT, MMO); 7032 } 7033 7034 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, 7035 SDValue Base, SDValue Offset, 7036 ISD::MemIndexedMode AM) { 7037 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 7038 assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); 7039 // Don't propagate the invariant or dereferenceable flags. 7040 auto MMOFlags = 7041 LD->getMemOperand()->getFlags() & 7042 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 7043 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 7044 LD->getChain(), Base, Offset, LD->getPointerInfo(), 7045 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); 7046 } 7047 7048 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7049 SDValue Ptr, MachinePointerInfo PtrInfo, 7050 Align Alignment, 7051 MachineMemOperand::Flags MMOFlags, 7052 const AAMDNodes &AAInfo) { 7053 assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); 7054 7055 MMOFlags |= MachineMemOperand::MOStore; 7056 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 7057 7058 if (PtrInfo.V.isNull()) 7059 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 7060 7061 MachineFunction &MF = getMachineFunction(); 7062 uint64_t Size = 7063 MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); 7064 MachineMemOperand *MMO = 7065 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); 7066 return getStore(Chain, dl, Val, Ptr, MMO); 7067 } 7068 7069 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7070 SDValue Ptr, MachineMemOperand *MMO) { 7071 assert(Chain.getValueType() == MVT::Other && 7072 "Invalid chain type"); 7073 EVT VT = Val.getValueType(); 7074 SDVTList VTs = getVTList(MVT::Other); 7075 SDValue Undef = getUNDEF(Ptr.getValueType()); 7076 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 7077 FoldingSetNodeID ID; 7078 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7079 ID.AddInteger(VT.getRawBits()); 7080 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 7081 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); 7082 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7083 void *IP = nullptr; 7084 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7085 cast<StoreSDNode>(E)->refineAlignment(MMO); 7086 return SDValue(E, 0); 7087 } 7088 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7089 ISD::UNINDEXED, false, VT, MMO); 7090 createOperands(N, Ops); 7091 7092 CSEMap.InsertNode(N, IP); 7093 InsertNode(N); 7094 SDValue V(N, 0); 7095 NewSDValueDbgMsg(V, "Creating new node: ", this); 7096 return V; 7097 } 7098 7099 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7100 SDValue Ptr, MachinePointerInfo PtrInfo, 7101 EVT SVT, Align Alignment, 7102 MachineMemOperand::Flags MMOFlags, 7103 const AAMDNodes &AAInfo) { 7104 assert(Chain.getValueType() == MVT::Other && 7105 "Invalid chain type"); 7106 7107 MMOFlags |= MachineMemOperand::MOStore; 7108 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 7109 7110 if (PtrInfo.V.isNull()) 7111 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 7112 7113 MachineFunction &MF = getMachineFunction(); 7114 MachineMemOperand *MMO = MF.getMachineMemOperand( 7115 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), 7116 Alignment, AAInfo); 7117 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 7118 } 7119 7120 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7121 SDValue Ptr, EVT SVT, 7122 MachineMemOperand *MMO) { 7123 EVT VT = Val.getValueType(); 7124 7125 assert(Chain.getValueType() == MVT::Other && 7126 "Invalid chain type"); 7127 if (VT == SVT) 7128 return getStore(Chain, dl, Val, Ptr, MMO); 7129 7130 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 7131 "Should only be a truncating store, not extending!"); 7132 assert(VT.isInteger() == SVT.isInteger() && 7133 "Can't do FP-INT conversion!"); 7134 assert(VT.isVector() == SVT.isVector() && 7135 "Cannot use trunc store to convert to or from a vector!"); 7136 assert((!VT.isVector() || 7137 VT.getVectorElementCount() == SVT.getVectorElementCount()) && 7138 "Cannot use trunc store to change the number of vector elements!"); 7139 7140 SDVTList VTs = getVTList(MVT::Other); 7141 SDValue Undef = getUNDEF(Ptr.getValueType()); 7142 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 7143 FoldingSetNodeID ID; 7144 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7145 ID.AddInteger(SVT.getRawBits()); 7146 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 7147 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); 7148 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7149 void *IP = nullptr; 7150 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7151 cast<StoreSDNode>(E)->refineAlignment(MMO); 7152 return SDValue(E, 0); 7153 } 7154 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7155 ISD::UNINDEXED, true, SVT, MMO); 7156 createOperands(N, Ops); 7157 7158 CSEMap.InsertNode(N, IP); 7159 InsertNode(N); 7160 SDValue V(N, 0); 7161 NewSDValueDbgMsg(V, "Creating new node: ", this); 7162 return V; 7163 } 7164 7165 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, 7166 SDValue Base, SDValue Offset, 7167 ISD::MemIndexedMode AM) { 7168 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 7169 assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); 7170 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 7171 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 7172 FoldingSetNodeID ID; 7173 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7174 ID.AddInteger(ST->getMemoryVT().getRawBits()); 7175 ID.AddInteger(ST->getRawSubclassData()); 7176 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 7177 void *IP = nullptr; 7178 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 7179 return SDValue(E, 0); 7180 7181 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 7182 ST->isTruncatingStore(), ST->getMemoryVT(), 7183 ST->getMemOperand()); 7184 createOperands(N, Ops); 7185 7186 CSEMap.InsertNode(N, IP); 7187 InsertNode(N); 7188 SDValue V(N, 0); 7189 NewSDValueDbgMsg(V, "Creating new node: ", this); 7190 return V; 7191 } 7192 7193 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, 7194 SDValue Base, SDValue Offset, SDValue Mask, 7195 SDValue PassThru, EVT MemVT, 7196 MachineMemOperand *MMO, 7197 ISD::MemIndexedMode AM, 7198 ISD::LoadExtType ExtTy, bool isExpanding) { 7199 bool Indexed = AM != ISD::UNINDEXED; 7200 assert((Indexed || Offset.isUndef()) && 7201 "Unindexed masked load with an offset!"); 7202 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) 7203 : getVTList(VT, MVT::Other); 7204 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; 7205 FoldingSetNodeID ID; 7206 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); 7207 ID.AddInteger(MemVT.getRawBits()); 7208 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>( 7209 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO)); 7210 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7211 void *IP = nullptr; 7212 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7213 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); 7214 return SDValue(E, 0); 7215 } 7216 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7217 AM, ExtTy, isExpanding, MemVT, MMO); 7218 createOperands(N, Ops); 7219 7220 CSEMap.InsertNode(N, IP); 7221 InsertNode(N); 7222 SDValue V(N, 0); 7223 NewSDValueDbgMsg(V, "Creating new node: ", this); 7224 return V; 7225 } 7226 7227 SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, 7228 SDValue Base, SDValue Offset, 7229 ISD::MemIndexedMode AM) { 7230 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad); 7231 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!"); 7232 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base, 7233 Offset, LD->getMask(), LD->getPassThru(), 7234 LD->getMemoryVT(), LD->getMemOperand(), AM, 7235 LD->getExtensionType(), LD->isExpandingLoad()); 7236 } 7237 7238 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, 7239 SDValue Val, SDValue Base, SDValue Offset, 7240 SDValue Mask, EVT MemVT, 7241 MachineMemOperand *MMO, 7242 ISD::MemIndexedMode AM, bool IsTruncating, 7243 bool IsCompressing) { 7244 assert(Chain.getValueType() == MVT::Other && 7245 "Invalid chain type"); 7246 bool Indexed = AM != ISD::UNINDEXED; 7247 assert((Indexed || Offset.isUndef()) && 7248 "Unindexed masked store with an offset!"); 7249 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) 7250 : getVTList(MVT::Other); 7251 SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; 7252 FoldingSetNodeID ID; 7253 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); 7254 ID.AddInteger(MemVT.getRawBits()); 7255 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>( 7256 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); 7257 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7258 void *IP = nullptr; 7259 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7260 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); 7261 return SDValue(E, 0); 7262 } 7263 auto *N = 7264 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 7265 IsTruncating, IsCompressing, MemVT, MMO); 7266 createOperands(N, Ops); 7267 7268 CSEMap.InsertNode(N, IP); 7269 InsertNode(N); 7270 SDValue V(N, 0); 7271 NewSDValueDbgMsg(V, "Creating new node: ", this); 7272 return V; 7273 } 7274 7275 SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, 7276 SDValue Base, SDValue Offset, 7277 ISD::MemIndexedMode AM) { 7278 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore); 7279 assert(ST->getOffset().isUndef() && 7280 "Masked store is already a indexed store!"); 7281 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset, 7282 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(), 7283 AM, ST->isTruncatingStore(), ST->isCompressingStore()); 7284 } 7285 7286 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, 7287 ArrayRef<SDValue> Ops, 7288 MachineMemOperand *MMO, 7289 ISD::MemIndexType IndexType) { 7290 assert(Ops.size() == 6 && "Incompatible number of operands"); 7291 7292 FoldingSetNodeID ID; 7293 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); 7294 ID.AddInteger(VT.getRawBits()); 7295 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( 7296 dl.getIROrder(), VTs, VT, MMO, IndexType)); 7297 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7298 void *IP = nullptr; 7299 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7300 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); 7301 return SDValue(E, 0); 7302 } 7303 7304 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), 7305 VTs, VT, MMO, IndexType); 7306 createOperands(N, Ops); 7307 7308 assert(N->getPassThru().getValueType() == N->getValueType(0) && 7309 "Incompatible type of the PassThru value in MaskedGatherSDNode"); 7310 assert(N->getMask().getValueType().getVectorNumElements() == 7311 N->getValueType(0).getVectorNumElements() && 7312 "Vector width mismatch between mask and data"); 7313 assert(N->getIndex().getValueType().getVectorNumElements() >= 7314 N->getValueType(0).getVectorNumElements() && 7315 "Vector width mismatch between index and data"); 7316 assert(isa<ConstantSDNode>(N->getScale()) && 7317 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 7318 "Scale should be a constant power of 2"); 7319 7320 CSEMap.InsertNode(N, IP); 7321 InsertNode(N); 7322 SDValue V(N, 0); 7323 NewSDValueDbgMsg(V, "Creating new node: ", this); 7324 return V; 7325 } 7326 7327 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, 7328 ArrayRef<SDValue> Ops, 7329 MachineMemOperand *MMO, 7330 ISD::MemIndexType IndexType, 7331 bool IsTrunc) { 7332 assert(Ops.size() == 6 && "Incompatible number of operands"); 7333 7334 FoldingSetNodeID ID; 7335 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); 7336 ID.AddInteger(VT.getRawBits()); 7337 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( 7338 dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc)); 7339 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7340 void *IP = nullptr; 7341 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7342 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); 7343 return SDValue(E, 0); 7344 } 7345 7346 IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); 7347 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), 7348 VTs, VT, MMO, IndexType, IsTrunc); 7349 createOperands(N, Ops); 7350 7351 assert(N->getMask().getValueType().getVectorElementCount() == 7352 N->getValue().getValueType().getVectorElementCount() && 7353 "Vector width mismatch between mask and data"); 7354 assert( 7355 N->getIndex().getValueType().getVectorElementCount().isScalable() == 7356 N->getValue().getValueType().getVectorElementCount().isScalable() && 7357 "Scalable flags of index and data do not match"); 7358 assert(ElementCount::isKnownGE( 7359 N->getIndex().getValueType().getVectorElementCount(), 7360 N->getValue().getValueType().getVectorElementCount()) && 7361 "Vector width mismatch between index and data"); 7362 assert(isa<ConstantSDNode>(N->getScale()) && 7363 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 7364 "Scale should be a constant power of 2"); 7365 7366 CSEMap.InsertNode(N, IP); 7367 InsertNode(N); 7368 SDValue V(N, 0); 7369 NewSDValueDbgMsg(V, "Creating new node: ", this); 7370 return V; 7371 } 7372 7373 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { 7374 // select undef, T, F --> T (if T is a constant), otherwise F 7375 // select, ?, undef, F --> F 7376 // select, ?, T, undef --> T 7377 if (Cond.isUndef()) 7378 return isConstantValueOfAnyType(T) ? T : F; 7379 if (T.isUndef()) 7380 return F; 7381 if (F.isUndef()) 7382 return T; 7383 7384 // select true, T, F --> T 7385 // select false, T, F --> F 7386 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond)) 7387 return CondC->isNullValue() ? F : T; 7388 7389 // TODO: This should simplify VSELECT with constant condition using something 7390 // like this (but check boolean contents to be complete?): 7391 // if (ISD::isBuildVectorAllOnes(Cond.getNode())) 7392 // return T; 7393 // if (ISD::isBuildVectorAllZeros(Cond.getNode())) 7394 // return F; 7395 7396 // select ?, T, T --> T 7397 if (T == F) 7398 return T; 7399 7400 return SDValue(); 7401 } 7402 7403 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { 7404 // shift undef, Y --> 0 (can always assume that the undef value is 0) 7405 if (X.isUndef()) 7406 return getConstant(0, SDLoc(X.getNode()), X.getValueType()); 7407 // shift X, undef --> undef (because it may shift by the bitwidth) 7408 if (Y.isUndef()) 7409 return getUNDEF(X.getValueType()); 7410 7411 // shift 0, Y --> 0 7412 // shift X, 0 --> X 7413 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y)) 7414 return X; 7415 7416 // shift X, C >= bitwidth(X) --> undef 7417 // All vector elements must be too big (or undef) to avoid partial undefs. 7418 auto isShiftTooBig = [X](ConstantSDNode *Val) { 7419 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits()); 7420 }; 7421 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) 7422 return getUNDEF(X.getValueType()); 7423 7424 return SDValue(); 7425 } 7426 7427 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, 7428 SDNodeFlags Flags) { 7429 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand 7430 // (an undef operand can be chosen to be Nan/Inf), then the result of this 7431 // operation is poison. That result can be relaxed to undef. 7432 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true); 7433 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true); 7434 bool HasNan = (XC && XC->getValueAPF().isNaN()) || 7435 (YC && YC->getValueAPF().isNaN()); 7436 bool HasInf = (XC && XC->getValueAPF().isInfinity()) || 7437 (YC && YC->getValueAPF().isInfinity()); 7438 7439 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef())) 7440 return getUNDEF(X.getValueType()); 7441 7442 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef())) 7443 return getUNDEF(X.getValueType()); 7444 7445 if (!YC) 7446 return SDValue(); 7447 7448 // X + -0.0 --> X 7449 if (Opcode == ISD::FADD) 7450 if (YC->getValueAPF().isNegZero()) 7451 return X; 7452 7453 // X - +0.0 --> X 7454 if (Opcode == ISD::FSUB) 7455 if (YC->getValueAPF().isPosZero()) 7456 return X; 7457 7458 // X * 1.0 --> X 7459 // X / 1.0 --> X 7460 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) 7461 if (YC->getValueAPF().isExactlyValue(1.0)) 7462 return X; 7463 7464 // X * 0.0 --> 0.0 7465 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) 7466 if (YC->getValueAPF().isZero()) 7467 return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); 7468 7469 return SDValue(); 7470 } 7471 7472 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, 7473 SDValue Ptr, SDValue SV, unsigned Align) { 7474 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; 7475 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); 7476 } 7477 7478 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7479 ArrayRef<SDUse> Ops) { 7480 switch (Ops.size()) { 7481 case 0: return getNode(Opcode, DL, VT); 7482 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); 7483 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 7484 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 7485 default: break; 7486 } 7487 7488 // Copy from an SDUse array into an SDValue array for use with 7489 // the regular getNode logic. 7490 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end()); 7491 return getNode(Opcode, DL, VT, NewOps); 7492 } 7493 7494 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7495 ArrayRef<SDValue> Ops) { 7496 SDNodeFlags Flags; 7497 if (Inserter) 7498 Flags = Inserter->getFlags(); 7499 return getNode(Opcode, DL, VT, Ops, Flags); 7500 } 7501 7502 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7503 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { 7504 unsigned NumOps = Ops.size(); 7505 switch (NumOps) { 7506 case 0: return getNode(Opcode, DL, VT); 7507 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); 7508 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); 7509 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags); 7510 default: break; 7511 } 7512 7513 switch (Opcode) { 7514 default: break; 7515 case ISD::BUILD_VECTOR: 7516 // Attempt to simplify BUILD_VECTOR. 7517 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 7518 return V; 7519 break; 7520 case ISD::CONCAT_VECTORS: 7521 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 7522 return V; 7523 break; 7524 case ISD::SELECT_CC: 7525 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 7526 assert(Ops[0].getValueType() == Ops[1].getValueType() && 7527 "LHS and RHS of condition must have same type!"); 7528 assert(Ops[2].getValueType() == Ops[3].getValueType() && 7529 "True and False arms of SelectCC must have same type!"); 7530 assert(Ops[2].getValueType() == VT && 7531 "select_cc node must be of same type as true and false value!"); 7532 break; 7533 case ISD::BR_CC: 7534 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 7535 assert(Ops[2].getValueType() == Ops[3].getValueType() && 7536 "LHS/RHS of comparison should match types!"); 7537 break; 7538 } 7539 7540 // Memoize nodes. 7541 SDNode *N; 7542 SDVTList VTs = getVTList(VT); 7543 7544 if (VT != MVT::Glue) { 7545 FoldingSetNodeID ID; 7546 AddNodeIDNode(ID, Opcode, VTs, Ops); 7547 void *IP = nullptr; 7548 7549 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 7550 return SDValue(E, 0); 7551 7552 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 7553 createOperands(N, Ops); 7554 7555 CSEMap.InsertNode(N, IP); 7556 } else { 7557 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 7558 createOperands(N, Ops); 7559 } 7560 7561 N->setFlags(Flags); 7562 InsertNode(N); 7563 SDValue V(N, 0); 7564 NewSDValueDbgMsg(V, "Creating new node: ", this); 7565 return V; 7566 } 7567 7568 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 7569 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { 7570 return getNode(Opcode, DL, getVTList(ResultTys), Ops); 7571 } 7572 7573 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7574 ArrayRef<SDValue> Ops) { 7575 SDNodeFlags Flags; 7576 if (Inserter) 7577 Flags = Inserter->getFlags(); 7578 return getNode(Opcode, DL, VTList, Ops, Flags); 7579 } 7580 7581 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7582 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { 7583 if (VTList.NumVTs == 1) 7584 return getNode(Opcode, DL, VTList.VTs[0], Ops); 7585 7586 switch (Opcode) { 7587 case ISD::STRICT_FP_EXTEND: 7588 assert(VTList.NumVTs == 2 && Ops.size() == 2 && 7589 "Invalid STRICT_FP_EXTEND!"); 7590 assert(VTList.VTs[0].isFloatingPoint() && 7591 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"); 7592 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && 7593 "STRICT_FP_EXTEND result type should be vector iff the operand " 7594 "type is vector!"); 7595 assert((!VTList.VTs[0].isVector() || 7596 VTList.VTs[0].getVectorNumElements() == 7597 Ops[1].getValueType().getVectorNumElements()) && 7598 "Vector element count mismatch!"); 7599 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && 7600 "Invalid fpext node, dst <= src!"); 7601 break; 7602 case ISD::STRICT_FP_ROUND: 7603 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!"); 7604 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && 7605 "STRICT_FP_ROUND result type should be vector iff the operand " 7606 "type is vector!"); 7607 assert((!VTList.VTs[0].isVector() || 7608 VTList.VTs[0].getVectorNumElements() == 7609 Ops[1].getValueType().getVectorNumElements()) && 7610 "Vector element count mismatch!"); 7611 assert(VTList.VTs[0].isFloatingPoint() && 7612 Ops[1].getValueType().isFloatingPoint() && 7613 VTList.VTs[0].bitsLT(Ops[1].getValueType()) && 7614 isa<ConstantSDNode>(Ops[2]) && 7615 (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || 7616 cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && 7617 "Invalid STRICT_FP_ROUND!"); 7618 break; 7619 #if 0 7620 // FIXME: figure out how to safely handle things like 7621 // int foo(int x) { return 1 << (x & 255); } 7622 // int bar() { return foo(256); } 7623 case ISD::SRA_PARTS: 7624 case ISD::SRL_PARTS: 7625 case ISD::SHL_PARTS: 7626 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 7627 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 7628 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 7629 else if (N3.getOpcode() == ISD::AND) 7630 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 7631 // If the and is only masking out bits that cannot effect the shift, 7632 // eliminate the and. 7633 unsigned NumBits = VT.getScalarSizeInBits()*2; 7634 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 7635 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 7636 } 7637 break; 7638 #endif 7639 } 7640 7641 // Memoize the node unless it returns a flag. 7642 SDNode *N; 7643 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 7644 FoldingSetNodeID ID; 7645 AddNodeIDNode(ID, Opcode, VTList, Ops); 7646 void *IP = nullptr; 7647 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 7648 return SDValue(E, 0); 7649 7650 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 7651 createOperands(N, Ops); 7652 CSEMap.InsertNode(N, IP); 7653 } else { 7654 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 7655 createOperands(N, Ops); 7656 } 7657 7658 N->setFlags(Flags); 7659 InsertNode(N); 7660 SDValue V(N, 0); 7661 NewSDValueDbgMsg(V, "Creating new node: ", this); 7662 return V; 7663 } 7664 7665 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 7666 SDVTList VTList) { 7667 return getNode(Opcode, DL, VTList, None); 7668 } 7669 7670 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7671 SDValue N1) { 7672 SDValue Ops[] = { N1 }; 7673 return getNode(Opcode, DL, VTList, Ops); 7674 } 7675 7676 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7677 SDValue N1, SDValue N2) { 7678 SDValue Ops[] = { N1, N2 }; 7679 return getNode(Opcode, DL, VTList, Ops); 7680 } 7681 7682 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7683 SDValue N1, SDValue N2, SDValue N3) { 7684 SDValue Ops[] = { N1, N2, N3 }; 7685 return getNode(Opcode, DL, VTList, Ops); 7686 } 7687 7688 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7689 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 7690 SDValue Ops[] = { N1, N2, N3, N4 }; 7691 return getNode(Opcode, DL, VTList, Ops); 7692 } 7693 7694 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 7695 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 7696 SDValue N5) { 7697 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 7698 return getNode(Opcode, DL, VTList, Ops); 7699 } 7700 7701 SDVTList SelectionDAG::getVTList(EVT VT) { 7702 return makeVTList(SDNode::getValueTypeList(VT), 1); 7703 } 7704 7705 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 7706 FoldingSetNodeID ID; 7707 ID.AddInteger(2U); 7708 ID.AddInteger(VT1.getRawBits()); 7709 ID.AddInteger(VT2.getRawBits()); 7710 7711 void *IP = nullptr; 7712 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 7713 if (!Result) { 7714 EVT *Array = Allocator.Allocate<EVT>(2); 7715 Array[0] = VT1; 7716 Array[1] = VT2; 7717 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 7718 VTListMap.InsertNode(Result, IP); 7719 } 7720 return Result->getSDVTList(); 7721 } 7722 7723 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 7724 FoldingSetNodeID ID; 7725 ID.AddInteger(3U); 7726 ID.AddInteger(VT1.getRawBits()); 7727 ID.AddInteger(VT2.getRawBits()); 7728 ID.AddInteger(VT3.getRawBits()); 7729 7730 void *IP = nullptr; 7731 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 7732 if (!Result) { 7733 EVT *Array = Allocator.Allocate<EVT>(3); 7734 Array[0] = VT1; 7735 Array[1] = VT2; 7736 Array[2] = VT3; 7737 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 7738 VTListMap.InsertNode(Result, IP); 7739 } 7740 return Result->getSDVTList(); 7741 } 7742 7743 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 7744 FoldingSetNodeID ID; 7745 ID.AddInteger(4U); 7746 ID.AddInteger(VT1.getRawBits()); 7747 ID.AddInteger(VT2.getRawBits()); 7748 ID.AddInteger(VT3.getRawBits()); 7749 ID.AddInteger(VT4.getRawBits()); 7750 7751 void *IP = nullptr; 7752 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 7753 if (!Result) { 7754 EVT *Array = Allocator.Allocate<EVT>(4); 7755 Array[0] = VT1; 7756 Array[1] = VT2; 7757 Array[2] = VT3; 7758 Array[3] = VT4; 7759 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 7760 VTListMap.InsertNode(Result, IP); 7761 } 7762 return Result->getSDVTList(); 7763 } 7764 7765 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { 7766 unsigned NumVTs = VTs.size(); 7767 FoldingSetNodeID ID; 7768 ID.AddInteger(NumVTs); 7769 for (unsigned index = 0; index < NumVTs; index++) { 7770 ID.AddInteger(VTs[index].getRawBits()); 7771 } 7772 7773 void *IP = nullptr; 7774 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 7775 if (!Result) { 7776 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 7777 llvm::copy(VTs, Array); 7778 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 7779 VTListMap.InsertNode(Result, IP); 7780 } 7781 return Result->getSDVTList(); 7782 } 7783 7784 7785 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the 7786 /// specified operands. If the resultant node already exists in the DAG, 7787 /// this does not modify the specified node, instead it returns the node that 7788 /// already exists. If the resultant node does not exist in the DAG, the 7789 /// input node is returned. As a degenerate case, if you specify the same 7790 /// input operands as the node already has, the input node is returned. 7791 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 7792 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 7793 7794 // Check to see if there is no change. 7795 if (Op == N->getOperand(0)) return N; 7796 7797 // See if the modified node already exists. 7798 void *InsertPos = nullptr; 7799 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 7800 return Existing; 7801 7802 // Nope it doesn't. Remove the node from its current place in the maps. 7803 if (InsertPos) 7804 if (!RemoveNodeFromCSEMaps(N)) 7805 InsertPos = nullptr; 7806 7807 // Now we update the operands. 7808 N->OperandList[0].set(Op); 7809 7810 updateDivergence(N); 7811 // If this gets put into a CSE map, add it. 7812 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 7813 return N; 7814 } 7815 7816 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 7817 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 7818 7819 // Check to see if there is no change. 7820 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 7821 return N; // No operands changed, just return the input node. 7822 7823 // See if the modified node already exists. 7824 void *InsertPos = nullptr; 7825 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 7826 return Existing; 7827 7828 // Nope it doesn't. Remove the node from its current place in the maps. 7829 if (InsertPos) 7830 if (!RemoveNodeFromCSEMaps(N)) 7831 InsertPos = nullptr; 7832 7833 // Now we update the operands. 7834 if (N->OperandList[0] != Op1) 7835 N->OperandList[0].set(Op1); 7836 if (N->OperandList[1] != Op2) 7837 N->OperandList[1].set(Op2); 7838 7839 updateDivergence(N); 7840 // If this gets put into a CSE map, add it. 7841 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 7842 return N; 7843 } 7844 7845 SDNode *SelectionDAG:: 7846 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 7847 SDValue Ops[] = { Op1, Op2, Op3 }; 7848 return UpdateNodeOperands(N, Ops); 7849 } 7850 7851 SDNode *SelectionDAG:: 7852 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 7853 SDValue Op3, SDValue Op4) { 7854 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 7855 return UpdateNodeOperands(N, Ops); 7856 } 7857 7858 SDNode *SelectionDAG:: 7859 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 7860 SDValue Op3, SDValue Op4, SDValue Op5) { 7861 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 7862 return UpdateNodeOperands(N, Ops); 7863 } 7864 7865 SDNode *SelectionDAG:: 7866 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { 7867 unsigned NumOps = Ops.size(); 7868 assert(N->getNumOperands() == NumOps && 7869 "Update with wrong number of operands"); 7870 7871 // If no operands changed just return the input node. 7872 if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) 7873 return N; 7874 7875 // See if the modified node already exists. 7876 void *InsertPos = nullptr; 7877 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) 7878 return Existing; 7879 7880 // Nope it doesn't. Remove the node from its current place in the maps. 7881 if (InsertPos) 7882 if (!RemoveNodeFromCSEMaps(N)) 7883 InsertPos = nullptr; 7884 7885 // Now we update the operands. 7886 for (unsigned i = 0; i != NumOps; ++i) 7887 if (N->OperandList[i] != Ops[i]) 7888 N->OperandList[i].set(Ops[i]); 7889 7890 updateDivergence(N); 7891 // If this gets put into a CSE map, add it. 7892 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 7893 return N; 7894 } 7895 7896 /// DropOperands - Release the operands and set this node to have 7897 /// zero operands. 7898 void SDNode::DropOperands() { 7899 // Unlike the code in MorphNodeTo that does this, we don't need to 7900 // watch for dead nodes here. 7901 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 7902 SDUse &Use = *I++; 7903 Use.set(SDValue()); 7904 } 7905 } 7906 7907 void SelectionDAG::setNodeMemRefs(MachineSDNode *N, 7908 ArrayRef<MachineMemOperand *> NewMemRefs) { 7909 if (NewMemRefs.empty()) { 7910 N->clearMemRefs(); 7911 return; 7912 } 7913 7914 // Check if we can avoid allocating by storing a single reference directly. 7915 if (NewMemRefs.size() == 1) { 7916 N->MemRefs = NewMemRefs[0]; 7917 N->NumMemRefs = 1; 7918 return; 7919 } 7920 7921 MachineMemOperand **MemRefsBuffer = 7922 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size()); 7923 llvm::copy(NewMemRefs, MemRefsBuffer); 7924 N->MemRefs = MemRefsBuffer; 7925 N->NumMemRefs = static_cast<int>(NewMemRefs.size()); 7926 } 7927 7928 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 7929 /// machine opcode. 7930 /// 7931 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7932 EVT VT) { 7933 SDVTList VTs = getVTList(VT); 7934 return SelectNodeTo(N, MachineOpc, VTs, None); 7935 } 7936 7937 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7938 EVT VT, SDValue Op1) { 7939 SDVTList VTs = getVTList(VT); 7940 SDValue Ops[] = { Op1 }; 7941 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7942 } 7943 7944 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7945 EVT VT, SDValue Op1, 7946 SDValue Op2) { 7947 SDVTList VTs = getVTList(VT); 7948 SDValue Ops[] = { Op1, Op2 }; 7949 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7950 } 7951 7952 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7953 EVT VT, SDValue Op1, 7954 SDValue Op2, SDValue Op3) { 7955 SDVTList VTs = getVTList(VT); 7956 SDValue Ops[] = { Op1, Op2, Op3 }; 7957 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7958 } 7959 7960 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7961 EVT VT, ArrayRef<SDValue> Ops) { 7962 SDVTList VTs = getVTList(VT); 7963 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7964 } 7965 7966 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7967 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { 7968 SDVTList VTs = getVTList(VT1, VT2); 7969 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7970 } 7971 7972 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7973 EVT VT1, EVT VT2) { 7974 SDVTList VTs = getVTList(VT1, VT2); 7975 return SelectNodeTo(N, MachineOpc, VTs, None); 7976 } 7977 7978 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7979 EVT VT1, EVT VT2, EVT VT3, 7980 ArrayRef<SDValue> Ops) { 7981 SDVTList VTs = getVTList(VT1, VT2, VT3); 7982 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7983 } 7984 7985 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7986 EVT VT1, EVT VT2, 7987 SDValue Op1, SDValue Op2) { 7988 SDVTList VTs = getVTList(VT1, VT2); 7989 SDValue Ops[] = { Op1, Op2 }; 7990 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7991 } 7992 7993 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7994 SDVTList VTs,ArrayRef<SDValue> Ops) { 7995 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); 7996 // Reset the NodeID to -1. 7997 New->setNodeId(-1); 7998 if (New != N) { 7999 ReplaceAllUsesWith(N, New); 8000 RemoveDeadNode(N); 8001 } 8002 return New; 8003 } 8004 8005 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away 8006 /// the line number information on the merged node since it is not possible to 8007 /// preserve the information that operation is associated with multiple lines. 8008 /// This will make the debugger working better at -O0, were there is a higher 8009 /// probability having other instructions associated with that line. 8010 /// 8011 /// For IROrder, we keep the smaller of the two 8012 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { 8013 DebugLoc NLoc = N->getDebugLoc(); 8014 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { 8015 N->setDebugLoc(DebugLoc()); 8016 } 8017 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 8018 N->setIROrder(Order); 8019 return N; 8020 } 8021 8022 /// MorphNodeTo - This *mutates* the specified node to have the specified 8023 /// return type, opcode, and operands. 8024 /// 8025 /// Note that MorphNodeTo returns the resultant node. If there is already a 8026 /// node of the specified opcode and operands, it returns that node instead of 8027 /// the current one. Note that the SDLoc need not be the same. 8028 /// 8029 /// Using MorphNodeTo is faster than creating a new node and swapping it in 8030 /// with ReplaceAllUsesWith both because it often avoids allocating a new 8031 /// node, and because it doesn't require CSE recalculation for any of 8032 /// the node's users. 8033 /// 8034 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. 8035 /// As a consequence it isn't appropriate to use from within the DAG combiner or 8036 /// the legalizer which maintain worklists that would need to be updated when 8037 /// deleting things. 8038 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 8039 SDVTList VTs, ArrayRef<SDValue> Ops) { 8040 // If an identical node already exists, use it. 8041 void *IP = nullptr; 8042 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 8043 FoldingSetNodeID ID; 8044 AddNodeIDNode(ID, Opc, VTs, Ops); 8045 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) 8046 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); 8047 } 8048 8049 if (!RemoveNodeFromCSEMaps(N)) 8050 IP = nullptr; 8051 8052 // Start the morphing. 8053 N->NodeType = Opc; 8054 N->ValueList = VTs.VTs; 8055 N->NumValues = VTs.NumVTs; 8056 8057 // Clear the operands list, updating used nodes to remove this from their 8058 // use list. Keep track of any operands that become dead as a result. 8059 SmallPtrSet<SDNode*, 16> DeadNodeSet; 8060 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 8061 SDUse &Use = *I++; 8062 SDNode *Used = Use.getNode(); 8063 Use.set(SDValue()); 8064 if (Used->use_empty()) 8065 DeadNodeSet.insert(Used); 8066 } 8067 8068 // For MachineNode, initialize the memory references information. 8069 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) 8070 MN->clearMemRefs(); 8071 8072 // Swap for an appropriately sized array from the recycler. 8073 removeOperands(N); 8074 createOperands(N, Ops); 8075 8076 // Delete any nodes that are still dead after adding the uses for the 8077 // new operands. 8078 if (!DeadNodeSet.empty()) { 8079 SmallVector<SDNode *, 16> DeadNodes; 8080 for (SDNode *N : DeadNodeSet) 8081 if (N->use_empty()) 8082 DeadNodes.push_back(N); 8083 RemoveDeadNodes(DeadNodes); 8084 } 8085 8086 if (IP) 8087 CSEMap.InsertNode(N, IP); // Memoize the new node. 8088 return N; 8089 } 8090 8091 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { 8092 unsigned OrigOpc = Node->getOpcode(); 8093 unsigned NewOpc; 8094 switch (OrigOpc) { 8095 default: 8096 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); 8097 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 8098 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; 8099 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 8100 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; 8101 #include "llvm/IR/ConstrainedOps.def" 8102 } 8103 8104 assert(Node->getNumValues() == 2 && "Unexpected number of results!"); 8105 8106 // We're taking this node out of the chain, so we need to re-link things. 8107 SDValue InputChain = Node->getOperand(0); 8108 SDValue OutputChain = SDValue(Node, 1); 8109 ReplaceAllUsesOfValueWith(OutputChain, InputChain); 8110 8111 SmallVector<SDValue, 3> Ops; 8112 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) 8113 Ops.push_back(Node->getOperand(i)); 8114 8115 SDVTList VTs = getVTList(Node->getValueType(0)); 8116 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops); 8117 8118 // MorphNodeTo can operate in two ways: if an existing node with the 8119 // specified operands exists, it can just return it. Otherwise, it 8120 // updates the node in place to have the requested operands. 8121 if (Res == Node) { 8122 // If we updated the node in place, reset the node ID. To the isel, 8123 // this should be just like a newly allocated machine node. 8124 Res->setNodeId(-1); 8125 } else { 8126 ReplaceAllUsesWith(Node, Res); 8127 RemoveDeadNode(Node); 8128 } 8129 8130 return Res; 8131 } 8132 8133 /// getMachineNode - These are used for target selectors to create a new node 8134 /// with specified return type(s), MachineInstr opcode, and operands. 8135 /// 8136 /// Note that getMachineNode returns the resultant node. If there is already a 8137 /// node of the specified opcode and operands, it returns that node instead of 8138 /// the current one. 8139 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8140 EVT VT) { 8141 SDVTList VTs = getVTList(VT); 8142 return getMachineNode(Opcode, dl, VTs, None); 8143 } 8144 8145 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8146 EVT VT, SDValue Op1) { 8147 SDVTList VTs = getVTList(VT); 8148 SDValue Ops[] = { Op1 }; 8149 return getMachineNode(Opcode, dl, VTs, Ops); 8150 } 8151 8152 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8153 EVT VT, SDValue Op1, SDValue Op2) { 8154 SDVTList VTs = getVTList(VT); 8155 SDValue Ops[] = { Op1, Op2 }; 8156 return getMachineNode(Opcode, dl, VTs, Ops); 8157 } 8158 8159 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8160 EVT VT, SDValue Op1, SDValue Op2, 8161 SDValue Op3) { 8162 SDVTList VTs = getVTList(VT); 8163 SDValue Ops[] = { Op1, Op2, Op3 }; 8164 return getMachineNode(Opcode, dl, VTs, Ops); 8165 } 8166 8167 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8168 EVT VT, ArrayRef<SDValue> Ops) { 8169 SDVTList VTs = getVTList(VT); 8170 return getMachineNode(Opcode, dl, VTs, Ops); 8171 } 8172 8173 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8174 EVT VT1, EVT VT2, SDValue Op1, 8175 SDValue Op2) { 8176 SDVTList VTs = getVTList(VT1, VT2); 8177 SDValue Ops[] = { Op1, Op2 }; 8178 return getMachineNode(Opcode, dl, VTs, Ops); 8179 } 8180 8181 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8182 EVT VT1, EVT VT2, SDValue Op1, 8183 SDValue Op2, SDValue Op3) { 8184 SDVTList VTs = getVTList(VT1, VT2); 8185 SDValue Ops[] = { Op1, Op2, Op3 }; 8186 return getMachineNode(Opcode, dl, VTs, Ops); 8187 } 8188 8189 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8190 EVT VT1, EVT VT2, 8191 ArrayRef<SDValue> Ops) { 8192 SDVTList VTs = getVTList(VT1, VT2); 8193 return getMachineNode(Opcode, dl, VTs, Ops); 8194 } 8195 8196 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8197 EVT VT1, EVT VT2, EVT VT3, 8198 SDValue Op1, SDValue Op2) { 8199 SDVTList VTs = getVTList(VT1, VT2, VT3); 8200 SDValue Ops[] = { Op1, Op2 }; 8201 return getMachineNode(Opcode, dl, VTs, Ops); 8202 } 8203 8204 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8205 EVT VT1, EVT VT2, EVT VT3, 8206 SDValue Op1, SDValue Op2, 8207 SDValue Op3) { 8208 SDVTList VTs = getVTList(VT1, VT2, VT3); 8209 SDValue Ops[] = { Op1, Op2, Op3 }; 8210 return getMachineNode(Opcode, dl, VTs, Ops); 8211 } 8212 8213 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8214 EVT VT1, EVT VT2, EVT VT3, 8215 ArrayRef<SDValue> Ops) { 8216 SDVTList VTs = getVTList(VT1, VT2, VT3); 8217 return getMachineNode(Opcode, dl, VTs, Ops); 8218 } 8219 8220 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8221 ArrayRef<EVT> ResultTys, 8222 ArrayRef<SDValue> Ops) { 8223 SDVTList VTs = getVTList(ResultTys); 8224 return getMachineNode(Opcode, dl, VTs, Ops); 8225 } 8226 8227 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, 8228 SDVTList VTs, 8229 ArrayRef<SDValue> Ops) { 8230 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 8231 MachineSDNode *N; 8232 void *IP = nullptr; 8233 8234 if (DoCSE) { 8235 FoldingSetNodeID ID; 8236 AddNodeIDNode(ID, ~Opcode, VTs, Ops); 8237 IP = nullptr; 8238 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 8239 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL)); 8240 } 8241 } 8242 8243 // Allocate a new MachineSDNode. 8244 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 8245 createOperands(N, Ops); 8246 8247 if (DoCSE) 8248 CSEMap.InsertNode(N, IP); 8249 8250 InsertNode(N); 8251 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this); 8252 return N; 8253 } 8254 8255 /// getTargetExtractSubreg - A convenience function for creating 8256 /// TargetOpcode::EXTRACT_SUBREG nodes. 8257 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 8258 SDValue Operand) { 8259 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 8260 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 8261 VT, Operand, SRIdxVal); 8262 return SDValue(Subreg, 0); 8263 } 8264 8265 /// getTargetInsertSubreg - A convenience function for creating 8266 /// TargetOpcode::INSERT_SUBREG nodes. 8267 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 8268 SDValue Operand, SDValue Subreg) { 8269 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 8270 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 8271 VT, Operand, Subreg, SRIdxVal); 8272 return SDValue(Result, 0); 8273 } 8274 8275 /// getNodeIfExists - Get the specified node if it's already available, or 8276 /// else return NULL. 8277 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 8278 ArrayRef<SDValue> Ops) { 8279 SDNodeFlags Flags; 8280 if (Inserter) 8281 Flags = Inserter->getFlags(); 8282 return getNodeIfExists(Opcode, VTList, Ops, Flags); 8283 } 8284 8285 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 8286 ArrayRef<SDValue> Ops, 8287 const SDNodeFlags Flags) { 8288 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { 8289 FoldingSetNodeID ID; 8290 AddNodeIDNode(ID, Opcode, VTList, Ops); 8291 void *IP = nullptr; 8292 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { 8293 E->intersectFlagsWith(Flags); 8294 return E; 8295 } 8296 } 8297 return nullptr; 8298 } 8299 8300 /// getDbgValue - Creates a SDDbgValue node. 8301 /// 8302 /// SDNode 8303 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, 8304 SDNode *N, unsigned R, bool IsIndirect, 8305 const DebugLoc &DL, unsigned O) { 8306 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8307 "Expected inlined-at fields to agree"); 8308 return new (DbgInfo->getAlloc()) 8309 SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O); 8310 } 8311 8312 /// Constant 8313 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, 8314 DIExpression *Expr, 8315 const Value *C, 8316 const DebugLoc &DL, unsigned O) { 8317 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8318 "Expected inlined-at fields to agree"); 8319 return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O); 8320 } 8321 8322 /// FrameIndex 8323 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, 8324 DIExpression *Expr, unsigned FI, 8325 bool IsIndirect, 8326 const DebugLoc &DL, 8327 unsigned O) { 8328 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8329 "Expected inlined-at fields to agree"); 8330 return new (DbgInfo->getAlloc()) 8331 SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX); 8332 } 8333 8334 /// VReg 8335 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, 8336 DIExpression *Expr, 8337 unsigned VReg, bool IsIndirect, 8338 const DebugLoc &DL, unsigned O) { 8339 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8340 "Expected inlined-at fields to agree"); 8341 return new (DbgInfo->getAlloc()) 8342 SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG); 8343 } 8344 8345 void SelectionDAG::transferDbgValues(SDValue From, SDValue To, 8346 unsigned OffsetInBits, unsigned SizeInBits, 8347 bool InvalidateDbg) { 8348 SDNode *FromNode = From.getNode(); 8349 SDNode *ToNode = To.getNode(); 8350 assert(FromNode && ToNode && "Can't modify dbg values"); 8351 8352 // PR35338 8353 // TODO: assert(From != To && "Redundant dbg value transfer"); 8354 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); 8355 if (From == To || FromNode == ToNode) 8356 return; 8357 8358 if (!FromNode->getHasDebugValue()) 8359 return; 8360 8361 SmallVector<SDDbgValue *, 2> ClonedDVs; 8362 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) { 8363 if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated()) 8364 continue; 8365 8366 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); 8367 8368 // Just transfer the dbg value attached to From. 8369 if (Dbg->getResNo() != From.getResNo()) 8370 continue; 8371 8372 DIVariable *Var = Dbg->getVariable(); 8373 auto *Expr = Dbg->getExpression(); 8374 // If a fragment is requested, update the expression. 8375 if (SizeInBits) { 8376 // When splitting a larger (e.g., sign-extended) value whose 8377 // lower bits are described with an SDDbgValue, do not attempt 8378 // to transfer the SDDbgValue to the upper bits. 8379 if (auto FI = Expr->getFragmentInfo()) 8380 if (OffsetInBits + SizeInBits > FI->SizeInBits) 8381 continue; 8382 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, 8383 SizeInBits); 8384 if (!Fragment) 8385 continue; 8386 Expr = *Fragment; 8387 } 8388 // Clone the SDDbgValue and move it to To. 8389 SDDbgValue *Clone = getDbgValue( 8390 Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), Dbg->getDebugLoc(), 8391 std::max(ToNode->getIROrder(), Dbg->getOrder())); 8392 ClonedDVs.push_back(Clone); 8393 8394 if (InvalidateDbg) { 8395 // Invalidate value and indicate the SDDbgValue should not be emitted. 8396 Dbg->setIsInvalidated(); 8397 Dbg->setIsEmitted(); 8398 } 8399 } 8400 8401 for (SDDbgValue *Dbg : ClonedDVs) 8402 AddDbgValue(Dbg, ToNode, false); 8403 } 8404 8405 void SelectionDAG::salvageDebugInfo(SDNode &N) { 8406 if (!N.getHasDebugValue()) 8407 return; 8408 8409 SmallVector<SDDbgValue *, 2> ClonedDVs; 8410 for (auto DV : GetDbgValues(&N)) { 8411 if (DV->isInvalidated()) 8412 continue; 8413 switch (N.getOpcode()) { 8414 default: 8415 break; 8416 case ISD::ADD: 8417 SDValue N0 = N.getOperand(0); 8418 SDValue N1 = N.getOperand(1); 8419 if (!isConstantIntBuildVectorOrConstantInt(N0) && 8420 isConstantIntBuildVectorOrConstantInt(N1)) { 8421 uint64_t Offset = N.getConstantOperandVal(1); 8422 // Rewrite an ADD constant node into a DIExpression. Since we are 8423 // performing arithmetic to compute the variable's *value* in the 8424 // DIExpression, we need to mark the expression with a 8425 // DW_OP_stack_value. 8426 auto *DIExpr = DV->getExpression(); 8427 DIExpr = 8428 DIExpression::prepend(DIExpr, DIExpression::StackValue, Offset); 8429 SDDbgValue *Clone = 8430 getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(), 8431 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder()); 8432 ClonedDVs.push_back(Clone); 8433 DV->setIsInvalidated(); 8434 DV->setIsEmitted(); 8435 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; 8436 N0.getNode()->dumprFull(this); 8437 dbgs() << " into " << *DIExpr << '\n'); 8438 } 8439 } 8440 } 8441 8442 for (SDDbgValue *Dbg : ClonedDVs) 8443 AddDbgValue(Dbg, Dbg->getSDNode(), false); 8444 } 8445 8446 /// Creates a SDDbgLabel node. 8447 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, 8448 const DebugLoc &DL, unsigned O) { 8449 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && 8450 "Expected inlined-at fields to agree"); 8451 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); 8452 } 8453 8454 namespace { 8455 8456 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 8457 /// pointed to by a use iterator is deleted, increment the use iterator 8458 /// so that it doesn't dangle. 8459 /// 8460 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 8461 SDNode::use_iterator &UI; 8462 SDNode::use_iterator &UE; 8463 8464 void NodeDeleted(SDNode *N, SDNode *E) override { 8465 // Increment the iterator as needed. 8466 while (UI != UE && N == *UI) 8467 ++UI; 8468 } 8469 8470 public: 8471 RAUWUpdateListener(SelectionDAG &d, 8472 SDNode::use_iterator &ui, 8473 SDNode::use_iterator &ue) 8474 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 8475 }; 8476 8477 } // end anonymous namespace 8478 8479 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 8480 /// This can cause recursive merging of nodes in the DAG. 8481 /// 8482 /// This version assumes From has a single result value. 8483 /// 8484 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 8485 SDNode *From = FromN.getNode(); 8486 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 8487 "Cannot replace with this method!"); 8488 assert(From != To.getNode() && "Cannot replace uses of with self"); 8489 8490 // Preserve Debug Values 8491 transferDbgValues(FromN, To); 8492 8493 // Iterate over all the existing uses of From. New uses will be added 8494 // to the beginning of the use list, which we avoid visiting. 8495 // This specifically avoids visiting uses of From that arise while the 8496 // replacement is happening, because any such uses would be the result 8497 // of CSE: If an existing node looks like From after one of its operands 8498 // is replaced by To, we don't want to replace of all its users with To 8499 // too. See PR3018 for more info. 8500 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 8501 RAUWUpdateListener Listener(*this, UI, UE); 8502 while (UI != UE) { 8503 SDNode *User = *UI; 8504 8505 // This node is about to morph, remove its old self from the CSE maps. 8506 RemoveNodeFromCSEMaps(User); 8507 8508 // A user can appear in a use list multiple times, and when this 8509 // happens the uses are usually next to each other in the list. 8510 // To help reduce the number of CSE recomputations, process all 8511 // the uses of this user that we can find this way. 8512 do { 8513 SDUse &Use = UI.getUse(); 8514 ++UI; 8515 Use.set(To); 8516 if (To->isDivergent() != From->isDivergent()) 8517 updateDivergence(User); 8518 } while (UI != UE && *UI == User); 8519 // Now that we have modified User, add it back to the CSE maps. If it 8520 // already exists there, recursively merge the results together. 8521 AddModifiedNodeToCSEMaps(User); 8522 } 8523 8524 // If we just RAUW'd the root, take note. 8525 if (FromN == getRoot()) 8526 setRoot(To); 8527 } 8528 8529 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 8530 /// This can cause recursive merging of nodes in the DAG. 8531 /// 8532 /// This version assumes that for each value of From, there is a 8533 /// corresponding value in To in the same position with the same type. 8534 /// 8535 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 8536 #ifndef NDEBUG 8537 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 8538 assert((!From->hasAnyUseOfValue(i) || 8539 From->getValueType(i) == To->getValueType(i)) && 8540 "Cannot use this version of ReplaceAllUsesWith!"); 8541 #endif 8542 8543 // Handle the trivial case. 8544 if (From == To) 8545 return; 8546 8547 // Preserve Debug Info. Only do this if there's a use. 8548 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 8549 if (From->hasAnyUseOfValue(i)) { 8550 assert((i < To->getNumValues()) && "Invalid To location"); 8551 transferDbgValues(SDValue(From, i), SDValue(To, i)); 8552 } 8553 8554 // Iterate over just the existing users of From. See the comments in 8555 // the ReplaceAllUsesWith above. 8556 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 8557 RAUWUpdateListener Listener(*this, UI, UE); 8558 while (UI != UE) { 8559 SDNode *User = *UI; 8560 8561 // This node is about to morph, remove its old self from the CSE maps. 8562 RemoveNodeFromCSEMaps(User); 8563 8564 // A user can appear in a use list multiple times, and when this 8565 // happens the uses are usually next to each other in the list. 8566 // To help reduce the number of CSE recomputations, process all 8567 // the uses of this user that we can find this way. 8568 do { 8569 SDUse &Use = UI.getUse(); 8570 ++UI; 8571 Use.setNode(To); 8572 if (To->isDivergent() != From->isDivergent()) 8573 updateDivergence(User); 8574 } while (UI != UE && *UI == User); 8575 8576 // Now that we have modified User, add it back to the CSE maps. If it 8577 // already exists there, recursively merge the results together. 8578 AddModifiedNodeToCSEMaps(User); 8579 } 8580 8581 // If we just RAUW'd the root, take note. 8582 if (From == getRoot().getNode()) 8583 setRoot(SDValue(To, getRoot().getResNo())); 8584 } 8585 8586 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 8587 /// This can cause recursive merging of nodes in the DAG. 8588 /// 8589 /// This version can replace From with any result values. To must match the 8590 /// number and types of values returned by From. 8591 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 8592 if (From->getNumValues() == 1) // Handle the simple case efficiently. 8593 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 8594 8595 // Preserve Debug Info. 8596 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 8597 transferDbgValues(SDValue(From, i), To[i]); 8598 8599 // Iterate over just the existing users of From. See the comments in 8600 // the ReplaceAllUsesWith above. 8601 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 8602 RAUWUpdateListener Listener(*this, UI, UE); 8603 while (UI != UE) { 8604 SDNode *User = *UI; 8605 8606 // This node is about to morph, remove its old self from the CSE maps. 8607 RemoveNodeFromCSEMaps(User); 8608 8609 // A user can appear in a use list multiple times, and when this happens the 8610 // uses are usually next to each other in the list. To help reduce the 8611 // number of CSE and divergence recomputations, process all the uses of this 8612 // user that we can find this way. 8613 bool To_IsDivergent = false; 8614 do { 8615 SDUse &Use = UI.getUse(); 8616 const SDValue &ToOp = To[Use.getResNo()]; 8617 ++UI; 8618 Use.set(ToOp); 8619 To_IsDivergent |= ToOp->isDivergent(); 8620 } while (UI != UE && *UI == User); 8621 8622 if (To_IsDivergent != From->isDivergent()) 8623 updateDivergence(User); 8624 8625 // Now that we have modified User, add it back to the CSE maps. If it 8626 // already exists there, recursively merge the results together. 8627 AddModifiedNodeToCSEMaps(User); 8628 } 8629 8630 // If we just RAUW'd the root, take note. 8631 if (From == getRoot().getNode()) 8632 setRoot(SDValue(To[getRoot().getResNo()])); 8633 } 8634 8635 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 8636 /// uses of other values produced by From.getNode() alone. The Deleted 8637 /// vector is handled the same way as for ReplaceAllUsesWith. 8638 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 8639 // Handle the really simple, really trivial case efficiently. 8640 if (From == To) return; 8641 8642 // Handle the simple, trivial, case efficiently. 8643 if (From.getNode()->getNumValues() == 1) { 8644 ReplaceAllUsesWith(From, To); 8645 return; 8646 } 8647 8648 // Preserve Debug Info. 8649 transferDbgValues(From, To); 8650 8651 // Iterate over just the existing users of From. See the comments in 8652 // the ReplaceAllUsesWith above. 8653 SDNode::use_iterator UI = From.getNode()->use_begin(), 8654 UE = From.getNode()->use_end(); 8655 RAUWUpdateListener Listener(*this, UI, UE); 8656 while (UI != UE) { 8657 SDNode *User = *UI; 8658 bool UserRemovedFromCSEMaps = false; 8659 8660 // A user can appear in a use list multiple times, and when this 8661 // happens the uses are usually next to each other in the list. 8662 // To help reduce the number of CSE recomputations, process all 8663 // the uses of this user that we can find this way. 8664 do { 8665 SDUse &Use = UI.getUse(); 8666 8667 // Skip uses of different values from the same node. 8668 if (Use.getResNo() != From.getResNo()) { 8669 ++UI; 8670 continue; 8671 } 8672 8673 // If this node hasn't been modified yet, it's still in the CSE maps, 8674 // so remove its old self from the CSE maps. 8675 if (!UserRemovedFromCSEMaps) { 8676 RemoveNodeFromCSEMaps(User); 8677 UserRemovedFromCSEMaps = true; 8678 } 8679 8680 ++UI; 8681 Use.set(To); 8682 if (To->isDivergent() != From->isDivergent()) 8683 updateDivergence(User); 8684 } while (UI != UE && *UI == User); 8685 // We are iterating over all uses of the From node, so if a use 8686 // doesn't use the specific value, no changes are made. 8687 if (!UserRemovedFromCSEMaps) 8688 continue; 8689 8690 // Now that we have modified User, add it back to the CSE maps. If it 8691 // already exists there, recursively merge the results together. 8692 AddModifiedNodeToCSEMaps(User); 8693 } 8694 8695 // If we just RAUW'd the root, take note. 8696 if (From == getRoot()) 8697 setRoot(To); 8698 } 8699 8700 namespace { 8701 8702 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 8703 /// to record information about a use. 8704 struct UseMemo { 8705 SDNode *User; 8706 unsigned Index; 8707 SDUse *Use; 8708 }; 8709 8710 /// operator< - Sort Memos by User. 8711 bool operator<(const UseMemo &L, const UseMemo &R) { 8712 return (intptr_t)L.User < (intptr_t)R.User; 8713 } 8714 8715 } // end anonymous namespace 8716 8717 bool SelectionDAG::calculateDivergence(SDNode *N) { 8718 if (TLI->isSDNodeAlwaysUniform(N)) { 8719 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && 8720 "Conflicting divergence information!"); 8721 return false; 8722 } 8723 if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) 8724 return true; 8725 for (auto &Op : N->ops()) { 8726 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) 8727 return true; 8728 } 8729 return false; 8730 } 8731 8732 void SelectionDAG::updateDivergence(SDNode *N) { 8733 SmallVector<SDNode *, 16> Worklist(1, N); 8734 do { 8735 N = Worklist.pop_back_val(); 8736 bool IsDivergent = calculateDivergence(N); 8737 if (N->SDNodeBits.IsDivergent != IsDivergent) { 8738 N->SDNodeBits.IsDivergent = IsDivergent; 8739 Worklist.insert(Worklist.end(), N->use_begin(), N->use_end()); 8740 } 8741 } while (!Worklist.empty()); 8742 } 8743 8744 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { 8745 DenseMap<SDNode *, unsigned> Degree; 8746 Order.reserve(AllNodes.size()); 8747 for (auto &N : allnodes()) { 8748 unsigned NOps = N.getNumOperands(); 8749 Degree[&N] = NOps; 8750 if (0 == NOps) 8751 Order.push_back(&N); 8752 } 8753 for (size_t I = 0; I != Order.size(); ++I) { 8754 SDNode *N = Order[I]; 8755 for (auto U : N->uses()) { 8756 unsigned &UnsortedOps = Degree[U]; 8757 if (0 == --UnsortedOps) 8758 Order.push_back(U); 8759 } 8760 } 8761 } 8762 8763 #ifndef NDEBUG 8764 void SelectionDAG::VerifyDAGDiverence() { 8765 std::vector<SDNode *> TopoOrder; 8766 CreateTopologicalOrder(TopoOrder); 8767 for (auto *N : TopoOrder) { 8768 assert(calculateDivergence(N) == N->isDivergent() && 8769 "Divergence bit inconsistency detected"); 8770 } 8771 } 8772 #endif 8773 8774 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 8775 /// uses of other values produced by From.getNode() alone. The same value 8776 /// may appear in both the From and To list. The Deleted vector is 8777 /// handled the same way as for ReplaceAllUsesWith. 8778 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 8779 const SDValue *To, 8780 unsigned Num){ 8781 // Handle the simple, trivial case efficiently. 8782 if (Num == 1) 8783 return ReplaceAllUsesOfValueWith(*From, *To); 8784 8785 transferDbgValues(*From, *To); 8786 8787 // Read up all the uses and make records of them. This helps 8788 // processing new uses that are introduced during the 8789 // replacement process. 8790 SmallVector<UseMemo, 4> Uses; 8791 for (unsigned i = 0; i != Num; ++i) { 8792 unsigned FromResNo = From[i].getResNo(); 8793 SDNode *FromNode = From[i].getNode(); 8794 for (SDNode::use_iterator UI = FromNode->use_begin(), 8795 E = FromNode->use_end(); UI != E; ++UI) { 8796 SDUse &Use = UI.getUse(); 8797 if (Use.getResNo() == FromResNo) { 8798 UseMemo Memo = { *UI, i, &Use }; 8799 Uses.push_back(Memo); 8800 } 8801 } 8802 } 8803 8804 // Sort the uses, so that all the uses from a given User are together. 8805 llvm::sort(Uses); 8806 8807 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 8808 UseIndex != UseIndexEnd; ) { 8809 // We know that this user uses some value of From. If it is the right 8810 // value, update it. 8811 SDNode *User = Uses[UseIndex].User; 8812 8813 // This node is about to morph, remove its old self from the CSE maps. 8814 RemoveNodeFromCSEMaps(User); 8815 8816 // The Uses array is sorted, so all the uses for a given User 8817 // are next to each other in the list. 8818 // To help reduce the number of CSE recomputations, process all 8819 // the uses of this user that we can find this way. 8820 do { 8821 unsigned i = Uses[UseIndex].Index; 8822 SDUse &Use = *Uses[UseIndex].Use; 8823 ++UseIndex; 8824 8825 Use.set(To[i]); 8826 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 8827 8828 // Now that we have modified User, add it back to the CSE maps. If it 8829 // already exists there, recursively merge the results together. 8830 AddModifiedNodeToCSEMaps(User); 8831 } 8832 } 8833 8834 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 8835 /// based on their topological order. It returns the maximum id and a vector 8836 /// of the SDNodes* in assigned order by reference. 8837 unsigned SelectionDAG::AssignTopologicalOrder() { 8838 unsigned DAGSize = 0; 8839 8840 // SortedPos tracks the progress of the algorithm. Nodes before it are 8841 // sorted, nodes after it are unsorted. When the algorithm completes 8842 // it is at the end of the list. 8843 allnodes_iterator SortedPos = allnodes_begin(); 8844 8845 // Visit all the nodes. Move nodes with no operands to the front of 8846 // the list immediately. Annotate nodes that do have operands with their 8847 // operand count. Before we do this, the Node Id fields of the nodes 8848 // may contain arbitrary values. After, the Node Id fields for nodes 8849 // before SortedPos will contain the topological sort index, and the 8850 // Node Id fields for nodes At SortedPos and after will contain the 8851 // count of outstanding operands. 8852 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 8853 SDNode *N = &*I++; 8854 checkForCycles(N, this); 8855 unsigned Degree = N->getNumOperands(); 8856 if (Degree == 0) { 8857 // A node with no uses, add it to the result array immediately. 8858 N->setNodeId(DAGSize++); 8859 allnodes_iterator Q(N); 8860 if (Q != SortedPos) 8861 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 8862 assert(SortedPos != AllNodes.end() && "Overran node list"); 8863 ++SortedPos; 8864 } else { 8865 // Temporarily use the Node Id as scratch space for the degree count. 8866 N->setNodeId(Degree); 8867 } 8868 } 8869 8870 // Visit all the nodes. As we iterate, move nodes into sorted order, 8871 // such that by the time the end is reached all nodes will be sorted. 8872 for (SDNode &Node : allnodes()) { 8873 SDNode *N = &Node; 8874 checkForCycles(N, this); 8875 // N is in sorted position, so all its uses have one less operand 8876 // that needs to be sorted. 8877 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 8878 UI != UE; ++UI) { 8879 SDNode *P = *UI; 8880 unsigned Degree = P->getNodeId(); 8881 assert(Degree != 0 && "Invalid node degree"); 8882 --Degree; 8883 if (Degree == 0) { 8884 // All of P's operands are sorted, so P may sorted now. 8885 P->setNodeId(DAGSize++); 8886 if (P->getIterator() != SortedPos) 8887 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 8888 assert(SortedPos != AllNodes.end() && "Overran node list"); 8889 ++SortedPos; 8890 } else { 8891 // Update P's outstanding operand count. 8892 P->setNodeId(Degree); 8893 } 8894 } 8895 if (Node.getIterator() == SortedPos) { 8896 #ifndef NDEBUG 8897 allnodes_iterator I(N); 8898 SDNode *S = &*++I; 8899 dbgs() << "Overran sorted position:\n"; 8900 S->dumprFull(this); dbgs() << "\n"; 8901 dbgs() << "Checking if this is due to cycles\n"; 8902 checkForCycles(this, true); 8903 #endif 8904 llvm_unreachable(nullptr); 8905 } 8906 } 8907 8908 assert(SortedPos == AllNodes.end() && 8909 "Topological sort incomplete!"); 8910 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 8911 "First node in topological sort is not the entry token!"); 8912 assert(AllNodes.front().getNodeId() == 0 && 8913 "First node in topological sort has non-zero id!"); 8914 assert(AllNodes.front().getNumOperands() == 0 && 8915 "First node in topological sort has operands!"); 8916 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 8917 "Last node in topologic sort has unexpected id!"); 8918 assert(AllNodes.back().use_empty() && 8919 "Last node in topologic sort has users!"); 8920 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 8921 return DAGSize; 8922 } 8923 8924 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 8925 /// value is produced by SD. 8926 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 8927 if (SD) { 8928 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); 8929 SD->setHasDebugValue(true); 8930 } 8931 DbgInfo->add(DB, SD, isParameter); 8932 } 8933 8934 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { 8935 DbgInfo->add(DB); 8936 } 8937 8938 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, 8939 SDValue NewMemOp) { 8940 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); 8941 // The new memory operation must have the same position as the old load in 8942 // terms of memory dependency. Create a TokenFactor for the old load and new 8943 // memory operation and update uses of the old load's output chain to use that 8944 // TokenFactor. 8945 SDValue OldChain = SDValue(OldLoad, 1); 8946 SDValue NewChain = SDValue(NewMemOp.getNode(), 1); 8947 if (OldChain == NewChain || !OldLoad->hasAnyUseOfValue(1)) 8948 return NewChain; 8949 8950 SDValue TokenFactor = 8951 getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain); 8952 ReplaceAllUsesOfValueWith(OldChain, TokenFactor); 8953 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain); 8954 return TokenFactor; 8955 } 8956 8957 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, 8958 Function **OutFunction) { 8959 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol"); 8960 8961 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 8962 auto *Module = MF->getFunction().getParent(); 8963 auto *Function = Module->getFunction(Symbol); 8964 8965 if (OutFunction != nullptr) 8966 *OutFunction = Function; 8967 8968 if (Function != nullptr) { 8969 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace()); 8970 return getGlobalAddress(Function, SDLoc(Op), PtrTy); 8971 } 8972 8973 std::string ErrorStr; 8974 raw_string_ostream ErrorFormatter(ErrorStr); 8975 8976 ErrorFormatter << "Undefined external symbol "; 8977 ErrorFormatter << '"' << Symbol << '"'; 8978 ErrorFormatter.flush(); 8979 8980 report_fatal_error(ErrorStr); 8981 } 8982 8983 //===----------------------------------------------------------------------===// 8984 // SDNode Class 8985 //===----------------------------------------------------------------------===// 8986 8987 bool llvm::isNullConstant(SDValue V) { 8988 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 8989 return Const != nullptr && Const->isNullValue(); 8990 } 8991 8992 bool llvm::isNullFPConstant(SDValue V) { 8993 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V); 8994 return Const != nullptr && Const->isZero() && !Const->isNegative(); 8995 } 8996 8997 bool llvm::isAllOnesConstant(SDValue V) { 8998 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 8999 return Const != nullptr && Const->isAllOnesValue(); 9000 } 9001 9002 bool llvm::isOneConstant(SDValue V) { 9003 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 9004 return Const != nullptr && Const->isOne(); 9005 } 9006 9007 SDValue llvm::peekThroughBitcasts(SDValue V) { 9008 while (V.getOpcode() == ISD::BITCAST) 9009 V = V.getOperand(0); 9010 return V; 9011 } 9012 9013 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { 9014 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) 9015 V = V.getOperand(0); 9016 return V; 9017 } 9018 9019 SDValue llvm::peekThroughExtractSubvectors(SDValue V) { 9020 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) 9021 V = V.getOperand(0); 9022 return V; 9023 } 9024 9025 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { 9026 if (V.getOpcode() != ISD::XOR) 9027 return false; 9028 V = peekThroughBitcasts(V.getOperand(1)); 9029 unsigned NumBits = V.getScalarValueSizeInBits(); 9030 ConstantSDNode *C = 9031 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true); 9032 return C && (C->getAPIntValue().countTrailingOnes() >= NumBits); 9033 } 9034 9035 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, 9036 bool AllowTruncation) { 9037 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 9038 return CN; 9039 9040 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9041 BitVector UndefElements; 9042 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); 9043 9044 // BuildVectors can truncate their operands. Ignore that case here unless 9045 // AllowTruncation is set. 9046 if (CN && (UndefElements.none() || AllowUndefs)) { 9047 EVT CVT = CN->getValueType(0); 9048 EVT NSVT = N.getValueType().getScalarType(); 9049 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); 9050 if (AllowTruncation || (CVT == NSVT)) 9051 return CN; 9052 } 9053 } 9054 9055 return nullptr; 9056 } 9057 9058 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, 9059 bool AllowUndefs, 9060 bool AllowTruncation) { 9061 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 9062 return CN; 9063 9064 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9065 BitVector UndefElements; 9066 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements); 9067 9068 // BuildVectors can truncate their operands. Ignore that case here unless 9069 // AllowTruncation is set. 9070 if (CN && (UndefElements.none() || AllowUndefs)) { 9071 EVT CVT = CN->getValueType(0); 9072 EVT NSVT = N.getValueType().getScalarType(); 9073 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); 9074 if (AllowTruncation || (CVT == NSVT)) 9075 return CN; 9076 } 9077 } 9078 9079 return nullptr; 9080 } 9081 9082 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { 9083 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) 9084 return CN; 9085 9086 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9087 BitVector UndefElements; 9088 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements); 9089 if (CN && (UndefElements.none() || AllowUndefs)) 9090 return CN; 9091 } 9092 9093 if (N.getOpcode() == ISD::SPLAT_VECTOR) 9094 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0))) 9095 return CN; 9096 9097 return nullptr; 9098 } 9099 9100 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, 9101 const APInt &DemandedElts, 9102 bool AllowUndefs) { 9103 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) 9104 return CN; 9105 9106 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9107 BitVector UndefElements; 9108 ConstantFPSDNode *CN = 9109 BV->getConstantFPSplatNode(DemandedElts, &UndefElements); 9110 if (CN && (UndefElements.none() || AllowUndefs)) 9111 return CN; 9112 } 9113 9114 return nullptr; 9115 } 9116 9117 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { 9118 // TODO: may want to use peekThroughBitcast() here. 9119 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); 9120 return C && C->isNullValue(); 9121 } 9122 9123 bool llvm::isOneOrOneSplat(SDValue N) { 9124 // TODO: may want to use peekThroughBitcast() here. 9125 unsigned BitWidth = N.getScalarValueSizeInBits(); 9126 ConstantSDNode *C = isConstOrConstSplat(N); 9127 return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth; 9128 } 9129 9130 bool llvm::isAllOnesOrAllOnesSplat(SDValue N) { 9131 N = peekThroughBitcasts(N); 9132 unsigned BitWidth = N.getScalarValueSizeInBits(); 9133 ConstantSDNode *C = isConstOrConstSplat(N); 9134 return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth; 9135 } 9136 9137 HandleSDNode::~HandleSDNode() { 9138 DropOperands(); 9139 } 9140 9141 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 9142 const DebugLoc &DL, 9143 const GlobalValue *GA, EVT VT, 9144 int64_t o, unsigned TF) 9145 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 9146 TheGlobal = GA; 9147 } 9148 9149 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, 9150 EVT VT, unsigned SrcAS, 9151 unsigned DestAS) 9152 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), 9153 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 9154 9155 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, 9156 SDVTList VTs, EVT memvt, MachineMemOperand *mmo) 9157 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 9158 MemSDNodeBits.IsVolatile = MMO->isVolatile(); 9159 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); 9160 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); 9161 MemSDNodeBits.IsInvariant = MMO->isInvariant(); 9162 9163 // We check here that the size of the memory operand fits within the size of 9164 // the MMO. This is because the MMO might indicate only a possible address 9165 // range instead of specifying the affected memory addresses precisely. 9166 // TODO: Make MachineMemOperands aware of scalable vectors. 9167 assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() && 9168 "Size mismatch!"); 9169 } 9170 9171 /// Profile - Gather unique data for the node. 9172 /// 9173 void SDNode::Profile(FoldingSetNodeID &ID) const { 9174 AddNodeIDNode(ID, this); 9175 } 9176 9177 namespace { 9178 9179 struct EVTArray { 9180 std::vector<EVT> VTs; 9181 9182 EVTArray() { 9183 VTs.reserve(MVT::LAST_VALUETYPE); 9184 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 9185 VTs.push_back(MVT((MVT::SimpleValueType)i)); 9186 } 9187 }; 9188 9189 } // end anonymous namespace 9190 9191 static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs; 9192 static ManagedStatic<EVTArray> SimpleVTArray; 9193 static ManagedStatic<sys::SmartMutex<true>> VTMutex; 9194 9195 /// getValueTypeList - Return a pointer to the specified value type. 9196 /// 9197 const EVT *SDNode::getValueTypeList(EVT VT) { 9198 if (VT.isExtended()) { 9199 sys::SmartScopedLock<true> Lock(*VTMutex); 9200 return &(*EVTs->insert(VT).first); 9201 } else { 9202 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 9203 "Value type out of range!"); 9204 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 9205 } 9206 } 9207 9208 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 9209 /// indicated value. This method ignores uses of other values defined by this 9210 /// operation. 9211 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 9212 assert(Value < getNumValues() && "Bad value!"); 9213 9214 // TODO: Only iterate over uses of a given value of the node 9215 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 9216 if (UI.getUse().getResNo() == Value) { 9217 if (NUses == 0) 9218 return false; 9219 --NUses; 9220 } 9221 } 9222 9223 // Found exactly the right number of uses? 9224 return NUses == 0; 9225 } 9226 9227 /// hasAnyUseOfValue - Return true if there are any use of the indicated 9228 /// value. This method ignores uses of other values defined by this operation. 9229 bool SDNode::hasAnyUseOfValue(unsigned Value) const { 9230 assert(Value < getNumValues() && "Bad value!"); 9231 9232 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 9233 if (UI.getUse().getResNo() == Value) 9234 return true; 9235 9236 return false; 9237 } 9238 9239 /// isOnlyUserOf - Return true if this node is the only use of N. 9240 bool SDNode::isOnlyUserOf(const SDNode *N) const { 9241 bool Seen = false; 9242 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 9243 SDNode *User = *I; 9244 if (User == this) 9245 Seen = true; 9246 else 9247 return false; 9248 } 9249 9250 return Seen; 9251 } 9252 9253 /// Return true if the only users of N are contained in Nodes. 9254 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { 9255 bool Seen = false; 9256 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 9257 SDNode *User = *I; 9258 if (llvm::any_of(Nodes, 9259 [&User](const SDNode *Node) { return User == Node; })) 9260 Seen = true; 9261 else 9262 return false; 9263 } 9264 9265 return Seen; 9266 } 9267 9268 /// isOperand - Return true if this node is an operand of N. 9269 bool SDValue::isOperandOf(const SDNode *N) const { 9270 return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; }); 9271 } 9272 9273 bool SDNode::isOperandOf(const SDNode *N) const { 9274 return any_of(N->op_values(), 9275 [this](SDValue Op) { return this == Op.getNode(); }); 9276 } 9277 9278 /// reachesChainWithoutSideEffects - Return true if this operand (which must 9279 /// be a chain) reaches the specified operand without crossing any 9280 /// side-effecting instructions on any chain path. In practice, this looks 9281 /// through token factors and non-volatile loads. In order to remain efficient, 9282 /// this only looks a couple of nodes in, it does not do an exhaustive search. 9283 /// 9284 /// Note that we only need to examine chains when we're searching for 9285 /// side-effects; SelectionDAG requires that all side-effects are represented 9286 /// by chains, even if another operand would force a specific ordering. This 9287 /// constraint is necessary to allow transformations like splitting loads. 9288 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 9289 unsigned Depth) const { 9290 if (*this == Dest) return true; 9291 9292 // Don't search too deeply, we just want to be able to see through 9293 // TokenFactor's etc. 9294 if (Depth == 0) return false; 9295 9296 // If this is a token factor, all inputs to the TF happen in parallel. 9297 if (getOpcode() == ISD::TokenFactor) { 9298 // First, try a shallow search. 9299 if (is_contained((*this)->ops(), Dest)) { 9300 // We found the chain we want as an operand of this TokenFactor. 9301 // Essentially, we reach the chain without side-effects if we could 9302 // serialize the TokenFactor into a simple chain of operations with 9303 // Dest as the last operation. This is automatically true if the 9304 // chain has one use: there are no other ordering constraints. 9305 // If the chain has more than one use, we give up: some other 9306 // use of Dest might force a side-effect between Dest and the current 9307 // node. 9308 if (Dest.hasOneUse()) 9309 return true; 9310 } 9311 // Next, try a deep search: check whether every operand of the TokenFactor 9312 // reaches Dest. 9313 return llvm::all_of((*this)->ops(), [=](SDValue Op) { 9314 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); 9315 }); 9316 } 9317 9318 // Loads don't have side effects, look through them. 9319 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 9320 if (Ld->isUnordered()) 9321 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 9322 } 9323 return false; 9324 } 9325 9326 bool SDNode::hasPredecessor(const SDNode *N) const { 9327 SmallPtrSet<const SDNode *, 32> Visited; 9328 SmallVector<const SDNode *, 16> Worklist; 9329 Worklist.push_back(this); 9330 return hasPredecessorHelper(N, Visited, Worklist); 9331 } 9332 9333 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { 9334 this->Flags.intersectWith(Flags); 9335 } 9336 9337 SDValue 9338 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, 9339 ArrayRef<ISD::NodeType> CandidateBinOps, 9340 bool AllowPartials) { 9341 // The pattern must end in an extract from index 0. 9342 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || 9343 !isNullConstant(Extract->getOperand(1))) 9344 return SDValue(); 9345 9346 // Match against one of the candidate binary ops. 9347 SDValue Op = Extract->getOperand(0); 9348 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { 9349 return Op.getOpcode() == unsigned(BinOp); 9350 })) 9351 return SDValue(); 9352 9353 // Floating-point reductions may require relaxed constraints on the final step 9354 // of the reduction because they may reorder intermediate operations. 9355 unsigned CandidateBinOp = Op.getOpcode(); 9356 if (Op.getValueType().isFloatingPoint()) { 9357 SDNodeFlags Flags = Op->getFlags(); 9358 switch (CandidateBinOp) { 9359 case ISD::FADD: 9360 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation()) 9361 return SDValue(); 9362 break; 9363 default: 9364 llvm_unreachable("Unhandled FP opcode for binop reduction"); 9365 } 9366 } 9367 9368 // Matching failed - attempt to see if we did enough stages that a partial 9369 // reduction from a subvector is possible. 9370 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) { 9371 if (!AllowPartials || !Op) 9372 return SDValue(); 9373 EVT OpVT = Op.getValueType(); 9374 EVT OpSVT = OpVT.getScalarType(); 9375 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts); 9376 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0)) 9377 return SDValue(); 9378 BinOp = (ISD::NodeType)CandidateBinOp; 9379 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, 9380 getVectorIdxConstant(0, SDLoc(Op))); 9381 }; 9382 9383 // At each stage, we're looking for something that looks like: 9384 // %s = shufflevector <8 x i32> %op, <8 x i32> undef, 9385 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, 9386 // i32 undef, i32 undef, i32 undef, i32 undef> 9387 // %a = binop <8 x i32> %op, %s 9388 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid, 9389 // we expect something like: 9390 // <4,5,6,7,u,u,u,u> 9391 // <2,3,u,u,u,u,u,u> 9392 // <1,u,u,u,u,u,u,u> 9393 // While a partial reduction match would be: 9394 // <2,3,u,u,u,u,u,u> 9395 // <1,u,u,u,u,u,u,u> 9396 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements()); 9397 SDValue PrevOp; 9398 for (unsigned i = 0; i < Stages; ++i) { 9399 unsigned MaskEnd = (1 << i); 9400 9401 if (Op.getOpcode() != CandidateBinOp) 9402 return PartialReduction(PrevOp, MaskEnd); 9403 9404 SDValue Op0 = Op.getOperand(0); 9405 SDValue Op1 = Op.getOperand(1); 9406 9407 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0); 9408 if (Shuffle) { 9409 Op = Op1; 9410 } else { 9411 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1); 9412 Op = Op0; 9413 } 9414 9415 // The first operand of the shuffle should be the same as the other operand 9416 // of the binop. 9417 if (!Shuffle || Shuffle->getOperand(0) != Op) 9418 return PartialReduction(PrevOp, MaskEnd); 9419 9420 // Verify the shuffle has the expected (at this stage of the pyramid) mask. 9421 for (int Index = 0; Index < (int)MaskEnd; ++Index) 9422 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index)) 9423 return PartialReduction(PrevOp, MaskEnd); 9424 9425 PrevOp = Op; 9426 } 9427 9428 // Handle subvector reductions, which tend to appear after the shuffle 9429 // reduction stages. 9430 while (Op.getOpcode() == CandidateBinOp) { 9431 unsigned NumElts = Op.getValueType().getVectorNumElements(); 9432 SDValue Op0 = Op.getOperand(0); 9433 SDValue Op1 = Op.getOperand(1); 9434 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || 9435 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || 9436 Op0.getOperand(0) != Op1.getOperand(0)) 9437 break; 9438 SDValue Src = Op0.getOperand(0); 9439 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 9440 if (NumSrcElts != (2 * NumElts)) 9441 break; 9442 if (!(Op0.getConstantOperandAPInt(1) == 0 && 9443 Op1.getConstantOperandAPInt(1) == NumElts) && 9444 !(Op1.getConstantOperandAPInt(1) == 0 && 9445 Op0.getConstantOperandAPInt(1) == NumElts)) 9446 break; 9447 Op = Src; 9448 } 9449 9450 BinOp = (ISD::NodeType)CandidateBinOp; 9451 return Op; 9452 } 9453 9454 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 9455 assert(N->getNumValues() == 1 && 9456 "Can't unroll a vector with multiple results!"); 9457 9458 EVT VT = N->getValueType(0); 9459 unsigned NE = VT.getVectorNumElements(); 9460 EVT EltVT = VT.getVectorElementType(); 9461 SDLoc dl(N); 9462 9463 SmallVector<SDValue, 8> Scalars; 9464 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 9465 9466 // If ResNE is 0, fully unroll the vector op. 9467 if (ResNE == 0) 9468 ResNE = NE; 9469 else if (NE > ResNE) 9470 NE = ResNE; 9471 9472 unsigned i; 9473 for (i= 0; i != NE; ++i) { 9474 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 9475 SDValue Operand = N->getOperand(j); 9476 EVT OperandVT = Operand.getValueType(); 9477 if (OperandVT.isVector()) { 9478 // A vector operand; extract a single element. 9479 EVT OperandEltVT = OperandVT.getVectorElementType(); 9480 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, 9481 Operand, getVectorIdxConstant(i, dl)); 9482 } else { 9483 // A scalar operand; just use it as is. 9484 Operands[j] = Operand; 9485 } 9486 } 9487 9488 switch (N->getOpcode()) { 9489 default: { 9490 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, 9491 N->getFlags())); 9492 break; 9493 } 9494 case ISD::VSELECT: 9495 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); 9496 break; 9497 case ISD::SHL: 9498 case ISD::SRA: 9499 case ISD::SRL: 9500 case ISD::ROTL: 9501 case ISD::ROTR: 9502 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 9503 getShiftAmountOperand(Operands[0].getValueType(), 9504 Operands[1]))); 9505 break; 9506 case ISD::SIGN_EXTEND_INREG: { 9507 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 9508 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 9509 Operands[0], 9510 getValueType(ExtVT))); 9511 } 9512 } 9513 } 9514 9515 for (; i < ResNE; ++i) 9516 Scalars.push_back(getUNDEF(EltVT)); 9517 9518 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); 9519 return getBuildVector(VecVT, dl, Scalars); 9520 } 9521 9522 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp( 9523 SDNode *N, unsigned ResNE) { 9524 unsigned Opcode = N->getOpcode(); 9525 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || 9526 Opcode == ISD::USUBO || Opcode == ISD::SSUBO || 9527 Opcode == ISD::UMULO || Opcode == ISD::SMULO) && 9528 "Expected an overflow opcode"); 9529 9530 EVT ResVT = N->getValueType(0); 9531 EVT OvVT = N->getValueType(1); 9532 EVT ResEltVT = ResVT.getVectorElementType(); 9533 EVT OvEltVT = OvVT.getVectorElementType(); 9534 SDLoc dl(N); 9535 9536 // If ResNE is 0, fully unroll the vector op. 9537 unsigned NE = ResVT.getVectorNumElements(); 9538 if (ResNE == 0) 9539 ResNE = NE; 9540 else if (NE > ResNE) 9541 NE = ResNE; 9542 9543 SmallVector<SDValue, 8> LHSScalars; 9544 SmallVector<SDValue, 8> RHSScalars; 9545 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE); 9546 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE); 9547 9548 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT); 9549 SDVTList VTs = getVTList(ResEltVT, SVT); 9550 SmallVector<SDValue, 8> ResScalars; 9551 SmallVector<SDValue, 8> OvScalars; 9552 for (unsigned i = 0; i < NE; ++i) { 9553 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]); 9554 SDValue Ov = 9555 getSelect(dl, OvEltVT, Res.getValue(1), 9556 getBoolConstant(true, dl, OvEltVT, ResVT), 9557 getConstant(0, dl, OvEltVT)); 9558 9559 ResScalars.push_back(Res); 9560 OvScalars.push_back(Ov); 9561 } 9562 9563 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT)); 9564 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT)); 9565 9566 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE); 9567 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE); 9568 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars), 9569 getBuildVector(NewOvVT, dl, OvScalars)); 9570 } 9571 9572 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, 9573 LoadSDNode *Base, 9574 unsigned Bytes, 9575 int Dist) const { 9576 if (LD->isVolatile() || Base->isVolatile()) 9577 return false; 9578 // TODO: probably too restrictive for atomics, revisit 9579 if (!LD->isSimple()) 9580 return false; 9581 if (LD->isIndexed() || Base->isIndexed()) 9582 return false; 9583 if (LD->getChain() != Base->getChain()) 9584 return false; 9585 EVT VT = LD->getValueType(0); 9586 if (VT.getSizeInBits() / 8 != Bytes) 9587 return false; 9588 9589 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this); 9590 auto LocDecomp = BaseIndexOffset::match(LD, *this); 9591 9592 int64_t Offset = 0; 9593 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) 9594 return (Dist * Bytes == Offset); 9595 return false; 9596 } 9597 9598 /// InferPtrAlignment - Infer alignment of a load / store address. Return None 9599 /// if it cannot be inferred. 9600 MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { 9601 // If this is a GlobalAddress + cst, return the alignment. 9602 const GlobalValue *GV = nullptr; 9603 int64_t GVOffset = 0; 9604 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 9605 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); 9606 KnownBits Known(PtrWidth); 9607 llvm::computeKnownBits(GV, Known, getDataLayout()); 9608 unsigned AlignBits = Known.countMinTrailingZeros(); 9609 if (AlignBits) 9610 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset); 9611 } 9612 9613 // If this is a direct reference to a stack slot, use information about the 9614 // stack slot's alignment. 9615 int FrameIdx = INT_MIN; 9616 int64_t FrameOffset = 0; 9617 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 9618 FrameIdx = FI->getIndex(); 9619 } else if (isBaseWithConstantOffset(Ptr) && 9620 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 9621 // Handle FI+Cst 9622 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 9623 FrameOffset = Ptr.getConstantOperandVal(1); 9624 } 9625 9626 if (FrameIdx != INT_MIN) { 9627 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); 9628 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset); 9629 } 9630 9631 return None; 9632 } 9633 9634 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 9635 /// which is split (or expanded) into two not necessarily identical pieces. 9636 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 9637 // Currently all types are split in half. 9638 EVT LoVT, HiVT; 9639 if (!VT.isVector()) 9640 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 9641 else 9642 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); 9643 9644 return std::make_pair(LoVT, HiVT); 9645 } 9646 9647 /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a 9648 /// type, dependent on an enveloping VT that has been split into two identical 9649 /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. 9650 std::pair<EVT, EVT> 9651 SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, 9652 bool *HiIsEmpty) const { 9653 EVT EltTp = VT.getVectorElementType(); 9654 // Examples: 9655 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) 9656 // custom VL=9 with enveloping VL=8/8 yields 8/1 9657 // custom VL=10 with enveloping VL=8/8 yields 8/2 9658 // etc. 9659 ElementCount VTNumElts = VT.getVectorElementCount(); 9660 ElementCount EnvNumElts = EnvVT.getVectorElementCount(); 9661 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && 9662 "Mixing fixed width and scalable vectors when enveloping a type"); 9663 EVT LoVT, HiVT; 9664 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { 9665 LoVT = EnvVT; 9666 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); 9667 *HiIsEmpty = false; 9668 } else { 9669 // Flag that hi type has zero storage size, but return split envelop type 9670 // (this would be easier if vector types with zero elements were allowed). 9671 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); 9672 HiVT = EnvVT; 9673 *HiIsEmpty = true; 9674 } 9675 return std::make_pair(LoVT, HiVT); 9676 } 9677 9678 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 9679 /// low/high part. 9680 std::pair<SDValue, SDValue> 9681 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 9682 const EVT &HiVT) { 9683 assert(LoVT.isScalableVector() == HiVT.isScalableVector() && 9684 LoVT.isScalableVector() == N.getValueType().isScalableVector() && 9685 "Splitting vector with an invalid mixture of fixed and scalable " 9686 "vector types"); 9687 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= 9688 N.getValueType().getVectorMinNumElements() && 9689 "More vector elements requested than available!"); 9690 SDValue Lo, Hi; 9691 Lo = 9692 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); 9693 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements() 9694 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales 9695 // IDX with the runtime scaling factor of the result vector type. For 9696 // fixed-width result vectors, that runtime scaling factor is 1. 9697 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 9698 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); 9699 return std::make_pair(Lo, Hi); 9700 } 9701 9702 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 9703 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { 9704 EVT VT = N.getValueType(); 9705 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 9706 NextPowerOf2(VT.getVectorNumElements())); 9707 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, 9708 getVectorIdxConstant(0, DL)); 9709 } 9710 9711 void SelectionDAG::ExtractVectorElements(SDValue Op, 9712 SmallVectorImpl<SDValue> &Args, 9713 unsigned Start, unsigned Count, 9714 EVT EltVT) { 9715 EVT VT = Op.getValueType(); 9716 if (Count == 0) 9717 Count = VT.getVectorNumElements(); 9718 if (EltVT == EVT()) 9719 EltVT = VT.getVectorElementType(); 9720 SDLoc SL(Op); 9721 for (unsigned i = Start, e = Start + Count; i != e; ++i) { 9722 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, 9723 getVectorIdxConstant(i, SL))); 9724 } 9725 } 9726 9727 // getAddressSpace - Return the address space this GlobalAddress belongs to. 9728 unsigned GlobalAddressSDNode::getAddressSpace() const { 9729 return getGlobal()->getType()->getAddressSpace(); 9730 } 9731 9732 Type *ConstantPoolSDNode::getType() const { 9733 if (isMachineConstantPoolEntry()) 9734 return Val.MachineCPVal->getType(); 9735 return Val.ConstVal->getType(); 9736 } 9737 9738 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, 9739 unsigned &SplatBitSize, 9740 bool &HasAnyUndefs, 9741 unsigned MinSplatBits, 9742 bool IsBigEndian) const { 9743 EVT VT = getValueType(0); 9744 assert(VT.isVector() && "Expected a vector type"); 9745 unsigned VecWidth = VT.getSizeInBits(); 9746 if (MinSplatBits > VecWidth) 9747 return false; 9748 9749 // FIXME: The widths are based on this node's type, but build vectors can 9750 // truncate their operands. 9751 SplatValue = APInt(VecWidth, 0); 9752 SplatUndef = APInt(VecWidth, 0); 9753 9754 // Get the bits. Bits with undefined values (when the corresponding element 9755 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 9756 // in SplatValue. If any of the values are not constant, give up and return 9757 // false. 9758 unsigned int NumOps = getNumOperands(); 9759 assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); 9760 unsigned EltWidth = VT.getScalarSizeInBits(); 9761 9762 for (unsigned j = 0; j < NumOps; ++j) { 9763 unsigned i = IsBigEndian ? NumOps - 1 - j : j; 9764 SDValue OpVal = getOperand(i); 9765 unsigned BitPos = j * EltWidth; 9766 9767 if (OpVal.isUndef()) 9768 SplatUndef.setBits(BitPos, BitPos + EltWidth); 9769 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal)) 9770 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); 9771 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 9772 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); 9773 else 9774 return false; 9775 } 9776 9777 // The build_vector is all constants or undefs. Find the smallest element 9778 // size that splats the vector. 9779 HasAnyUndefs = (SplatUndef != 0); 9780 9781 // FIXME: This does not work for vectors with elements less than 8 bits. 9782 while (VecWidth > 8) { 9783 unsigned HalfSize = VecWidth / 2; 9784 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 9785 APInt LowValue = SplatValue.trunc(HalfSize); 9786 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 9787 APInt LowUndef = SplatUndef.trunc(HalfSize); 9788 9789 // If the two halves do not match (ignoring undef bits), stop here. 9790 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 9791 MinSplatBits > HalfSize) 9792 break; 9793 9794 SplatValue = HighValue | LowValue; 9795 SplatUndef = HighUndef & LowUndef; 9796 9797 VecWidth = HalfSize; 9798 } 9799 9800 SplatBitSize = VecWidth; 9801 return true; 9802 } 9803 9804 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, 9805 BitVector *UndefElements) const { 9806 unsigned NumOps = getNumOperands(); 9807 if (UndefElements) { 9808 UndefElements->clear(); 9809 UndefElements->resize(NumOps); 9810 } 9811 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); 9812 if (!DemandedElts) 9813 return SDValue(); 9814 SDValue Splatted; 9815 for (unsigned i = 0; i != NumOps; ++i) { 9816 if (!DemandedElts[i]) 9817 continue; 9818 SDValue Op = getOperand(i); 9819 if (Op.isUndef()) { 9820 if (UndefElements) 9821 (*UndefElements)[i] = true; 9822 } else if (!Splatted) { 9823 Splatted = Op; 9824 } else if (Splatted != Op) { 9825 return SDValue(); 9826 } 9827 } 9828 9829 if (!Splatted) { 9830 unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros(); 9831 assert(getOperand(FirstDemandedIdx).isUndef() && 9832 "Can only have a splat without a constant for all undefs."); 9833 return getOperand(FirstDemandedIdx); 9834 } 9835 9836 return Splatted; 9837 } 9838 9839 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { 9840 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); 9841 return getSplatValue(DemandedElts, UndefElements); 9842 } 9843 9844 bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, 9845 SmallVectorImpl<SDValue> &Sequence, 9846 BitVector *UndefElements) const { 9847 unsigned NumOps = getNumOperands(); 9848 Sequence.clear(); 9849 if (UndefElements) { 9850 UndefElements->clear(); 9851 UndefElements->resize(NumOps); 9852 } 9853 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); 9854 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) 9855 return false; 9856 9857 // Set the undefs even if we don't find a sequence (like getSplatValue). 9858 if (UndefElements) 9859 for (unsigned I = 0; I != NumOps; ++I) 9860 if (DemandedElts[I] && getOperand(I).isUndef()) 9861 (*UndefElements)[I] = true; 9862 9863 // Iteratively widen the sequence length looking for repetitions. 9864 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { 9865 Sequence.append(SeqLen, SDValue()); 9866 for (unsigned I = 0; I != NumOps; ++I) { 9867 if (!DemandedElts[I]) 9868 continue; 9869 SDValue &SeqOp = Sequence[I % SeqLen]; 9870 SDValue Op = getOperand(I); 9871 if (Op.isUndef()) { 9872 if (!SeqOp) 9873 SeqOp = Op; 9874 continue; 9875 } 9876 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { 9877 Sequence.clear(); 9878 break; 9879 } 9880 SeqOp = Op; 9881 } 9882 if (!Sequence.empty()) 9883 return true; 9884 } 9885 9886 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); 9887 return false; 9888 } 9889 9890 bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, 9891 BitVector *UndefElements) const { 9892 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); 9893 return getRepeatedSequence(DemandedElts, Sequence, UndefElements); 9894 } 9895 9896 ConstantSDNode * 9897 BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, 9898 BitVector *UndefElements) const { 9899 return dyn_cast_or_null<ConstantSDNode>( 9900 getSplatValue(DemandedElts, UndefElements)); 9901 } 9902 9903 ConstantSDNode * 9904 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { 9905 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements)); 9906 } 9907 9908 ConstantFPSDNode * 9909 BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, 9910 BitVector *UndefElements) const { 9911 return dyn_cast_or_null<ConstantFPSDNode>( 9912 getSplatValue(DemandedElts, UndefElements)); 9913 } 9914 9915 ConstantFPSDNode * 9916 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { 9917 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements)); 9918 } 9919 9920 int32_t 9921 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, 9922 uint32_t BitWidth) const { 9923 if (ConstantFPSDNode *CN = 9924 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) { 9925 bool IsExact; 9926 APSInt IntVal(BitWidth); 9927 const APFloat &APF = CN->getValueAPF(); 9928 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != 9929 APFloat::opOK || 9930 !IsExact) 9931 return -1; 9932 9933 return IntVal.exactLogBase2(); 9934 } 9935 return -1; 9936 } 9937 9938 bool BuildVectorSDNode::isConstant() const { 9939 for (const SDValue &Op : op_values()) { 9940 unsigned Opc = Op.getOpcode(); 9941 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) 9942 return false; 9943 } 9944 return true; 9945 } 9946 9947 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 9948 // Find the first non-undef value in the shuffle mask. 9949 unsigned i, e; 9950 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 9951 /* search */; 9952 9953 // If all elements are undefined, this shuffle can be considered a splat 9954 // (although it should eventually get simplified away completely). 9955 if (i == e) 9956 return true; 9957 9958 // Make sure all remaining elements are either undef or the same as the first 9959 // non-undef value. 9960 for (int Idx = Mask[i]; i != e; ++i) 9961 if (Mask[i] >= 0 && Mask[i] != Idx) 9962 return false; 9963 return true; 9964 } 9965 9966 // Returns the SDNode if it is a constant integer BuildVector 9967 // or constant integer. 9968 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) { 9969 if (isa<ConstantSDNode>(N)) 9970 return N.getNode(); 9971 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) 9972 return N.getNode(); 9973 // Treat a GlobalAddress supporting constant offset folding as a 9974 // constant integer. 9975 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N)) 9976 if (GA->getOpcode() == ISD::GlobalAddress && 9977 TLI->isOffsetFoldingLegal(GA)) 9978 return GA; 9979 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && 9980 isa<ConstantSDNode>(N.getOperand(0))) 9981 return N.getNode(); 9982 return nullptr; 9983 } 9984 9985 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) { 9986 if (isa<ConstantFPSDNode>(N)) 9987 return N.getNode(); 9988 9989 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) 9990 return N.getNode(); 9991 9992 return nullptr; 9993 } 9994 9995 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { 9996 assert(!Node->OperandList && "Node already has operands"); 9997 assert(SDNode::getMaxNumOperands() >= Vals.size() && 9998 "too many operands to fit into SDNode"); 9999 SDUse *Ops = OperandRecycler.allocate( 10000 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator); 10001 10002 bool IsDivergent = false; 10003 for (unsigned I = 0; I != Vals.size(); ++I) { 10004 Ops[I].setUser(Node); 10005 Ops[I].setInitial(Vals[I]); 10006 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. 10007 IsDivergent |= Ops[I].getNode()->isDivergent(); 10008 } 10009 Node->NumOperands = Vals.size(); 10010 Node->OperandList = Ops; 10011 if (!TLI->isSDNodeAlwaysUniform(Node)) { 10012 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); 10013 Node->SDNodeBits.IsDivergent = IsDivergent; 10014 } 10015 checkForCycles(Node); 10016 } 10017 10018 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, 10019 SmallVectorImpl<SDValue> &Vals) { 10020 size_t Limit = SDNode::getMaxNumOperands(); 10021 while (Vals.size() > Limit) { 10022 unsigned SliceIdx = Vals.size() - Limit; 10023 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit); 10024 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); 10025 Vals.erase(Vals.begin() + SliceIdx, Vals.end()); 10026 Vals.emplace_back(NewTF); 10027 } 10028 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); 10029 } 10030 10031 SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, 10032 EVT VT, SDNodeFlags Flags) { 10033 switch (Opcode) { 10034 default: 10035 return SDValue(); 10036 case ISD::ADD: 10037 case ISD::OR: 10038 case ISD::XOR: 10039 case ISD::UMAX: 10040 return getConstant(0, DL, VT); 10041 case ISD::MUL: 10042 return getConstant(1, DL, VT); 10043 case ISD::AND: 10044 case ISD::UMIN: 10045 return getAllOnesConstant(DL, VT); 10046 case ISD::SMAX: 10047 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); 10048 case ISD::SMIN: 10049 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); 10050 case ISD::FADD: 10051 return getConstantFP(-0.0, DL, VT); 10052 case ISD::FMUL: 10053 return getConstantFP(1.0, DL, VT); 10054 case ISD::FMINNUM: 10055 case ISD::FMAXNUM: { 10056 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. 10057 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); 10058 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : 10059 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) : 10060 APFloat::getLargest(Semantics); 10061 if (Opcode == ISD::FMAXNUM) 10062 NeutralAF.changeSign(); 10063 10064 return getConstantFP(NeutralAF, DL, VT); 10065 } 10066 } 10067 } 10068 10069 #ifndef NDEBUG 10070 static void checkForCyclesHelper(const SDNode *N, 10071 SmallPtrSetImpl<const SDNode*> &Visited, 10072 SmallPtrSetImpl<const SDNode*> &Checked, 10073 const llvm::SelectionDAG *DAG) { 10074 // If this node has already been checked, don't check it again. 10075 if (Checked.count(N)) 10076 return; 10077 10078 // If a node has already been visited on this depth-first walk, reject it as 10079 // a cycle. 10080 if (!Visited.insert(N).second) { 10081 errs() << "Detected cycle in SelectionDAG\n"; 10082 dbgs() << "Offending node:\n"; 10083 N->dumprFull(DAG); dbgs() << "\n"; 10084 abort(); 10085 } 10086 10087 for (const SDValue &Op : N->op_values()) 10088 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); 10089 10090 Checked.insert(N); 10091 Visited.erase(N); 10092 } 10093 #endif 10094 10095 void llvm::checkForCycles(const llvm::SDNode *N, 10096 const llvm::SelectionDAG *DAG, 10097 bool force) { 10098 #ifndef NDEBUG 10099 bool check = force; 10100 #ifdef EXPENSIVE_CHECKS 10101 check = true; 10102 #endif // EXPENSIVE_CHECKS 10103 if (check) { 10104 assert(N && "Checking nonexistent SDNode"); 10105 SmallPtrSet<const SDNode*, 32> visited; 10106 SmallPtrSet<const SDNode*, 32> checked; 10107 checkForCyclesHelper(N, visited, checked, DAG); 10108 } 10109 #endif // !NDEBUG 10110 } 10111 10112 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { 10113 checkForCycles(DAG->getRoot().getNode(), DAG, force); 10114 } 10115