1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the RISCV target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVISelDAGToDAG.h" 14 #include "MCTargetDesc/RISCVMCTargetDesc.h" 15 #include "MCTargetDesc/RISCVMatInt.h" 16 #include "RISCVISelLowering.h" 17 #include "RISCVMachineFunctionInfo.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/IR/IntrinsicsRISCV.h" 20 #include "llvm/Support/Alignment.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/KnownBits.h" 23 #include "llvm/Support/MathExtras.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 #define DEBUG_TYPE "riscv-isel" 29 30 namespace llvm { 31 namespace RISCV { 32 #define GET_RISCVVSSEGTable_IMPL 33 #define GET_RISCVVLSEGTable_IMPL 34 #define GET_RISCVVLXSEGTable_IMPL 35 #define GET_RISCVVSXSEGTable_IMPL 36 #define GET_RISCVVLETable_IMPL 37 #define GET_RISCVVSETable_IMPL 38 #define GET_RISCVVLXTable_IMPL 39 #define GET_RISCVVSXTable_IMPL 40 #define GET_RISCVMaskedPseudosTable_IMPL 41 #include "RISCVGenSearchableTables.inc" 42 } // namespace RISCV 43 } // namespace llvm 44 45 void RISCVDAGToDAGISel::PreprocessISelDAG() { 46 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 47 E = CurDAG->allnodes_end(); 48 I != E;) { 49 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 50 51 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point 52 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden. 53 if (N->getOpcode() == ISD::SPLAT_VECTOR) { 54 MVT VT = N->getSimpleValueType(0); 55 unsigned Opc = 56 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL; 57 SDLoc DL(N); 58 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()); 59 SDValue Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), 60 N->getOperand(0), VL); 61 62 --I; 63 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 64 ++I; 65 CurDAG->DeleteNode(N); 66 continue; 67 } 68 69 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector 70 // load. Done after lowering and combining so that we have a chance to 71 // optimize this to VMV_V_X_VL when the upper bits aren't needed. 72 if (N->getOpcode() != RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) 73 continue; 74 75 assert(N->getNumOperands() == 4 && "Unexpected number of operands"); 76 MVT VT = N->getSimpleValueType(0); 77 SDValue Passthru = N->getOperand(0); 78 SDValue Lo = N->getOperand(1); 79 SDValue Hi = N->getOperand(2); 80 SDValue VL = N->getOperand(3); 81 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() && 82 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 && 83 "Unexpected VTs!"); 84 MachineFunction &MF = CurDAG->getMachineFunction(); 85 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>(); 86 SDLoc DL(N); 87 88 // We use the same frame index we use for moving two i32s into 64-bit FPR. 89 // This is an analogous operation. 90 int FI = FuncInfo->getMoveF64FrameIndex(MF); 91 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); 92 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 93 SDValue StackSlot = 94 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout())); 95 96 SDValue Chain = CurDAG->getEntryNode(); 97 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8)); 98 99 SDValue OffsetSlot = 100 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL); 101 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4), 102 Align(8)); 103 104 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 105 106 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other}); 107 SDValue IntID = 108 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64); 109 SDValue Ops[] = {Chain, 110 IntID, 111 Passthru, 112 StackSlot, 113 CurDAG->getRegister(RISCV::X0, MVT::i64), 114 VL}; 115 116 SDValue Result = CurDAG->getMemIntrinsicNode( 117 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MVT::i64, MPI, Align(8), 118 MachineMemOperand::MOLoad); 119 120 // We're about to replace all uses of the SPLAT_VECTOR_SPLIT_I64 with the 121 // vlse we created. This will cause general havok on the dag because 122 // anything below the conversion could be folded into other existing nodes. 123 // To avoid invalidating 'I', back it up to the convert node. 124 --I; 125 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 126 127 // Now that we did that, the node is dead. Increment the iterator to the 128 // next node to process, then delete N. 129 ++I; 130 CurDAG->DeleteNode(N); 131 } 132 } 133 134 void RISCVDAGToDAGISel::PostprocessISelDAG() { 135 HandleSDNode Dummy(CurDAG->getRoot()); 136 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 137 138 bool MadeChange = false; 139 while (Position != CurDAG->allnodes_begin()) { 140 SDNode *N = &*--Position; 141 // Skip dead nodes and any non-machine opcodes. 142 if (N->use_empty() || !N->isMachineOpcode()) 143 continue; 144 145 MadeChange |= doPeepholeSExtW(N); 146 MadeChange |= doPeepholeLoadStoreADDI(N); 147 MadeChange |= doPeepholeMaskedRVV(N); 148 } 149 150 CurDAG->setRoot(Dummy.getValue()); 151 152 if (MadeChange) 153 CurDAG->RemoveDeadNodes(); 154 } 155 156 // Returns true if N is a MachineSDNode that has a reg and simm12 memory 157 // operand. The indices of the base pointer and offset are returned in BaseOpIdx 158 // and OffsetOpIdx. 159 static bool hasMemOffset(SDNode *N, unsigned &BaseOpIdx, 160 unsigned &OffsetOpIdx) { 161 switch (N->getMachineOpcode()) { 162 case RISCV::LB: 163 case RISCV::LH: 164 case RISCV::LW: 165 case RISCV::LBU: 166 case RISCV::LHU: 167 case RISCV::LWU: 168 case RISCV::LD: 169 case RISCV::FLH: 170 case RISCV::FLW: 171 case RISCV::FLD: 172 BaseOpIdx = 0; 173 OffsetOpIdx = 1; 174 return true; 175 case RISCV::SB: 176 case RISCV::SH: 177 case RISCV::SW: 178 case RISCV::SD: 179 case RISCV::FSH: 180 case RISCV::FSW: 181 case RISCV::FSD: 182 BaseOpIdx = 1; 183 OffsetOpIdx = 2; 184 return true; 185 } 186 187 return false; 188 } 189 190 static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL, 191 const MVT VT, int64_t Imm, 192 const RISCVSubtarget &Subtarget) { 193 assert(VT == MVT::i64 && "Expecting MVT::i64"); 194 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering(); 195 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool( 196 ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT)); 197 SDValue Addr = TLI->getAddr(CP, *CurDAG); 198 SDValue Offset = CurDAG->getTargetConstant(0, DL, VT); 199 // Since there is no data race, the chain can be the entry node. 200 SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset, 201 CurDAG->getEntryNode()); 202 MachineFunction &MF = CurDAG->getMachineFunction(); 203 MachineMemOperand *MemOp = MF.getMachineMemOperand( 204 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 205 LLT(VT), CP->getAlign()); 206 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp}); 207 return Load; 208 } 209 210 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 211 int64_t Imm, const RISCVSubtarget &Subtarget) { 212 MVT XLenVT = Subtarget.getXLenVT(); 213 RISCVMatInt::InstSeq Seq = 214 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits()); 215 216 // If Imm is expensive to build, then we put it into constant pool. 217 if (Subtarget.useConstantPoolForLargeInts() && 218 Seq.size() > Subtarget.getMaxBuildIntsCost()) 219 return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget); 220 221 SDNode *Result = nullptr; 222 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT); 223 for (RISCVMatInt::Inst &Inst : Seq) { 224 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT); 225 switch (Inst.getOpndKind()) { 226 case RISCVMatInt::Imm: 227 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SDImm); 228 break; 229 case RISCVMatInt::RegX0: 230 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, 231 CurDAG->getRegister(RISCV::X0, XLenVT)); 232 break; 233 case RISCVMatInt::RegReg: 234 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SrcReg); 235 break; 236 case RISCVMatInt::RegImm: 237 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm); 238 break; 239 } 240 241 // Only the first instruction has X0 as its source. 242 SrcReg = SDValue(Result, 0); 243 } 244 245 return Result; 246 } 247 248 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 249 unsigned RegClassID, unsigned SubReg0) { 250 assert(Regs.size() >= 2 && Regs.size() <= 8); 251 252 SDLoc DL(Regs[0]); 253 SmallVector<SDValue, 8> Ops; 254 255 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32)); 256 257 for (unsigned I = 0; I < Regs.size(); ++I) { 258 Ops.push_back(Regs[I]); 259 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32)); 260 } 261 SDNode *N = 262 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 263 return SDValue(N, 0); 264 } 265 266 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 267 unsigned NF) { 268 static const unsigned RegClassIDs[] = { 269 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID, 270 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID, 271 RISCV::VRN8M1RegClassID}; 272 273 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0); 274 } 275 276 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 277 unsigned NF) { 278 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID, 279 RISCV::VRN3M2RegClassID, 280 RISCV::VRN4M2RegClassID}; 281 282 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0); 283 } 284 285 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 286 unsigned NF) { 287 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID, 288 RISCV::sub_vrm4_0); 289 } 290 291 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 292 unsigned NF, RISCVII::VLMUL LMUL) { 293 switch (LMUL) { 294 default: 295 llvm_unreachable("Invalid LMUL."); 296 case RISCVII::VLMUL::LMUL_F8: 297 case RISCVII::VLMUL::LMUL_F4: 298 case RISCVII::VLMUL::LMUL_F2: 299 case RISCVII::VLMUL::LMUL_1: 300 return createM1Tuple(CurDAG, Regs, NF); 301 case RISCVII::VLMUL::LMUL_2: 302 return createM2Tuple(CurDAG, Regs, NF); 303 case RISCVII::VLMUL::LMUL_4: 304 return createM4Tuple(CurDAG, Regs, NF); 305 } 306 } 307 308 void RISCVDAGToDAGISel::addVectorLoadStoreOperands( 309 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp, 310 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands, 311 bool IsLoad, MVT *IndexVT) { 312 SDValue Chain = Node->getOperand(0); 313 SDValue Glue; 314 315 SDValue Base; 316 SelectBaseAddr(Node->getOperand(CurOp++), Base); 317 Operands.push_back(Base); // Base pointer. 318 319 if (IsStridedOrIndexed) { 320 Operands.push_back(Node->getOperand(CurOp++)); // Index. 321 if (IndexVT) 322 *IndexVT = Operands.back()->getSimpleValueType(0); 323 } 324 325 if (IsMasked) { 326 // Mask needs to be copied to V0. 327 SDValue Mask = Node->getOperand(CurOp++); 328 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); 329 Glue = Chain.getValue(1); 330 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); 331 } 332 SDValue VL; 333 selectVLOp(Node->getOperand(CurOp++), VL); 334 Operands.push_back(VL); 335 336 MVT XLenVT = Subtarget->getXLenVT(); 337 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 338 Operands.push_back(SEWOp); 339 340 // Masked load has the tail policy argument. 341 if (IsMasked && IsLoad) { 342 // Policy must be a constant. 343 uint64_t Policy = Node->getConstantOperandVal(CurOp++); 344 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT); 345 Operands.push_back(PolicyOp); 346 } 347 348 Operands.push_back(Chain); // Chain. 349 if (Glue) 350 Operands.push_back(Glue); 351 } 352 353 static bool isAllUndef(ArrayRef<SDValue> Values) { 354 return llvm::all_of(Values, [](SDValue V) { return V->isUndef(); }); 355 } 356 357 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked, 358 bool IsStrided) { 359 SDLoc DL(Node); 360 unsigned NF = Node->getNumValues() - 1; 361 MVT VT = Node->getSimpleValueType(0); 362 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 363 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 364 365 unsigned CurOp = 2; 366 SmallVector<SDValue, 8> Operands; 367 368 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 369 Node->op_begin() + CurOp + NF); 370 bool IsTU = IsMasked || !isAllUndef(Regs); 371 if (IsTU) { 372 SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL); 373 Operands.push_back(Merge); 374 } 375 CurOp += NF; 376 377 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 378 Operands, /*IsLoad=*/true); 379 380 const RISCV::VLSEGPseudo *P = 381 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 382 static_cast<unsigned>(LMUL)); 383 MachineSDNode *Load = 384 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 385 386 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 387 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 388 389 SDValue SuperReg = SDValue(Load, 0); 390 for (unsigned I = 0; I < NF; ++I) { 391 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 392 ReplaceUses(SDValue(Node, I), 393 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 394 } 395 396 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 397 CurDAG->RemoveDeadNode(Node); 398 } 399 400 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) { 401 SDLoc DL(Node); 402 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. 403 MVT VT = Node->getSimpleValueType(0); 404 MVT XLenVT = Subtarget->getXLenVT(); 405 unsigned SEW = VT.getScalarSizeInBits(); 406 unsigned Log2SEW = Log2_32(SEW); 407 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 408 409 unsigned CurOp = 2; 410 SmallVector<SDValue, 7> Operands; 411 412 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 413 Node->op_begin() + CurOp + NF); 414 bool IsTU = IsMasked || !isAllUndef(Regs); 415 if (IsTU) { 416 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 417 Operands.push_back(MaskedOff); 418 } 419 CurOp += NF; 420 421 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 422 /*IsStridedOrIndexed*/ false, Operands, 423 /*IsLoad=*/true); 424 425 const RISCV::VLSEGPseudo *P = 426 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 427 Log2SEW, static_cast<unsigned>(LMUL)); 428 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, 429 MVT::Other, MVT::Glue, Operands); 430 bool TailAgnostic = true; 431 bool MaskAgnostic = false; 432 if (IsMasked) { 433 uint64_t Policy = Node->getConstantOperandVal(Node->getNumOperands() - 1); 434 TailAgnostic = Policy & RISCVII::TAIL_AGNOSTIC; 435 MaskAgnostic = Policy & RISCVII::MASK_AGNOSTIC; 436 } 437 unsigned VType = 438 RISCVVType::encodeVTYPE(LMUL, SEW, TailAgnostic, MaskAgnostic); 439 SDValue VTypeOp = CurDAG->getTargetConstant(VType, DL, XLenVT); 440 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, 441 VTypeOp, /*Glue*/ SDValue(Load, 2)); 442 443 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 444 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 445 446 SDValue SuperReg = SDValue(Load, 0); 447 for (unsigned I = 0; I < NF; ++I) { 448 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 449 ReplaceUses(SDValue(Node, I), 450 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 451 } 452 453 ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL 454 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain 455 CurDAG->RemoveDeadNode(Node); 456 } 457 458 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked, 459 bool IsOrdered) { 460 SDLoc DL(Node); 461 unsigned NF = Node->getNumValues() - 1; 462 MVT VT = Node->getSimpleValueType(0); 463 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 464 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 465 466 unsigned CurOp = 2; 467 SmallVector<SDValue, 8> Operands; 468 469 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 470 Node->op_begin() + CurOp + NF); 471 bool IsTU = IsMasked || !isAllUndef(Regs); 472 if (IsTU) { 473 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 474 Operands.push_back(MaskedOff); 475 } 476 CurOp += NF; 477 478 MVT IndexVT; 479 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 480 /*IsStridedOrIndexed*/ true, Operands, 481 /*IsLoad=*/true, &IndexVT); 482 483 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 484 "Element count mismatch"); 485 486 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 487 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 488 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 489 report_fatal_error("The V extension does not support EEW=64 for index " 490 "values when XLEN=32"); 491 } 492 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo( 493 NF, IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 494 static_cast<unsigned>(IndexLMUL)); 495 MachineSDNode *Load = 496 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 497 498 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 499 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 500 501 SDValue SuperReg = SDValue(Load, 0); 502 for (unsigned I = 0; I < NF; ++I) { 503 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 504 ReplaceUses(SDValue(Node, I), 505 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 506 } 507 508 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 509 CurDAG->RemoveDeadNode(Node); 510 } 511 512 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked, 513 bool IsStrided) { 514 SDLoc DL(Node); 515 unsigned NF = Node->getNumOperands() - 4; 516 if (IsStrided) 517 NF--; 518 if (IsMasked) 519 NF--; 520 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 521 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 522 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 523 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 524 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 525 526 SmallVector<SDValue, 8> Operands; 527 Operands.push_back(StoreVal); 528 unsigned CurOp = 2 + NF; 529 530 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 531 Operands); 532 533 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo( 534 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 535 MachineSDNode *Store = 536 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 537 538 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 539 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 540 541 ReplaceNode(Node, Store); 542 } 543 544 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked, 545 bool IsOrdered) { 546 SDLoc DL(Node); 547 unsigned NF = Node->getNumOperands() - 5; 548 if (IsMasked) 549 --NF; 550 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 551 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 552 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 553 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 554 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 555 556 SmallVector<SDValue, 8> Operands; 557 Operands.push_back(StoreVal); 558 unsigned CurOp = 2 + NF; 559 560 MVT IndexVT; 561 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 562 /*IsStridedOrIndexed*/ true, Operands, 563 /*IsLoad=*/false, &IndexVT); 564 565 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 566 "Element count mismatch"); 567 568 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 569 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 570 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 571 report_fatal_error("The V extension does not support EEW=64 for index " 572 "values when XLEN=32"); 573 } 574 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo( 575 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 576 static_cast<unsigned>(IndexLMUL)); 577 MachineSDNode *Store = 578 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 579 580 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 581 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 582 583 ReplaceNode(Node, Store); 584 } 585 586 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) { 587 if (!Subtarget->hasVInstructions()) 588 return; 589 590 assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN || 591 Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) && 592 "Unexpected opcode"); 593 594 SDLoc DL(Node); 595 MVT XLenVT = Subtarget->getXLenVT(); 596 597 bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN; 598 unsigned IntNoOffset = HasChain ? 1 : 0; 599 unsigned IntNo = Node->getConstantOperandVal(IntNoOffset); 600 601 assert((IntNo == Intrinsic::riscv_vsetvli || 602 IntNo == Intrinsic::riscv_vsetvlimax || 603 IntNo == Intrinsic::riscv_vsetvli_opt || 604 IntNo == Intrinsic::riscv_vsetvlimax_opt) && 605 "Unexpected vsetvli intrinsic"); 606 607 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax || 608 IntNo == Intrinsic::riscv_vsetvlimax_opt; 609 unsigned Offset = IntNoOffset + (VLMax ? 1 : 2); 610 611 assert(Node->getNumOperands() == Offset + 2 && 612 "Unexpected number of operands"); 613 614 unsigned SEW = 615 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7); 616 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>( 617 Node->getConstantOperandVal(Offset + 1) & 0x7); 618 619 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true, 620 /*MaskAgnostic*/ false); 621 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT); 622 623 SmallVector<EVT, 2> VTs = {XLenVT}; 624 if (HasChain) 625 VTs.push_back(MVT::Other); 626 627 SDValue VLOperand; 628 unsigned Opcode = RISCV::PseudoVSETVLI; 629 if (VLMax) { 630 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT); 631 Opcode = RISCV::PseudoVSETVLIX0; 632 } else { 633 VLOperand = Node->getOperand(IntNoOffset + 1); 634 635 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { 636 uint64_t AVL = C->getZExtValue(); 637 if (isUInt<5>(AVL)) { 638 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); 639 SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp}; 640 if (HasChain) 641 Ops.push_back(Node->getOperand(0)); 642 ReplaceNode( 643 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops)); 644 return; 645 } 646 } 647 } 648 649 SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp}; 650 if (HasChain) 651 Ops.push_back(Node->getOperand(0)); 652 653 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops)); 654 } 655 656 void RISCVDAGToDAGISel::Select(SDNode *Node) { 657 // If we have a custom node, we have already selected. 658 if (Node->isMachineOpcode()) { 659 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 660 Node->setNodeId(-1); 661 return; 662 } 663 664 // Instruction Selection not handled by the auto-generated tablegen selection 665 // should be handled here. 666 unsigned Opcode = Node->getOpcode(); 667 MVT XLenVT = Subtarget->getXLenVT(); 668 SDLoc DL(Node); 669 MVT VT = Node->getSimpleValueType(0); 670 671 switch (Opcode) { 672 case ISD::Constant: { 673 auto *ConstNode = cast<ConstantSDNode>(Node); 674 if (VT == XLenVT && ConstNode->isZero()) { 675 SDValue New = 676 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT); 677 ReplaceNode(Node, New.getNode()); 678 return; 679 } 680 int64_t Imm = ConstNode->getSExtValue(); 681 // If the upper XLen-16 bits are not used, try to convert this to a simm12 682 // by sign extending bit 15. 683 if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) && 684 hasAllHUsers(Node)) 685 Imm = SignExtend64<16>(Imm); 686 // If the upper 32-bits are not used try to convert this into a simm32 by 687 // sign extending bit 32. 688 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node)) 689 Imm = SignExtend64<32>(Imm); 690 691 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget)); 692 return; 693 } 694 case ISD::FrameIndex: { 695 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT); 696 int FI = cast<FrameIndexSDNode>(Node)->getIndex(); 697 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT); 698 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm)); 699 return; 700 } 701 case ISD::ADD: { 702 // Try to select ADD + immediate used as memory addresses to 703 // (ADDI (ADD X, Imm-Lo12), Lo12) if it will allow the ADDI to be removed by 704 // doPeepholeLoadStoreADDI. 705 706 // LHS should be an immediate. 707 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 708 if (!N1C) 709 break; 710 711 int64_t Offset = N1C->getSExtValue(); 712 int64_t Lo12 = SignExtend64<12>(Offset); 713 714 // Don't do this if the lower 12 bits are 0 or we could use ADDI directly. 715 if (Lo12 == 0 || isInt<12>(Offset)) 716 break; 717 718 // Don't do this if we can use a pair of ADDIs. 719 if (isInt<12>(Offset / 2) && isInt<12>(Offset - Offset / 2)) 720 break; 721 722 bool AllPointerUses = true; 723 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 724 SDNode *User = *UI; 725 726 // Is this user a memory instruction that uses a register and immediate 727 // that has this ADD as its pointer. 728 unsigned BaseOpIdx, OffsetOpIdx; 729 if (!User->isMachineOpcode() || 730 !hasMemOffset(User, BaseOpIdx, OffsetOpIdx) || 731 UI.getOperandNo() != BaseOpIdx) { 732 AllPointerUses = false; 733 break; 734 } 735 736 // If the memory instruction already has an offset, make sure the combined 737 // offset is foldable. 738 int64_t MemOffs = 739 cast<ConstantSDNode>(User->getOperand(OffsetOpIdx))->getSExtValue(); 740 MemOffs += Lo12; 741 if (!isInt<12>(MemOffs)) { 742 AllPointerUses = false; 743 break; 744 } 745 } 746 747 if (!AllPointerUses) 748 break; 749 750 Offset -= Lo12; 751 // Restore sign bits for RV32. 752 if (!Subtarget->is64Bit()) 753 Offset = SignExtend64<32>(Offset); 754 755 // Emit (ADDI (ADD X, Hi), Lo) 756 SDNode *Imm = selectImm(CurDAG, DL, VT, Offset, *Subtarget); 757 SDNode *ADD = CurDAG->getMachineNode(RISCV::ADD, DL, VT, 758 Node->getOperand(0), SDValue(Imm, 0)); 759 SDNode *ADDI = 760 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, SDValue(ADD, 0), 761 CurDAG->getTargetConstant(Lo12, DL, VT)); 762 ReplaceNode(Node, ADDI); 763 return; 764 } 765 case ISD::SRL: { 766 // Optimize (srl (and X, C2), C) -> 767 // (srli (slli X, (XLen-C3), (XLen-C3) + C) 768 // Where C2 is a mask with C3 trailing ones. 769 // Taking into account that the C2 may have had lower bits unset by 770 // SimplifyDemandedBits. This avoids materializing the C2 immediate. 771 // This pattern occurs when type legalizing right shifts for types with 772 // less than XLen bits. 773 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 774 if (!N1C) 775 break; 776 SDValue N0 = Node->getOperand(0); 777 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 778 !isa<ConstantSDNode>(N0.getOperand(1))) 779 break; 780 unsigned ShAmt = N1C->getZExtValue(); 781 uint64_t Mask = N0.getConstantOperandVal(1); 782 Mask |= maskTrailingOnes<uint64_t>(ShAmt); 783 if (!isMask_64(Mask)) 784 break; 785 unsigned TrailingOnes = countTrailingOnes(Mask); 786 // 32 trailing ones should use srliw via tablegen pattern. 787 if (TrailingOnes == 32 || ShAmt >= TrailingOnes) 788 break; 789 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes; 790 SDNode *SLLI = 791 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 792 CurDAG->getTargetConstant(LShAmt, DL, VT)); 793 SDNode *SRLI = CurDAG->getMachineNode( 794 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 795 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 796 ReplaceNode(Node, SRLI); 797 return; 798 } 799 case ISD::SRA: { 800 // Optimize (sra (sext_inreg X, i16), C) -> 801 // (srai (slli X, (XLen-16), (XLen-16) + C) 802 // And (sra (sext_inreg X, i8), C) -> 803 // (srai (slli X, (XLen-8), (XLen-8) + C) 804 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal. 805 // This transform matches the code we get without Zbb. The shifts are more 806 // compressible, and this can help expose CSE opportunities in the sdiv by 807 // constant optimization. 808 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 809 if (!N1C) 810 break; 811 SDValue N0 = Node->getOperand(0); 812 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse()) 813 break; 814 unsigned ShAmt = N1C->getZExtValue(); 815 unsigned ExtSize = 816 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits(); 817 // ExtSize of 32 should use sraiw via tablegen pattern. 818 if (ExtSize >= 32 || ShAmt >= ExtSize) 819 break; 820 unsigned LShAmt = Subtarget->getXLen() - ExtSize; 821 SDNode *SLLI = 822 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 823 CurDAG->getTargetConstant(LShAmt, DL, VT)); 824 SDNode *SRAI = CurDAG->getMachineNode( 825 RISCV::SRAI, DL, VT, SDValue(SLLI, 0), 826 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 827 ReplaceNode(Node, SRAI); 828 return; 829 } 830 case ISD::AND: { 831 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 832 if (!N1C) 833 break; 834 835 SDValue N0 = Node->getOperand(0); 836 837 bool LeftShift = N0.getOpcode() == ISD::SHL; 838 if (!LeftShift && N0.getOpcode() != ISD::SRL) 839 break; 840 841 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 842 if (!C) 843 break; 844 uint64_t C2 = C->getZExtValue(); 845 unsigned XLen = Subtarget->getXLen(); 846 if (!C2 || C2 >= XLen) 847 break; 848 849 uint64_t C1 = N1C->getZExtValue(); 850 851 // Keep track of whether this is a c.andi. If we can't use c.andi, the 852 // shift pair might offer more compression opportunities. 853 // TODO: We could check for C extension here, but we don't have many lit 854 // tests with the C extension enabled so not checking gets better coverage. 855 // TODO: What if ANDI faster than shift? 856 bool IsCANDI = isInt<6>(N1C->getSExtValue()); 857 858 // Clear irrelevant bits in the mask. 859 if (LeftShift) 860 C1 &= maskTrailingZeros<uint64_t>(C2); 861 else 862 C1 &= maskTrailingOnes<uint64_t>(XLen - C2); 863 864 // Some transforms should only be done if the shift has a single use or 865 // the AND would become (srli (slli X, 32), 32) 866 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF); 867 868 SDValue X = N0.getOperand(0); 869 870 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask 871 // with c3 leading zeros. 872 if (!LeftShift && isMask_64(C1)) { 873 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 874 if (C2 < C3) { 875 // If the number of leading zeros is C2+32 this can be SRLIW. 876 if (C2 + 32 == C3) { 877 SDNode *SRLIW = 878 CurDAG->getMachineNode(RISCV::SRLIW, DL, XLenVT, X, 879 CurDAG->getTargetConstant(C2, DL, XLenVT)); 880 ReplaceNode(Node, SRLIW); 881 return; 882 } 883 884 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if 885 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1. 886 // 887 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type 888 // legalized and goes through DAG combine. 889 if (C2 >= 32 && (C3 - C2) == 1 && N0.hasOneUse() && 890 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 891 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) { 892 SDNode *SRAIW = 893 CurDAG->getMachineNode(RISCV::SRAIW, DL, XLenVT, X.getOperand(0), 894 CurDAG->getTargetConstant(31, DL, XLenVT)); 895 SDNode *SRLIW = CurDAG->getMachineNode( 896 RISCV::SRLIW, DL, XLenVT, SDValue(SRAIW, 0), 897 CurDAG->getTargetConstant(C3 - 32, DL, XLenVT)); 898 ReplaceNode(Node, SRLIW); 899 return; 900 } 901 902 // (srli (slli x, c3-c2), c3). 903 // Skip if we could use (zext.w (sraiw X, C2)). 904 bool Skip = Subtarget->hasStdExtZba() && C3 == 32 && 905 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 906 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32; 907 // Also Skip if we can use bexti. 908 Skip |= Subtarget->hasStdExtZbs() && C3 == XLen - 1; 909 if (OneUseOrZExtW && !Skip) { 910 SDNode *SLLI = CurDAG->getMachineNode( 911 RISCV::SLLI, DL, XLenVT, X, 912 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 913 SDNode *SRLI = 914 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 915 CurDAG->getTargetConstant(C3, DL, XLenVT)); 916 ReplaceNode(Node, SRLI); 917 return; 918 } 919 } 920 } 921 922 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask 923 // shifted by c2 bits with c3 leading zeros. 924 if (LeftShift && isShiftedMask_64(C1)) { 925 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 926 927 if (C2 + C3 < XLen && 928 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + C3)) << C2)) { 929 // Use slli.uw when possible. 930 if ((XLen - (C2 + C3)) == 32 && Subtarget->hasStdExtZba()) { 931 SDNode *SLLI_UW = 932 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, XLenVT, X, 933 CurDAG->getTargetConstant(C2, DL, XLenVT)); 934 ReplaceNode(Node, SLLI_UW); 935 return; 936 } 937 938 // (srli (slli c2+c3), c3) 939 if (OneUseOrZExtW && !IsCANDI) { 940 SDNode *SLLI = CurDAG->getMachineNode( 941 RISCV::SLLI, DL, XLenVT, X, 942 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 943 SDNode *SRLI = 944 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 945 CurDAG->getTargetConstant(C3, DL, XLenVT)); 946 ReplaceNode(Node, SRLI); 947 return; 948 } 949 } 950 } 951 952 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a 953 // shifted mask with c2 leading zeros and c3 trailing zeros. 954 if (!LeftShift && isShiftedMask_64(C1)) { 955 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 956 uint64_t C3 = countTrailingZeros(C1); 957 if (Leading == C2 && C2 + C3 < XLen && OneUseOrZExtW && !IsCANDI) { 958 unsigned SrliOpc = RISCV::SRLI; 959 // If the input is zexti32 we should use SRLIW. 960 if (X.getOpcode() == ISD::AND && isa<ConstantSDNode>(X.getOperand(1)) && 961 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) { 962 SrliOpc = RISCV::SRLIW; 963 X = X.getOperand(0); 964 } 965 SDNode *SRLI = CurDAG->getMachineNode( 966 SrliOpc, DL, XLenVT, X, 967 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 968 SDNode *SLLI = 969 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 970 CurDAG->getTargetConstant(C3, DL, XLenVT)); 971 ReplaceNode(Node, SLLI); 972 return; 973 } 974 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI. 975 if (Leading > 32 && (Leading - 32) == C2 && C2 + C3 < 32 && 976 OneUseOrZExtW && !IsCANDI) { 977 SDNode *SRLIW = CurDAG->getMachineNode( 978 RISCV::SRLIW, DL, XLenVT, X, 979 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 980 SDNode *SLLI = 981 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 982 CurDAG->getTargetConstant(C3, DL, XLenVT)); 983 ReplaceNode(Node, SLLI); 984 return; 985 } 986 } 987 988 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a 989 // shifted mask with no leading zeros and c3 trailing zeros. 990 if (LeftShift && isShiftedMask_64(C1)) { 991 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 992 uint64_t C3 = countTrailingZeros(C1); 993 if (Leading == 0 && C2 < C3 && OneUseOrZExtW && !IsCANDI) { 994 SDNode *SRLI = CurDAG->getMachineNode( 995 RISCV::SRLI, DL, XLenVT, X, 996 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 997 SDNode *SLLI = 998 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 999 CurDAG->getTargetConstant(C3, DL, XLenVT)); 1000 ReplaceNode(Node, SLLI); 1001 return; 1002 } 1003 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI. 1004 if (C2 < C3 && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) { 1005 SDNode *SRLIW = CurDAG->getMachineNode( 1006 RISCV::SRLIW, DL, XLenVT, X, 1007 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 1008 SDNode *SLLI = 1009 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 1010 CurDAG->getTargetConstant(C3, DL, XLenVT)); 1011 ReplaceNode(Node, SLLI); 1012 return; 1013 } 1014 } 1015 1016 break; 1017 } 1018 case ISD::MUL: { 1019 // Special case for calculating (mul (and X, C2), C1) where the full product 1020 // fits in XLen bits. We can shift X left by the number of leading zeros in 1021 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final 1022 // product has XLen trailing zeros, putting it in the output of MULHU. This 1023 // can avoid materializing a constant in a register for C2. 1024 1025 // RHS should be a constant. 1026 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1027 if (!N1C || !N1C->hasOneUse()) 1028 break; 1029 1030 // LHS should be an AND with constant. 1031 SDValue N0 = Node->getOperand(0); 1032 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) 1033 break; 1034 1035 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 1036 1037 // Constant should be a mask. 1038 if (!isMask_64(C2)) 1039 break; 1040 1041 // This should be the only use of the AND unless we will use 1042 // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND 1043 // constants. 1044 if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) 1045 break; 1046 1047 // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this 1048 // optimization. 1049 if (isInt<12>(C2) || 1050 (C2 == UINT64_C(0xFFFF) && 1051 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || 1052 (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) 1053 break; 1054 1055 // We need to shift left the AND input and C1 by a total of XLen bits. 1056 1057 // How far left do we need to shift the AND input? 1058 unsigned XLen = Subtarget->getXLen(); 1059 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); 1060 1061 // The constant gets shifted by the remaining amount unless that would 1062 // shift bits out. 1063 uint64_t C1 = N1C->getZExtValue(); 1064 unsigned ConstantShift = XLen - LeadingZeros; 1065 if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) 1066 break; 1067 1068 uint64_t ShiftedC1 = C1 << ConstantShift; 1069 // If this RV32, we need to sign extend the constant. 1070 if (XLen == 32) 1071 ShiftedC1 = SignExtend64<32>(ShiftedC1); 1072 1073 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). 1074 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget); 1075 SDNode *SLLI = 1076 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), 1077 CurDAG->getTargetConstant(LeadingZeros, DL, VT)); 1078 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, 1079 SDValue(SLLI, 0), SDValue(Imm, 0)); 1080 ReplaceNode(Node, MULHU); 1081 return; 1082 } 1083 case ISD::INTRINSIC_WO_CHAIN: { 1084 unsigned IntNo = Node->getConstantOperandVal(0); 1085 switch (IntNo) { 1086 // By default we do not custom select any intrinsic. 1087 default: 1088 break; 1089 case Intrinsic::riscv_vmsgeu: 1090 case Intrinsic::riscv_vmsge: { 1091 SDValue Src1 = Node->getOperand(1); 1092 SDValue Src2 = Node->getOperand(2); 1093 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu; 1094 bool IsCmpUnsignedZero = false; 1095 // Only custom select scalar second operand. 1096 if (Src2.getValueType() != XLenVT) 1097 break; 1098 // Small constants are handled with patterns. 1099 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1100 int64_t CVal = C->getSExtValue(); 1101 if (CVal >= -15 && CVal <= 16) { 1102 if (!IsUnsigned || CVal != 0) 1103 break; 1104 IsCmpUnsignedZero = true; 1105 } 1106 } 1107 MVT Src1VT = Src1.getSimpleValueType(); 1108 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode; 1109 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1110 default: 1111 llvm_unreachable("Unexpected LMUL!"); 1112 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 1113 case RISCVII::VLMUL::lmulenum: \ 1114 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1115 : RISCV::PseudoVMSLT_VX_##suffix; \ 1116 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \ 1117 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 1118 break; 1119 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1) 1120 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2) 1121 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4) 1122 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8) 1123 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16) 1124 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32) 1125 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64) 1126 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES 1127 } 1128 SDValue SEW = CurDAG->getTargetConstant( 1129 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1130 SDValue VL; 1131 selectVLOp(Node->getOperand(3), VL); 1132 1133 // If vmsgeu with 0 immediate, expand it to vmset. 1134 if (IsCmpUnsignedZero) { 1135 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW)); 1136 return; 1137 } 1138 1139 // Expand to 1140 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd 1141 SDValue Cmp = SDValue( 1142 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1143 0); 1144 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT, 1145 {Cmp, Cmp, VL, SEW})); 1146 return; 1147 } 1148 case Intrinsic::riscv_vmsgeu_mask: 1149 case Intrinsic::riscv_vmsge_mask: { 1150 SDValue Src1 = Node->getOperand(2); 1151 SDValue Src2 = Node->getOperand(3); 1152 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask; 1153 bool IsCmpUnsignedZero = false; 1154 // Only custom select scalar second operand. 1155 if (Src2.getValueType() != XLenVT) 1156 break; 1157 // Small constants are handled with patterns. 1158 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1159 int64_t CVal = C->getSExtValue(); 1160 if (CVal >= -15 && CVal <= 16) { 1161 if (!IsUnsigned || CVal != 0) 1162 break; 1163 IsCmpUnsignedZero = true; 1164 } 1165 } 1166 MVT Src1VT = Src1.getSimpleValueType(); 1167 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode, 1168 VMOROpcode; 1169 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1170 default: 1171 llvm_unreachable("Unexpected LMUL!"); 1172 #define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \ 1173 case RISCVII::VLMUL::lmulenum: \ 1174 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1175 : RISCV::PseudoVMSLT_VX_##suffix; \ 1176 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \ 1177 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \ 1178 break; 1179 CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1) 1180 CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2) 1181 CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4) 1182 CASE_VMSLT_OPCODES(LMUL_1, M1, B8) 1183 CASE_VMSLT_OPCODES(LMUL_2, M2, B16) 1184 CASE_VMSLT_OPCODES(LMUL_4, M4, B32) 1185 CASE_VMSLT_OPCODES(LMUL_8, M8, B64) 1186 #undef CASE_VMSLT_OPCODES 1187 } 1188 // Mask operations use the LMUL from the mask type. 1189 switch (RISCVTargetLowering::getLMUL(VT)) { 1190 default: 1191 llvm_unreachable("Unexpected LMUL!"); 1192 #define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \ 1193 case RISCVII::VLMUL::lmulenum: \ 1194 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \ 1195 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \ 1196 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \ 1197 break; 1198 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8) 1199 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4) 1200 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2) 1201 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, M1) 1202 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, M2) 1203 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, M4) 1204 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, M8) 1205 #undef CASE_VMXOR_VMANDN_VMOR_OPCODES 1206 } 1207 SDValue SEW = CurDAG->getTargetConstant( 1208 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1209 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT); 1210 SDValue VL; 1211 selectVLOp(Node->getOperand(5), VL); 1212 SDValue MaskedOff = Node->getOperand(1); 1213 SDValue Mask = Node->getOperand(4); 1214 1215 // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff. 1216 if (IsCmpUnsignedZero) { 1217 // We don't need vmor if the MaskedOff and the Mask are the same 1218 // value. 1219 if (Mask == MaskedOff) { 1220 ReplaceUses(Node, Mask.getNode()); 1221 return; 1222 } 1223 ReplaceNode(Node, 1224 CurDAG->getMachineNode(VMOROpcode, DL, VT, 1225 {Mask, MaskedOff, VL, MaskSEW})); 1226 return; 1227 } 1228 1229 // If the MaskedOff value and the Mask are the same value use 1230 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt 1231 // This avoids needing to copy v0 to vd before starting the next sequence. 1232 if (Mask == MaskedOff) { 1233 SDValue Cmp = SDValue( 1234 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1235 0); 1236 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT, 1237 {Mask, Cmp, VL, MaskSEW})); 1238 return; 1239 } 1240 1241 // Mask needs to be copied to V0. 1242 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, 1243 RISCV::V0, Mask, SDValue()); 1244 SDValue Glue = Chain.getValue(1); 1245 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); 1246 1247 // Otherwise use 1248 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0 1249 // The result is mask undisturbed. 1250 // We use the same instructions to emulate mask agnostic behavior, because 1251 // the agnostic result can be either undisturbed or all 1. 1252 SDValue Cmp = SDValue( 1253 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, 1254 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), 1255 0); 1256 // vmxor.mm vd, vd, v0 is used to update active value. 1257 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, 1258 {Cmp, Mask, VL, MaskSEW})); 1259 return; 1260 } 1261 case Intrinsic::riscv_vsetvli_opt: 1262 case Intrinsic::riscv_vsetvlimax_opt: 1263 return selectVSETVLI(Node); 1264 } 1265 break; 1266 } 1267 case ISD::INTRINSIC_W_CHAIN: { 1268 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1269 switch (IntNo) { 1270 // By default we do not custom select any intrinsic. 1271 default: 1272 break; 1273 case Intrinsic::riscv_vsetvli: 1274 case Intrinsic::riscv_vsetvlimax: 1275 return selectVSETVLI(Node); 1276 case Intrinsic::riscv_vlseg2: 1277 case Intrinsic::riscv_vlseg3: 1278 case Intrinsic::riscv_vlseg4: 1279 case Intrinsic::riscv_vlseg5: 1280 case Intrinsic::riscv_vlseg6: 1281 case Intrinsic::riscv_vlseg7: 1282 case Intrinsic::riscv_vlseg8: { 1283 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1284 return; 1285 } 1286 case Intrinsic::riscv_vlseg2_mask: 1287 case Intrinsic::riscv_vlseg3_mask: 1288 case Intrinsic::riscv_vlseg4_mask: 1289 case Intrinsic::riscv_vlseg5_mask: 1290 case Intrinsic::riscv_vlseg6_mask: 1291 case Intrinsic::riscv_vlseg7_mask: 1292 case Intrinsic::riscv_vlseg8_mask: { 1293 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1294 return; 1295 } 1296 case Intrinsic::riscv_vlsseg2: 1297 case Intrinsic::riscv_vlsseg3: 1298 case Intrinsic::riscv_vlsseg4: 1299 case Intrinsic::riscv_vlsseg5: 1300 case Intrinsic::riscv_vlsseg6: 1301 case Intrinsic::riscv_vlsseg7: 1302 case Intrinsic::riscv_vlsseg8: { 1303 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1304 return; 1305 } 1306 case Intrinsic::riscv_vlsseg2_mask: 1307 case Intrinsic::riscv_vlsseg3_mask: 1308 case Intrinsic::riscv_vlsseg4_mask: 1309 case Intrinsic::riscv_vlsseg5_mask: 1310 case Intrinsic::riscv_vlsseg6_mask: 1311 case Intrinsic::riscv_vlsseg7_mask: 1312 case Intrinsic::riscv_vlsseg8_mask: { 1313 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1314 return; 1315 } 1316 case Intrinsic::riscv_vloxseg2: 1317 case Intrinsic::riscv_vloxseg3: 1318 case Intrinsic::riscv_vloxseg4: 1319 case Intrinsic::riscv_vloxseg5: 1320 case Intrinsic::riscv_vloxseg6: 1321 case Intrinsic::riscv_vloxseg7: 1322 case Intrinsic::riscv_vloxseg8: 1323 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1324 return; 1325 case Intrinsic::riscv_vluxseg2: 1326 case Intrinsic::riscv_vluxseg3: 1327 case Intrinsic::riscv_vluxseg4: 1328 case Intrinsic::riscv_vluxseg5: 1329 case Intrinsic::riscv_vluxseg6: 1330 case Intrinsic::riscv_vluxseg7: 1331 case Intrinsic::riscv_vluxseg8: 1332 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1333 return; 1334 case Intrinsic::riscv_vloxseg2_mask: 1335 case Intrinsic::riscv_vloxseg3_mask: 1336 case Intrinsic::riscv_vloxseg4_mask: 1337 case Intrinsic::riscv_vloxseg5_mask: 1338 case Intrinsic::riscv_vloxseg6_mask: 1339 case Intrinsic::riscv_vloxseg7_mask: 1340 case Intrinsic::riscv_vloxseg8_mask: 1341 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1342 return; 1343 case Intrinsic::riscv_vluxseg2_mask: 1344 case Intrinsic::riscv_vluxseg3_mask: 1345 case Intrinsic::riscv_vluxseg4_mask: 1346 case Intrinsic::riscv_vluxseg5_mask: 1347 case Intrinsic::riscv_vluxseg6_mask: 1348 case Intrinsic::riscv_vluxseg7_mask: 1349 case Intrinsic::riscv_vluxseg8_mask: 1350 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1351 return; 1352 case Intrinsic::riscv_vlseg8ff: 1353 case Intrinsic::riscv_vlseg7ff: 1354 case Intrinsic::riscv_vlseg6ff: 1355 case Intrinsic::riscv_vlseg5ff: 1356 case Intrinsic::riscv_vlseg4ff: 1357 case Intrinsic::riscv_vlseg3ff: 1358 case Intrinsic::riscv_vlseg2ff: { 1359 selectVLSEGFF(Node, /*IsMasked*/ false); 1360 return; 1361 } 1362 case Intrinsic::riscv_vlseg8ff_mask: 1363 case Intrinsic::riscv_vlseg7ff_mask: 1364 case Intrinsic::riscv_vlseg6ff_mask: 1365 case Intrinsic::riscv_vlseg5ff_mask: 1366 case Intrinsic::riscv_vlseg4ff_mask: 1367 case Intrinsic::riscv_vlseg3ff_mask: 1368 case Intrinsic::riscv_vlseg2ff_mask: { 1369 selectVLSEGFF(Node, /*IsMasked*/ true); 1370 return; 1371 } 1372 case Intrinsic::riscv_vloxei: 1373 case Intrinsic::riscv_vloxei_mask: 1374 case Intrinsic::riscv_vluxei: 1375 case Intrinsic::riscv_vluxei_mask: { 1376 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask || 1377 IntNo == Intrinsic::riscv_vluxei_mask; 1378 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei || 1379 IntNo == Intrinsic::riscv_vloxei_mask; 1380 1381 MVT VT = Node->getSimpleValueType(0); 1382 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1383 1384 unsigned CurOp = 2; 1385 // Masked intrinsic only have TU version pseduo instructions. 1386 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1387 SmallVector<SDValue, 8> Operands; 1388 if (IsTU) 1389 Operands.push_back(Node->getOperand(CurOp++)); 1390 else 1391 // Skip the undef passthru operand for nomask TA version pseudo 1392 CurOp++; 1393 1394 MVT IndexVT; 1395 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1396 /*IsStridedOrIndexed*/ true, Operands, 1397 /*IsLoad=*/true, &IndexVT); 1398 1399 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1400 "Element count mismatch"); 1401 1402 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1403 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1404 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1405 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1406 report_fatal_error("The V extension does not support EEW=64 for index " 1407 "values when XLEN=32"); 1408 } 1409 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( 1410 IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 1411 static_cast<unsigned>(IndexLMUL)); 1412 MachineSDNode *Load = 1413 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1414 1415 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1416 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1417 1418 ReplaceNode(Node, Load); 1419 return; 1420 } 1421 case Intrinsic::riscv_vlm: 1422 case Intrinsic::riscv_vle: 1423 case Intrinsic::riscv_vle_mask: 1424 case Intrinsic::riscv_vlse: 1425 case Intrinsic::riscv_vlse_mask: { 1426 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask || 1427 IntNo == Intrinsic::riscv_vlse_mask; 1428 bool IsStrided = 1429 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask; 1430 1431 MVT VT = Node->getSimpleValueType(0); 1432 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1433 1434 unsigned CurOp = 2; 1435 // The riscv_vlm intrinsic are always tail agnostic and no passthru operand. 1436 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm; 1437 // Masked intrinsic only have TU version pseduo instructions. 1438 bool IsTU = 1439 HasPassthruOperand && 1440 ((!IsMasked && !Node->getOperand(CurOp).isUndef()) || IsMasked); 1441 SmallVector<SDValue, 8> Operands; 1442 if (IsTU) 1443 Operands.push_back(Node->getOperand(CurOp++)); 1444 else if (HasPassthruOperand) 1445 // Skip the undef passthru operand for nomask TA version pseudo 1446 CurOp++; 1447 1448 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1449 Operands, /*IsLoad=*/true); 1450 1451 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1452 const RISCV::VLEPseudo *P = 1453 RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 1454 static_cast<unsigned>(LMUL)); 1455 MachineSDNode *Load = 1456 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1457 1458 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1459 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1460 1461 ReplaceNode(Node, Load); 1462 return; 1463 } 1464 case Intrinsic::riscv_vleff: 1465 case Intrinsic::riscv_vleff_mask: { 1466 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask; 1467 1468 MVT VT = Node->getSimpleValueType(0); 1469 unsigned SEW = VT.getScalarSizeInBits(); 1470 unsigned Log2SEW = Log2_32(SEW); 1471 1472 unsigned CurOp = 2; 1473 // Masked intrinsic only have TU version pseduo instructions. 1474 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1475 SmallVector<SDValue, 7> Operands; 1476 if (IsTU) 1477 Operands.push_back(Node->getOperand(CurOp++)); 1478 else 1479 // Skip the undef passthru operand for nomask TA version pseudo 1480 CurOp++; 1481 1482 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1483 /*IsStridedOrIndexed*/ false, Operands, 1484 /*IsLoad=*/true); 1485 1486 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1487 const RISCV::VLEPseudo *P = 1488 RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 1489 Log2SEW, static_cast<unsigned>(LMUL)); 1490 MachineSDNode *Load = 1491 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), 1492 MVT::Other, MVT::Glue, Operands); 1493 bool TailAgnostic = !IsTU; 1494 bool MaskAgnostic = false; 1495 if (IsMasked) { 1496 uint64_t Policy = 1497 Node->getConstantOperandVal(Node->getNumOperands() - 1); 1498 TailAgnostic = Policy & RISCVII::TAIL_AGNOSTIC; 1499 MaskAgnostic = Policy & RISCVII::MASK_AGNOSTIC; 1500 } 1501 unsigned VType = 1502 RISCVVType::encodeVTYPE(LMUL, SEW, TailAgnostic, MaskAgnostic); 1503 SDValue VTypeOp = CurDAG->getTargetConstant(VType, DL, XLenVT); 1504 SDNode *ReadVL = 1505 CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, VTypeOp, 1506 /*Glue*/ SDValue(Load, 2)); 1507 1508 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1509 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1510 1511 ReplaceUses(SDValue(Node, 0), SDValue(Load, 0)); 1512 ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL 1513 ReplaceUses(SDValue(Node, 2), SDValue(Load, 1)); // Chain 1514 CurDAG->RemoveDeadNode(Node); 1515 return; 1516 } 1517 } 1518 break; 1519 } 1520 case ISD::INTRINSIC_VOID: { 1521 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1522 switch (IntNo) { 1523 case Intrinsic::riscv_vsseg2: 1524 case Intrinsic::riscv_vsseg3: 1525 case Intrinsic::riscv_vsseg4: 1526 case Intrinsic::riscv_vsseg5: 1527 case Intrinsic::riscv_vsseg6: 1528 case Intrinsic::riscv_vsseg7: 1529 case Intrinsic::riscv_vsseg8: { 1530 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1531 return; 1532 } 1533 case Intrinsic::riscv_vsseg2_mask: 1534 case Intrinsic::riscv_vsseg3_mask: 1535 case Intrinsic::riscv_vsseg4_mask: 1536 case Intrinsic::riscv_vsseg5_mask: 1537 case Intrinsic::riscv_vsseg6_mask: 1538 case Intrinsic::riscv_vsseg7_mask: 1539 case Intrinsic::riscv_vsseg8_mask: { 1540 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1541 return; 1542 } 1543 case Intrinsic::riscv_vssseg2: 1544 case Intrinsic::riscv_vssseg3: 1545 case Intrinsic::riscv_vssseg4: 1546 case Intrinsic::riscv_vssseg5: 1547 case Intrinsic::riscv_vssseg6: 1548 case Intrinsic::riscv_vssseg7: 1549 case Intrinsic::riscv_vssseg8: { 1550 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1551 return; 1552 } 1553 case Intrinsic::riscv_vssseg2_mask: 1554 case Intrinsic::riscv_vssseg3_mask: 1555 case Intrinsic::riscv_vssseg4_mask: 1556 case Intrinsic::riscv_vssseg5_mask: 1557 case Intrinsic::riscv_vssseg6_mask: 1558 case Intrinsic::riscv_vssseg7_mask: 1559 case Intrinsic::riscv_vssseg8_mask: { 1560 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1561 return; 1562 } 1563 case Intrinsic::riscv_vsoxseg2: 1564 case Intrinsic::riscv_vsoxseg3: 1565 case Intrinsic::riscv_vsoxseg4: 1566 case Intrinsic::riscv_vsoxseg5: 1567 case Intrinsic::riscv_vsoxseg6: 1568 case Intrinsic::riscv_vsoxseg7: 1569 case Intrinsic::riscv_vsoxseg8: 1570 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1571 return; 1572 case Intrinsic::riscv_vsuxseg2: 1573 case Intrinsic::riscv_vsuxseg3: 1574 case Intrinsic::riscv_vsuxseg4: 1575 case Intrinsic::riscv_vsuxseg5: 1576 case Intrinsic::riscv_vsuxseg6: 1577 case Intrinsic::riscv_vsuxseg7: 1578 case Intrinsic::riscv_vsuxseg8: 1579 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1580 return; 1581 case Intrinsic::riscv_vsoxseg2_mask: 1582 case Intrinsic::riscv_vsoxseg3_mask: 1583 case Intrinsic::riscv_vsoxseg4_mask: 1584 case Intrinsic::riscv_vsoxseg5_mask: 1585 case Intrinsic::riscv_vsoxseg6_mask: 1586 case Intrinsic::riscv_vsoxseg7_mask: 1587 case Intrinsic::riscv_vsoxseg8_mask: 1588 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1589 return; 1590 case Intrinsic::riscv_vsuxseg2_mask: 1591 case Intrinsic::riscv_vsuxseg3_mask: 1592 case Intrinsic::riscv_vsuxseg4_mask: 1593 case Intrinsic::riscv_vsuxseg5_mask: 1594 case Intrinsic::riscv_vsuxseg6_mask: 1595 case Intrinsic::riscv_vsuxseg7_mask: 1596 case Intrinsic::riscv_vsuxseg8_mask: 1597 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1598 return; 1599 case Intrinsic::riscv_vsoxei: 1600 case Intrinsic::riscv_vsoxei_mask: 1601 case Intrinsic::riscv_vsuxei: 1602 case Intrinsic::riscv_vsuxei_mask: { 1603 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask || 1604 IntNo == Intrinsic::riscv_vsuxei_mask; 1605 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei || 1606 IntNo == Intrinsic::riscv_vsoxei_mask; 1607 1608 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1609 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1610 1611 unsigned CurOp = 2; 1612 SmallVector<SDValue, 8> Operands; 1613 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1614 1615 MVT IndexVT; 1616 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1617 /*IsStridedOrIndexed*/ true, Operands, 1618 /*IsLoad=*/false, &IndexVT); 1619 1620 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1621 "Element count mismatch"); 1622 1623 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1624 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1625 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1626 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1627 report_fatal_error("The V extension does not support EEW=64 for index " 1628 "values when XLEN=32"); 1629 } 1630 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( 1631 IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW, 1632 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL)); 1633 MachineSDNode *Store = 1634 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1635 1636 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1637 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1638 1639 ReplaceNode(Node, Store); 1640 return; 1641 } 1642 case Intrinsic::riscv_vsm: 1643 case Intrinsic::riscv_vse: 1644 case Intrinsic::riscv_vse_mask: 1645 case Intrinsic::riscv_vsse: 1646 case Intrinsic::riscv_vsse_mask: { 1647 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask || 1648 IntNo == Intrinsic::riscv_vsse_mask; 1649 bool IsStrided = 1650 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask; 1651 1652 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1653 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1654 1655 unsigned CurOp = 2; 1656 SmallVector<SDValue, 8> Operands; 1657 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1658 1659 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1660 Operands); 1661 1662 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1663 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( 1664 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 1665 MachineSDNode *Store = 1666 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1667 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1668 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1669 1670 ReplaceNode(Node, Store); 1671 return; 1672 } 1673 } 1674 break; 1675 } 1676 case ISD::BITCAST: { 1677 MVT SrcVT = Node->getOperand(0).getSimpleValueType(); 1678 // Just drop bitcasts between vectors if both are fixed or both are 1679 // scalable. 1680 if ((VT.isScalableVector() && SrcVT.isScalableVector()) || 1681 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) { 1682 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 1683 CurDAG->RemoveDeadNode(Node); 1684 return; 1685 } 1686 break; 1687 } 1688 case ISD::INSERT_SUBVECTOR: { 1689 SDValue V = Node->getOperand(0); 1690 SDValue SubV = Node->getOperand(1); 1691 SDLoc DL(SubV); 1692 auto Idx = Node->getConstantOperandVal(2); 1693 MVT SubVecVT = SubV.getSimpleValueType(); 1694 1695 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1696 MVT SubVecContainerVT = SubVecVT; 1697 // Establish the correct scalable-vector types for any fixed-length type. 1698 if (SubVecVT.isFixedLengthVector()) 1699 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT); 1700 if (VT.isFixedLengthVector()) 1701 VT = TLI.getContainerForFixedLengthVector(VT); 1702 1703 const auto *TRI = Subtarget->getRegisterInfo(); 1704 unsigned SubRegIdx; 1705 std::tie(SubRegIdx, Idx) = 1706 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1707 VT, SubVecContainerVT, Idx, TRI); 1708 1709 // If the Idx hasn't been completely eliminated then this is a subvector 1710 // insert which doesn't naturally align to a vector register. These must 1711 // be handled using instructions to manipulate the vector registers. 1712 if (Idx != 0) 1713 break; 1714 1715 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT); 1716 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 || 1717 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 || 1718 SubVecLMUL == RISCVII::VLMUL::LMUL_F8; 1719 (void)IsSubVecPartReg; // Silence unused variable warning without asserts. 1720 assert((!IsSubVecPartReg || V.isUndef()) && 1721 "Expecting lowering to have created legal INSERT_SUBVECTORs when " 1722 "the subvector is smaller than a full-sized register"); 1723 1724 // If we haven't set a SubRegIdx, then we must be going between 1725 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. 1726 if (SubRegIdx == RISCV::NoSubRegister) { 1727 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT); 1728 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1729 InRegClassID && 1730 "Unexpected subvector extraction"); 1731 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1732 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1733 DL, VT, SubV, RC); 1734 ReplaceNode(Node, NewNode); 1735 return; 1736 } 1737 1738 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV); 1739 ReplaceNode(Node, Insert.getNode()); 1740 return; 1741 } 1742 case ISD::EXTRACT_SUBVECTOR: { 1743 SDValue V = Node->getOperand(0); 1744 auto Idx = Node->getConstantOperandVal(1); 1745 MVT InVT = V.getSimpleValueType(); 1746 SDLoc DL(V); 1747 1748 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1749 MVT SubVecContainerVT = VT; 1750 // Establish the correct scalable-vector types for any fixed-length type. 1751 if (VT.isFixedLengthVector()) 1752 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT); 1753 if (InVT.isFixedLengthVector()) 1754 InVT = TLI.getContainerForFixedLengthVector(InVT); 1755 1756 const auto *TRI = Subtarget->getRegisterInfo(); 1757 unsigned SubRegIdx; 1758 std::tie(SubRegIdx, Idx) = 1759 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1760 InVT, SubVecContainerVT, Idx, TRI); 1761 1762 // If the Idx hasn't been completely eliminated then this is a subvector 1763 // extract which doesn't naturally align to a vector register. These must 1764 // be handled using instructions to manipulate the vector registers. 1765 if (Idx != 0) 1766 break; 1767 1768 // If we haven't set a SubRegIdx, then we must be going between 1769 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy. 1770 if (SubRegIdx == RISCV::NoSubRegister) { 1771 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT); 1772 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1773 InRegClassID && 1774 "Unexpected subvector extraction"); 1775 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1776 SDNode *NewNode = 1777 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC); 1778 ReplaceNode(Node, NewNode); 1779 return; 1780 } 1781 1782 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V); 1783 ReplaceNode(Node, Extract.getNode()); 1784 return; 1785 } 1786 case ISD::SPLAT_VECTOR: 1787 case RISCVISD::VMV_S_X_VL: 1788 case RISCVISD::VFMV_S_F_VL: 1789 case RISCVISD::VMV_V_X_VL: 1790 case RISCVISD::VFMV_V_F_VL: { 1791 // Try to match splat of a scalar load to a strided load with stride of x0. 1792 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL || 1793 Node->getOpcode() == RISCVISD::VFMV_S_F_VL; 1794 bool HasPassthruOperand = Node->getOpcode() != ISD::SPLAT_VECTOR; 1795 if (HasPassthruOperand && !Node->getOperand(0).isUndef()) 1796 break; 1797 SDValue Src = HasPassthruOperand ? Node->getOperand(1) : Node->getOperand(0); 1798 auto *Ld = dyn_cast<LoadSDNode>(Src); 1799 if (!Ld) 1800 break; 1801 EVT MemVT = Ld->getMemoryVT(); 1802 // The memory VT should be the same size as the element type. 1803 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize()) 1804 break; 1805 if (!IsProfitableToFold(Src, Node, Node) || 1806 !IsLegalToFold(Src, Node, Node, TM.getOptLevel())) 1807 break; 1808 1809 SDValue VL; 1810 if (Node->getOpcode() == ISD::SPLAT_VECTOR) 1811 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); 1812 else if (IsScalarMove) { 1813 // We could deal with more VL if we update the VSETVLI insert pass to 1814 // avoid introducing more VSETVLI. 1815 if (!isOneConstant(Node->getOperand(2))) 1816 break; 1817 selectVLOp(Node->getOperand(2), VL); 1818 } else 1819 selectVLOp(Node->getOperand(2), VL); 1820 1821 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1822 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 1823 1824 SDValue Operands[] = {Ld->getBasePtr(), 1825 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW, 1826 Ld->getChain()}; 1827 1828 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1829 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo( 1830 /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false, 1831 Log2SEW, static_cast<unsigned>(LMUL)); 1832 MachineSDNode *Load = 1833 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1834 1835 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()}); 1836 1837 ReplaceNode(Node, Load); 1838 return; 1839 } 1840 } 1841 1842 // Select the default instruction. 1843 SelectCode(Node); 1844 } 1845 1846 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand( 1847 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 1848 switch (ConstraintID) { 1849 case InlineAsm::Constraint_m: 1850 // We just support simple memory operands that have a single address 1851 // operand and need no special handling. 1852 OutOps.push_back(Op); 1853 return false; 1854 case InlineAsm::Constraint_A: 1855 OutOps.push_back(Op); 1856 return false; 1857 default: 1858 break; 1859 } 1860 1861 return true; 1862 } 1863 1864 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) { 1865 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1866 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1867 return true; 1868 } 1869 return false; 1870 } 1871 1872 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) { 1873 // If this is FrameIndex, select it directly. Otherwise just let it get 1874 // selected to a register independently. 1875 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) 1876 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1877 else 1878 Base = Addr; 1879 return true; 1880 } 1881 1882 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, 1883 SDValue &ShAmt) { 1884 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift 1885 // amount. If there is an AND on the shift amount, we can bypass it if it 1886 // doesn't affect any of those bits. 1887 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 1888 const APInt &AndMask = N->getConstantOperandAPInt(1); 1889 1890 // Since the max shift amount is a power of 2 we can subtract 1 to make a 1891 // mask that covers the bits needed to represent all shift amounts. 1892 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!"); 1893 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); 1894 1895 if (ShMask.isSubsetOf(AndMask)) { 1896 ShAmt = N.getOperand(0); 1897 return true; 1898 } 1899 1900 // SimplifyDemandedBits may have optimized the mask so try restoring any 1901 // bits that are known zero. 1902 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0)); 1903 if (ShMask.isSubsetOf(AndMask | Known.Zero)) { 1904 ShAmt = N.getOperand(0); 1905 return true; 1906 } 1907 } else if (N.getOpcode() == ISD::SUB && 1908 isa<ConstantSDNode>(N.getOperand(0))) { 1909 uint64_t Imm = N.getConstantOperandVal(0); 1910 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to 1911 // generate a NEG instead of a SUB of a constant. 1912 if (Imm != 0 && Imm % ShiftWidth == 0) { 1913 SDLoc DL(N); 1914 EVT VT = N.getValueType(); 1915 SDValue Zero = 1916 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT); 1917 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB; 1918 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero, 1919 N.getOperand(1)); 1920 ShAmt = SDValue(Neg, 0); 1921 return true; 1922 } 1923 } 1924 1925 ShAmt = N; 1926 return true; 1927 } 1928 1929 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) { 1930 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG && 1931 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) { 1932 Val = N.getOperand(0); 1933 return true; 1934 } 1935 MVT VT = N.getSimpleValueType(); 1936 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) { 1937 Val = N; 1938 return true; 1939 } 1940 1941 return false; 1942 } 1943 1944 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) { 1945 if (N.getOpcode() == ISD::AND) { 1946 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1947 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) { 1948 Val = N.getOperand(0); 1949 return true; 1950 } 1951 } 1952 MVT VT = N.getSimpleValueType(); 1953 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32); 1954 if (CurDAG->MaskedValueIsZero(N, Mask)) { 1955 Val = N; 1956 return true; 1957 } 1958 1959 return false; 1960 } 1961 1962 // Return true if all users of this SDNode* only consume the lower \p Bits. 1963 // This can be used to form W instructions for add/sub/mul/shl even when the 1964 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if 1965 // SimplifyDemandedBits has made it so some users see a sext_inreg and some 1966 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave 1967 // the add/sub/mul/shl to become non-W instructions. By checking the users we 1968 // may be able to use a W instruction and CSE with the other instruction if 1969 // this has happened. We could try to detect that the CSE opportunity exists 1970 // before doing this, but that would be more complicated. 1971 // TODO: Does this need to look through AND/OR/XOR to their users to find more 1972 // opportunities. 1973 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const { 1974 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB || 1975 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL || 1976 Node->getOpcode() == ISD::SRL || 1977 Node->getOpcode() == ISD::SIGN_EXTEND_INREG || 1978 Node->getOpcode() == RISCVISD::GREV || 1979 Node->getOpcode() == RISCVISD::GORC || 1980 isa<ConstantSDNode>(Node)) && 1981 "Unexpected opcode"); 1982 1983 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 1984 SDNode *User = *UI; 1985 // Users of this node should have already been instruction selected 1986 if (!User->isMachineOpcode()) 1987 return false; 1988 1989 // TODO: Add more opcodes? 1990 switch (User->getMachineOpcode()) { 1991 default: 1992 return false; 1993 case RISCV::ADDW: 1994 case RISCV::ADDIW: 1995 case RISCV::SUBW: 1996 case RISCV::MULW: 1997 case RISCV::SLLW: 1998 case RISCV::SLLIW: 1999 case RISCV::SRAW: 2000 case RISCV::SRAIW: 2001 case RISCV::SRLW: 2002 case RISCV::SRLIW: 2003 case RISCV::DIVW: 2004 case RISCV::DIVUW: 2005 case RISCV::REMW: 2006 case RISCV::REMUW: 2007 case RISCV::ROLW: 2008 case RISCV::RORW: 2009 case RISCV::RORIW: 2010 case RISCV::CLZW: 2011 case RISCV::CTZW: 2012 case RISCV::CPOPW: 2013 case RISCV::SLLI_UW: 2014 case RISCV::FMV_W_X: 2015 case RISCV::FCVT_H_W: 2016 case RISCV::FCVT_H_WU: 2017 case RISCV::FCVT_S_W: 2018 case RISCV::FCVT_S_WU: 2019 case RISCV::FCVT_D_W: 2020 case RISCV::FCVT_D_WU: 2021 if (Bits < 32) 2022 return false; 2023 break; 2024 case RISCV::SLLI: 2025 // SLLI only uses the lower (XLen - ShAmt) bits. 2026 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1)) 2027 return false; 2028 break; 2029 case RISCV::ANDI: 2030 if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1)))) 2031 return false; 2032 break; 2033 case RISCV::SEXT_B: 2034 if (Bits < 8) 2035 return false; 2036 break; 2037 case RISCV::SEXT_H: 2038 case RISCV::FMV_H_X: 2039 case RISCV::ZEXT_H_RV32: 2040 case RISCV::ZEXT_H_RV64: 2041 if (Bits < 16) 2042 return false; 2043 break; 2044 case RISCV::ADD_UW: 2045 case RISCV::SH1ADD_UW: 2046 case RISCV::SH2ADD_UW: 2047 case RISCV::SH3ADD_UW: 2048 // The first operand to add.uw/shXadd.uw is implicitly zero extended from 2049 // 32 bits. 2050 if (UI.getOperandNo() != 0 || Bits < 32) 2051 return false; 2052 break; 2053 case RISCV::SB: 2054 if (UI.getOperandNo() != 0 || Bits < 8) 2055 return false; 2056 break; 2057 case RISCV::SH: 2058 if (UI.getOperandNo() != 0 || Bits < 16) 2059 return false; 2060 break; 2061 case RISCV::SW: 2062 if (UI.getOperandNo() != 0 || Bits < 32) 2063 return false; 2064 break; 2065 } 2066 } 2067 2068 return true; 2069 } 2070 2071 // Select VL as a 5 bit immediate or a value that will become a register. This 2072 // allows us to choose betwen VSETIVLI or VSETVLI later. 2073 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { 2074 auto *C = dyn_cast<ConstantSDNode>(N); 2075 if (C && isUInt<5>(C->getZExtValue())) { 2076 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N), 2077 N->getValueType(0)); 2078 } else if (C && C->isAllOnesValue()) { 2079 // Treat all ones as VLMax. 2080 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2081 N->getValueType(0)); 2082 } else if (isa<RegisterSDNode>(N) && 2083 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) { 2084 // All our VL operands use an operand that allows GPRNoX0 or an immediate 2085 // as the register class. Convert X0 to a special immediate to pass the 2086 // MachineVerifier. This is recognized specially by the vsetvli insertion 2087 // pass. 2088 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2089 N->getValueType(0)); 2090 } else { 2091 VL = N; 2092 } 2093 2094 return true; 2095 } 2096 2097 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { 2098 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef()) 2099 return false; 2100 SplatVal = N.getOperand(1); 2101 return true; 2102 } 2103 2104 using ValidateFn = bool (*)(int64_t); 2105 2106 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, 2107 SelectionDAG &DAG, 2108 const RISCVSubtarget &Subtarget, 2109 ValidateFn ValidateImm) { 2110 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2111 !isa<ConstantSDNode>(N.getOperand(1))) 2112 return false; 2113 2114 int64_t SplatImm = 2115 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2116 2117 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand 2118 // type is wider than the resulting vector element type: an implicit 2119 // truncation first takes place. Therefore, perform a manual 2120 // truncation/sign-extension in order to ignore any truncated bits and catch 2121 // any zero-extended immediate. 2122 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first 2123 // sign-extending to (XLenVT -1). 2124 MVT XLenVT = Subtarget.getXLenVT(); 2125 assert(XLenVT == N.getOperand(1).getSimpleValueType() && 2126 "Unexpected splat operand type"); 2127 MVT EltVT = N.getSimpleValueType().getVectorElementType(); 2128 if (EltVT.bitsLT(XLenVT)) 2129 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits()); 2130 2131 if (!ValidateImm(SplatImm)) 2132 return false; 2133 2134 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT); 2135 return true; 2136 } 2137 2138 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) { 2139 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget, 2140 [](int64_t Imm) { return isInt<5>(Imm); }); 2141 } 2142 2143 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) { 2144 return selectVSplatSimmHelper( 2145 N, SplatVal, *CurDAG, *Subtarget, 2146 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; }); 2147 } 2148 2149 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N, 2150 SDValue &SplatVal) { 2151 return selectVSplatSimmHelper( 2152 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) { 2153 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16); 2154 }); 2155 } 2156 2157 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) { 2158 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2159 !isa<ConstantSDNode>(N.getOperand(1))) 2160 return false; 2161 2162 int64_t SplatImm = 2163 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2164 2165 if (!isUInt<5>(SplatImm)) 2166 return false; 2167 2168 SplatVal = 2169 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT()); 2170 2171 return true; 2172 } 2173 2174 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width, 2175 SDValue &Imm) { 2176 if (auto *C = dyn_cast<ConstantSDNode>(N)) { 2177 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width); 2178 2179 if (!isInt<5>(ImmVal)) 2180 return false; 2181 2182 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT()); 2183 return true; 2184 } 2185 2186 return false; 2187 } 2188 2189 // Merge an ADDI into the offset of a load/store instruction where possible. 2190 // (load (addi base, off1), off2) -> (load base, off1+off2) 2191 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2) 2192 // (load (add base, (addi src, off1)), off2) 2193 // -> (load (add base, src), off1+off2) 2194 // (store val, (add base, (addi src, off1)), off2) 2195 // -> (store val, (add base, src), off1+off2) 2196 // This is possible when off1+off2 fits a 12-bit immediate. 2197 bool RISCVDAGToDAGISel::doPeepholeLoadStoreADDI(SDNode *N) { 2198 unsigned OffsetOpIdx, BaseOpIdx; 2199 if (!hasMemOffset(N, BaseOpIdx, OffsetOpIdx)) 2200 return false; 2201 2202 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx))) 2203 return false; 2204 2205 SDValue Base = N->getOperand(BaseOpIdx); 2206 2207 if (!Base.isMachineOpcode()) 2208 return false; 2209 2210 // If the base is an ADDI, we can merge it in to the load/store. 2211 if (Base.getMachineOpcode() != RISCV::ADDI) 2212 return false; 2213 2214 SDValue ImmOperand = Base.getOperand(1); 2215 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx); 2216 2217 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) { 2218 int64_t Offset1 = Const->getSExtValue(); 2219 int64_t CombinedOffset = Offset1 + Offset2; 2220 if (!isInt<12>(CombinedOffset)) 2221 return false; 2222 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand), 2223 ImmOperand.getValueType()); 2224 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) { 2225 // If the off1 in (addi base, off1) is a global variable's address (its 2226 // low part, really), then we can rely on the alignment of that variable 2227 // to provide a margin of safety before off1 can overflow the 12 bits. 2228 // Check if off2 falls within that margin; if so off1+off2 can't overflow. 2229 const DataLayout &DL = CurDAG->getDataLayout(); 2230 Align Alignment = GA->getGlobal()->getPointerAlignment(DL); 2231 if (Offset2 != 0 && Alignment <= Offset2) 2232 return false; 2233 int64_t Offset1 = GA->getOffset(); 2234 int64_t CombinedOffset = Offset1 + Offset2; 2235 ImmOperand = CurDAG->getTargetGlobalAddress( 2236 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(), 2237 CombinedOffset, GA->getTargetFlags()); 2238 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) { 2239 // Ditto. 2240 Align Alignment = CP->getAlign(); 2241 if (Offset2 != 0 && Alignment <= Offset2) 2242 return false; 2243 int64_t Offset1 = CP->getOffset(); 2244 int64_t CombinedOffset = Offset1 + Offset2; 2245 ImmOperand = CurDAG->getTargetConstantPool( 2246 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(), 2247 CombinedOffset, CP->getTargetFlags()); 2248 } else { 2249 return false; 2250 } 2251 2252 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 2253 LLVM_DEBUG(Base->dump(CurDAG)); 2254 LLVM_DEBUG(dbgs() << "\nN: "); 2255 LLVM_DEBUG(N->dump(CurDAG)); 2256 LLVM_DEBUG(dbgs() << "\n"); 2257 2258 // Modify the offset operand of the load/store. 2259 if (BaseOpIdx == 0) { // Load 2260 N = CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand, 2261 N->getOperand(2)); 2262 } else { // Store 2263 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0), 2264 ImmOperand, N->getOperand(3)); 2265 } 2266 2267 return true; 2268 } 2269 2270 // Try to remove sext.w if the input is a W instruction or can be made into 2271 // a W instruction cheaply. 2272 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { 2273 // Look for the sext.w pattern, addiw rd, rs1, 0. 2274 if (N->getMachineOpcode() != RISCV::ADDIW || 2275 !isNullConstant(N->getOperand(1))) 2276 return false; 2277 2278 SDValue N0 = N->getOperand(0); 2279 if (!N0.isMachineOpcode()) 2280 return false; 2281 2282 switch (N0.getMachineOpcode()) { 2283 default: 2284 break; 2285 case RISCV::ADD: 2286 case RISCV::ADDI: 2287 case RISCV::SUB: 2288 case RISCV::MUL: 2289 case RISCV::SLLI: { 2290 // Convert sext.w+add/sub/mul to their W instructions. This will create 2291 // a new independent instruction. This improves latency. 2292 unsigned Opc; 2293 switch (N0.getMachineOpcode()) { 2294 default: 2295 llvm_unreachable("Unexpected opcode!"); 2296 case RISCV::ADD: Opc = RISCV::ADDW; break; 2297 case RISCV::ADDI: Opc = RISCV::ADDIW; break; 2298 case RISCV::SUB: Opc = RISCV::SUBW; break; 2299 case RISCV::MUL: Opc = RISCV::MULW; break; 2300 case RISCV::SLLI: Opc = RISCV::SLLIW; break; 2301 } 2302 2303 SDValue N00 = N0.getOperand(0); 2304 SDValue N01 = N0.getOperand(1); 2305 2306 // Shift amount needs to be uimm5. 2307 if (N0.getMachineOpcode() == RISCV::SLLI && 2308 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue())) 2309 break; 2310 2311 SDNode *Result = 2312 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), 2313 N00, N01); 2314 ReplaceUses(N, Result); 2315 return true; 2316 } 2317 case RISCV::ADDW: 2318 case RISCV::ADDIW: 2319 case RISCV::SUBW: 2320 case RISCV::MULW: 2321 case RISCV::SLLIW: 2322 case RISCV::GREVIW: 2323 case RISCV::GORCIW: 2324 // Result is already sign extended just remove the sext.w. 2325 // NOTE: We only handle the nodes that are selected with hasAllWUsers. 2326 ReplaceUses(N, N0.getNode()); 2327 return true; 2328 } 2329 2330 return false; 2331 } 2332 2333 // Optimize masked RVV pseudo instructions with a known all-ones mask to their 2334 // corresponding "unmasked" pseudo versions. The mask we're interested in will 2335 // take the form of a V0 physical register operand, with a glued 2336 // register-setting instruction. 2337 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) { 2338 const RISCV::RISCVMaskedPseudoInfo *I = 2339 RISCV::getMaskedPseudoInfo(N->getMachineOpcode()); 2340 if (!I) 2341 return false; 2342 2343 unsigned MaskOpIdx = I->MaskOpIdx; 2344 2345 // Check that we're using V0 as a mask register. 2346 if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) || 2347 cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0) 2348 return false; 2349 2350 // The glued user defines V0. 2351 const auto *Glued = N->getGluedNode(); 2352 2353 if (!Glued || Glued->getOpcode() != ISD::CopyToReg) 2354 return false; 2355 2356 // Check that we're defining V0 as a mask register. 2357 if (!isa<RegisterSDNode>(Glued->getOperand(1)) || 2358 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0) 2359 return false; 2360 2361 // Check the instruction defining V0; it needs to be a VMSET pseudo. 2362 SDValue MaskSetter = Glued->getOperand(2); 2363 2364 const auto IsVMSet = [](unsigned Opc) { 2365 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 || 2366 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 || 2367 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 || 2368 Opc == RISCV::PseudoVMSET_M_B8; 2369 }; 2370 2371 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has 2372 // undefined behaviour if it's the wrong bitwidth, so we could choose to 2373 // assume that it's all-ones? Same applies to its VL. 2374 if (!MaskSetter->isMachineOpcode() || !IsVMSet(MaskSetter.getMachineOpcode())) 2375 return false; 2376 2377 // Retrieve the tail policy operand index, if any. 2378 Optional<unsigned> TailPolicyOpIdx; 2379 const RISCVInstrInfo *TII = static_cast<const RISCVInstrInfo *>( 2380 CurDAG->getSubtarget().getInstrInfo()); 2381 2382 const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode()); 2383 2384 bool IsTA = true; 2385 if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) { 2386 // The last operand of the pseudo is the policy op, but we might have a 2387 // Glue operand last. We might also have a chain. 2388 TailPolicyOpIdx = N->getNumOperands() - 1; 2389 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Glue) 2390 (*TailPolicyOpIdx)--; 2391 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Other) 2392 (*TailPolicyOpIdx)--; 2393 2394 if (!(N->getConstantOperandVal(*TailPolicyOpIdx) & 2395 RISCVII::TAIL_AGNOSTIC)) { 2396 // Keep the true-masked instruction when there is no unmasked TU 2397 // instruction 2398 if (I->UnmaskedTUPseudo == I->MaskedPseudo && !N->getOperand(0).isUndef()) 2399 return false; 2400 // We can't use TA if the tie-operand is not IMPLICIT_DEF 2401 if (!N->getOperand(0).isUndef()) 2402 IsTA = false; 2403 } 2404 } 2405 2406 if (IsTA) { 2407 uint64_t TSFlags = TII->get(I->UnmaskedPseudo).TSFlags; 2408 2409 // Check that we're dropping the merge operand, the mask operand, and any 2410 // policy operand when we transform to this unmasked pseudo. 2411 assert(!RISCVII::hasMergeOp(TSFlags) && RISCVII::hasDummyMaskOp(TSFlags) && 2412 !RISCVII::hasVecPolicyOp(TSFlags) && 2413 "Unexpected pseudo to transform to"); 2414 (void)TSFlags; 2415 } else { 2416 uint64_t TSFlags = TII->get(I->UnmaskedTUPseudo).TSFlags; 2417 2418 // Check that we're dropping the mask operand, and any policy operand 2419 // when we transform to this unmasked tu pseudo. 2420 assert(RISCVII::hasMergeOp(TSFlags) && RISCVII::hasDummyMaskOp(TSFlags) && 2421 !RISCVII::hasVecPolicyOp(TSFlags) && 2422 "Unexpected pseudo to transform to"); 2423 (void)TSFlags; 2424 } 2425 2426 unsigned Opc = IsTA ? I->UnmaskedPseudo : I->UnmaskedTUPseudo; 2427 SmallVector<SDValue, 8> Ops; 2428 // Skip the merge operand at index 0 if IsTA 2429 for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) { 2430 // Skip the mask, the policy, and the Glue. 2431 SDValue Op = N->getOperand(I); 2432 if (I == MaskOpIdx || I == TailPolicyOpIdx || 2433 Op.getValueType() == MVT::Glue) 2434 continue; 2435 Ops.push_back(Op); 2436 } 2437 2438 // Transitively apply any node glued to our new node. 2439 if (auto *TGlued = Glued->getGluedNode()) 2440 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1)); 2441 2442 SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 2443 ReplaceUses(N, Result); 2444 2445 return true; 2446 } 2447 2448 // This pass converts a legalized DAG into a RISCV-specific DAG, ready 2449 // for instruction scheduling. 2450 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM, 2451 CodeGenOpt::Level OptLevel) { 2452 return new RISCVDAGToDAGISel(TM, OptLevel); 2453 } 2454