1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the RISCV target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVISelDAGToDAG.h" 14 #include "MCTargetDesc/RISCVMCTargetDesc.h" 15 #include "MCTargetDesc/RISCVMatInt.h" 16 #include "RISCVISelLowering.h" 17 #include "RISCVMachineFunctionInfo.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/IR/IntrinsicsRISCV.h" 20 #include "llvm/Support/Alignment.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/KnownBits.h" 23 #include "llvm/Support/MathExtras.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 #define DEBUG_TYPE "riscv-isel" 29 30 namespace llvm { 31 namespace RISCV { 32 #define GET_RISCVVSSEGTable_IMPL 33 #define GET_RISCVVLSEGTable_IMPL 34 #define GET_RISCVVLXSEGTable_IMPL 35 #define GET_RISCVVSXSEGTable_IMPL 36 #define GET_RISCVVLETable_IMPL 37 #define GET_RISCVVSETable_IMPL 38 #define GET_RISCVVLXTable_IMPL 39 #define GET_RISCVVSXTable_IMPL 40 #define GET_RISCVMaskedPseudosTable_IMPL 41 #include "RISCVGenSearchableTables.inc" 42 } // namespace RISCV 43 } // namespace llvm 44 45 void RISCVDAGToDAGISel::PreprocessISelDAG() { 46 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 47 E = CurDAG->allnodes_end(); 48 I != E;) { 49 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 50 51 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point 52 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden. 53 if (N->getOpcode() == ISD::SPLAT_VECTOR) { 54 MVT VT = N->getSimpleValueType(0); 55 unsigned Opc = 56 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL; 57 SDLoc DL(N); 58 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()); 59 SDValue Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), 60 N->getOperand(0), VL); 61 62 --I; 63 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 64 ++I; 65 CurDAG->DeleteNode(N); 66 continue; 67 } 68 69 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector 70 // load. Done after lowering and combining so that we have a chance to 71 // optimize this to VMV_V_X_VL when the upper bits aren't needed. 72 if (N->getOpcode() != RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) 73 continue; 74 75 assert(N->getNumOperands() == 4 && "Unexpected number of operands"); 76 MVT VT = N->getSimpleValueType(0); 77 SDValue Passthru = N->getOperand(0); 78 SDValue Lo = N->getOperand(1); 79 SDValue Hi = N->getOperand(2); 80 SDValue VL = N->getOperand(3); 81 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() && 82 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 && 83 "Unexpected VTs!"); 84 MachineFunction &MF = CurDAG->getMachineFunction(); 85 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>(); 86 SDLoc DL(N); 87 88 // We use the same frame index we use for moving two i32s into 64-bit FPR. 89 // This is an analogous operation. 90 int FI = FuncInfo->getMoveF64FrameIndex(MF); 91 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); 92 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 93 SDValue StackSlot = 94 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout())); 95 96 SDValue Chain = CurDAG->getEntryNode(); 97 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8)); 98 99 SDValue OffsetSlot = 100 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL); 101 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4), 102 Align(8)); 103 104 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 105 106 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other}); 107 SDValue IntID = 108 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64); 109 SDValue Ops[] = {Chain, 110 IntID, 111 Passthru, 112 StackSlot, 113 CurDAG->getRegister(RISCV::X0, MVT::i64), 114 VL}; 115 116 SDValue Result = CurDAG->getMemIntrinsicNode( 117 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MVT::i64, MPI, Align(8), 118 MachineMemOperand::MOLoad); 119 120 // We're about to replace all uses of the SPLAT_VECTOR_SPLIT_I64 with the 121 // vlse we created. This will cause general havok on the dag because 122 // anything below the conversion could be folded into other existing nodes. 123 // To avoid invalidating 'I', back it up to the convert node. 124 --I; 125 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 126 127 // Now that we did that, the node is dead. Increment the iterator to the 128 // next node to process, then delete N. 129 ++I; 130 CurDAG->DeleteNode(N); 131 } 132 } 133 134 void RISCVDAGToDAGISel::PostprocessISelDAG() { 135 HandleSDNode Dummy(CurDAG->getRoot()); 136 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 137 138 bool MadeChange = false; 139 while (Position != CurDAG->allnodes_begin()) { 140 SDNode *N = &*--Position; 141 // Skip dead nodes and any non-machine opcodes. 142 if (N->use_empty() || !N->isMachineOpcode()) 143 continue; 144 145 MadeChange |= doPeepholeSExtW(N); 146 MadeChange |= doPeepholeLoadStoreADDI(N); 147 MadeChange |= doPeepholeMaskedRVV(N); 148 } 149 150 CurDAG->setRoot(Dummy.getValue()); 151 152 if (MadeChange) 153 CurDAG->RemoveDeadNodes(); 154 } 155 156 // Returns true if N is a MachineSDNode that has a reg and simm12 memory 157 // operand. The indices of the base pointer and offset are returned in BaseOpIdx 158 // and OffsetOpIdx. 159 static bool hasMemOffset(SDNode *N, unsigned &BaseOpIdx, 160 unsigned &OffsetOpIdx) { 161 switch (N->getMachineOpcode()) { 162 case RISCV::LB: 163 case RISCV::LH: 164 case RISCV::LW: 165 case RISCV::LBU: 166 case RISCV::LHU: 167 case RISCV::LWU: 168 case RISCV::LD: 169 case RISCV::FLH: 170 case RISCV::FLW: 171 case RISCV::FLD: 172 BaseOpIdx = 0; 173 OffsetOpIdx = 1; 174 return true; 175 case RISCV::SB: 176 case RISCV::SH: 177 case RISCV::SW: 178 case RISCV::SD: 179 case RISCV::FSH: 180 case RISCV::FSW: 181 case RISCV::FSD: 182 BaseOpIdx = 1; 183 OffsetOpIdx = 2; 184 return true; 185 } 186 187 return false; 188 } 189 190 static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL, 191 const MVT VT, int64_t Imm, 192 const RISCVSubtarget &Subtarget) { 193 assert(VT == MVT::i64 && "Expecting MVT::i64"); 194 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering(); 195 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool( 196 ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT)); 197 SDValue Addr = TLI->getAddr(CP, *CurDAG); 198 SDValue Offset = CurDAG->getTargetConstant(0, DL, VT); 199 // Since there is no data race, the chain can be the entry node. 200 SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset, 201 CurDAG->getEntryNode()); 202 MachineFunction &MF = CurDAG->getMachineFunction(); 203 MachineMemOperand *MemOp = MF.getMachineMemOperand( 204 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 205 LLT(VT), CP->getAlign()); 206 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp}); 207 return Load; 208 } 209 210 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 211 int64_t Imm, const RISCVSubtarget &Subtarget) { 212 MVT XLenVT = Subtarget.getXLenVT(); 213 RISCVMatInt::InstSeq Seq = 214 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits()); 215 216 // If Imm is expensive to build, then we put it into constant pool. 217 if (Subtarget.useConstantPoolForLargeInts() && 218 Seq.size() > Subtarget.getMaxBuildIntsCost()) 219 return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget); 220 221 SDNode *Result = nullptr; 222 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT); 223 for (RISCVMatInt::Inst &Inst : Seq) { 224 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT); 225 switch (Inst.getOpndKind()) { 226 case RISCVMatInt::Imm: 227 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SDImm); 228 break; 229 case RISCVMatInt::RegX0: 230 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, 231 CurDAG->getRegister(RISCV::X0, XLenVT)); 232 break; 233 case RISCVMatInt::RegReg: 234 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SrcReg); 235 break; 236 case RISCVMatInt::RegImm: 237 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm); 238 break; 239 } 240 241 // Only the first instruction has X0 as its source. 242 SrcReg = SDValue(Result, 0); 243 } 244 245 return Result; 246 } 247 248 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 249 unsigned RegClassID, unsigned SubReg0) { 250 assert(Regs.size() >= 2 && Regs.size() <= 8); 251 252 SDLoc DL(Regs[0]); 253 SmallVector<SDValue, 8> Ops; 254 255 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32)); 256 257 for (unsigned I = 0; I < Regs.size(); ++I) { 258 Ops.push_back(Regs[I]); 259 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32)); 260 } 261 SDNode *N = 262 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 263 return SDValue(N, 0); 264 } 265 266 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 267 unsigned NF) { 268 static const unsigned RegClassIDs[] = { 269 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID, 270 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID, 271 RISCV::VRN8M1RegClassID}; 272 273 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0); 274 } 275 276 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 277 unsigned NF) { 278 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID, 279 RISCV::VRN3M2RegClassID, 280 RISCV::VRN4M2RegClassID}; 281 282 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0); 283 } 284 285 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 286 unsigned NF) { 287 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID, 288 RISCV::sub_vrm4_0); 289 } 290 291 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 292 unsigned NF, RISCVII::VLMUL LMUL) { 293 switch (LMUL) { 294 default: 295 llvm_unreachable("Invalid LMUL."); 296 case RISCVII::VLMUL::LMUL_F8: 297 case RISCVII::VLMUL::LMUL_F4: 298 case RISCVII::VLMUL::LMUL_F2: 299 case RISCVII::VLMUL::LMUL_1: 300 return createM1Tuple(CurDAG, Regs, NF); 301 case RISCVII::VLMUL::LMUL_2: 302 return createM2Tuple(CurDAG, Regs, NF); 303 case RISCVII::VLMUL::LMUL_4: 304 return createM4Tuple(CurDAG, Regs, NF); 305 } 306 } 307 308 void RISCVDAGToDAGISel::addVectorLoadStoreOperands( 309 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp, 310 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands, 311 bool IsLoad, MVT *IndexVT) { 312 SDValue Chain = Node->getOperand(0); 313 SDValue Glue; 314 315 SDValue Base; 316 SelectBaseAddr(Node->getOperand(CurOp++), Base); 317 Operands.push_back(Base); // Base pointer. 318 319 if (IsStridedOrIndexed) { 320 Operands.push_back(Node->getOperand(CurOp++)); // Index. 321 if (IndexVT) 322 *IndexVT = Operands.back()->getSimpleValueType(0); 323 } 324 325 if (IsMasked) { 326 // Mask needs to be copied to V0. 327 SDValue Mask = Node->getOperand(CurOp++); 328 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); 329 Glue = Chain.getValue(1); 330 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); 331 } 332 SDValue VL; 333 selectVLOp(Node->getOperand(CurOp++), VL); 334 Operands.push_back(VL); 335 336 MVT XLenVT = Subtarget->getXLenVT(); 337 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 338 Operands.push_back(SEWOp); 339 340 // Masked load has the tail policy argument. 341 if (IsMasked && IsLoad) { 342 // Policy must be a constant. 343 uint64_t Policy = Node->getConstantOperandVal(CurOp++); 344 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT); 345 Operands.push_back(PolicyOp); 346 } 347 348 Operands.push_back(Chain); // Chain. 349 if (Glue) 350 Operands.push_back(Glue); 351 } 352 353 static bool isAllUndef(ArrayRef<SDValue> Values) { 354 return llvm::all_of(Values, [](SDValue V) { return V->isUndef(); }); 355 } 356 357 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked, 358 bool IsStrided) { 359 SDLoc DL(Node); 360 unsigned NF = Node->getNumValues() - 1; 361 MVT VT = Node->getSimpleValueType(0); 362 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 363 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 364 365 unsigned CurOp = 2; 366 SmallVector<SDValue, 8> Operands; 367 368 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 369 Node->op_begin() + CurOp + NF); 370 bool IsTU = IsMasked || !isAllUndef(Regs); 371 if (IsTU) { 372 SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL); 373 Operands.push_back(Merge); 374 } 375 CurOp += NF; 376 377 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 378 Operands, /*IsLoad=*/true); 379 380 const RISCV::VLSEGPseudo *P = 381 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 382 static_cast<unsigned>(LMUL)); 383 MachineSDNode *Load = 384 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 385 386 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 387 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 388 389 SDValue SuperReg = SDValue(Load, 0); 390 for (unsigned I = 0; I < NF; ++I) { 391 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 392 ReplaceUses(SDValue(Node, I), 393 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 394 } 395 396 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 397 CurDAG->RemoveDeadNode(Node); 398 } 399 400 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) { 401 SDLoc DL(Node); 402 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. 403 MVT VT = Node->getSimpleValueType(0); 404 MVT XLenVT = Subtarget->getXLenVT(); 405 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 406 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 407 408 unsigned CurOp = 2; 409 SmallVector<SDValue, 7> Operands; 410 411 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 412 Node->op_begin() + CurOp + NF); 413 bool IsTU = IsMasked || !isAllUndef(Regs); 414 if (IsTU) { 415 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 416 Operands.push_back(MaskedOff); 417 } 418 CurOp += NF; 419 420 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 421 /*IsStridedOrIndexed*/ false, Operands, 422 /*IsLoad=*/true); 423 424 const RISCV::VLSEGPseudo *P = 425 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 426 Log2SEW, static_cast<unsigned>(LMUL)); 427 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, 428 XLenVT, MVT::Other, Operands); 429 430 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 431 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 432 433 SDValue SuperReg = SDValue(Load, 0); 434 for (unsigned I = 0; I < NF; ++I) { 435 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 436 ReplaceUses(SDValue(Node, I), 437 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 438 } 439 440 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL 441 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain 442 CurDAG->RemoveDeadNode(Node); 443 } 444 445 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked, 446 bool IsOrdered) { 447 SDLoc DL(Node); 448 unsigned NF = Node->getNumValues() - 1; 449 MVT VT = Node->getSimpleValueType(0); 450 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 451 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 452 453 unsigned CurOp = 2; 454 SmallVector<SDValue, 8> Operands; 455 456 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 457 Node->op_begin() + CurOp + NF); 458 bool IsTU = IsMasked || !isAllUndef(Regs); 459 if (IsTU) { 460 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 461 Operands.push_back(MaskedOff); 462 } 463 CurOp += NF; 464 465 MVT IndexVT; 466 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 467 /*IsStridedOrIndexed*/ true, Operands, 468 /*IsLoad=*/true, &IndexVT); 469 470 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 471 "Element count mismatch"); 472 473 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 474 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 475 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 476 report_fatal_error("The V extension does not support EEW=64 for index " 477 "values when XLEN=32"); 478 } 479 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo( 480 NF, IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 481 static_cast<unsigned>(IndexLMUL)); 482 MachineSDNode *Load = 483 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 484 485 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 486 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 487 488 SDValue SuperReg = SDValue(Load, 0); 489 for (unsigned I = 0; I < NF; ++I) { 490 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 491 ReplaceUses(SDValue(Node, I), 492 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 493 } 494 495 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 496 CurDAG->RemoveDeadNode(Node); 497 } 498 499 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked, 500 bool IsStrided) { 501 SDLoc DL(Node); 502 unsigned NF = Node->getNumOperands() - 4; 503 if (IsStrided) 504 NF--; 505 if (IsMasked) 506 NF--; 507 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 508 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 509 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 510 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 511 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 512 513 SmallVector<SDValue, 8> Operands; 514 Operands.push_back(StoreVal); 515 unsigned CurOp = 2 + NF; 516 517 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 518 Operands); 519 520 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo( 521 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 522 MachineSDNode *Store = 523 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 524 525 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 526 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 527 528 ReplaceNode(Node, Store); 529 } 530 531 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked, 532 bool IsOrdered) { 533 SDLoc DL(Node); 534 unsigned NF = Node->getNumOperands() - 5; 535 if (IsMasked) 536 --NF; 537 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 538 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 539 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 540 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 541 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 542 543 SmallVector<SDValue, 8> Operands; 544 Operands.push_back(StoreVal); 545 unsigned CurOp = 2 + NF; 546 547 MVT IndexVT; 548 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 549 /*IsStridedOrIndexed*/ true, Operands, 550 /*IsLoad=*/false, &IndexVT); 551 552 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 553 "Element count mismatch"); 554 555 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 556 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 557 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 558 report_fatal_error("The V extension does not support EEW=64 for index " 559 "values when XLEN=32"); 560 } 561 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo( 562 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 563 static_cast<unsigned>(IndexLMUL)); 564 MachineSDNode *Store = 565 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 566 567 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 568 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 569 570 ReplaceNode(Node, Store); 571 } 572 573 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) { 574 if (!Subtarget->hasVInstructions()) 575 return; 576 577 assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN || 578 Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) && 579 "Unexpected opcode"); 580 581 SDLoc DL(Node); 582 MVT XLenVT = Subtarget->getXLenVT(); 583 584 bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN; 585 unsigned IntNoOffset = HasChain ? 1 : 0; 586 unsigned IntNo = Node->getConstantOperandVal(IntNoOffset); 587 588 assert((IntNo == Intrinsic::riscv_vsetvli || 589 IntNo == Intrinsic::riscv_vsetvlimax || 590 IntNo == Intrinsic::riscv_vsetvli_opt || 591 IntNo == Intrinsic::riscv_vsetvlimax_opt) && 592 "Unexpected vsetvli intrinsic"); 593 594 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax || 595 IntNo == Intrinsic::riscv_vsetvlimax_opt; 596 unsigned Offset = IntNoOffset + (VLMax ? 1 : 2); 597 598 assert(Node->getNumOperands() == Offset + 2 && 599 "Unexpected number of operands"); 600 601 unsigned SEW = 602 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7); 603 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>( 604 Node->getConstantOperandVal(Offset + 1) & 0x7); 605 606 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true, 607 /*MaskAgnostic*/ false); 608 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT); 609 610 SmallVector<EVT, 2> VTs = {XLenVT}; 611 if (HasChain) 612 VTs.push_back(MVT::Other); 613 614 SDValue VLOperand; 615 unsigned Opcode = RISCV::PseudoVSETVLI; 616 if (VLMax) { 617 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT); 618 Opcode = RISCV::PseudoVSETVLIX0; 619 } else { 620 VLOperand = Node->getOperand(IntNoOffset + 1); 621 622 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { 623 uint64_t AVL = C->getZExtValue(); 624 if (isUInt<5>(AVL)) { 625 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); 626 SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp}; 627 if (HasChain) 628 Ops.push_back(Node->getOperand(0)); 629 ReplaceNode( 630 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops)); 631 return; 632 } 633 } 634 } 635 636 SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp}; 637 if (HasChain) 638 Ops.push_back(Node->getOperand(0)); 639 640 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops)); 641 } 642 643 void RISCVDAGToDAGISel::Select(SDNode *Node) { 644 // If we have a custom node, we have already selected. 645 if (Node->isMachineOpcode()) { 646 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 647 Node->setNodeId(-1); 648 return; 649 } 650 651 // Instruction Selection not handled by the auto-generated tablegen selection 652 // should be handled here. 653 unsigned Opcode = Node->getOpcode(); 654 MVT XLenVT = Subtarget->getXLenVT(); 655 SDLoc DL(Node); 656 MVT VT = Node->getSimpleValueType(0); 657 658 switch (Opcode) { 659 case ISD::Constant: { 660 auto *ConstNode = cast<ConstantSDNode>(Node); 661 if (VT == XLenVT && ConstNode->isZero()) { 662 SDValue New = 663 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT); 664 ReplaceNode(Node, New.getNode()); 665 return; 666 } 667 int64_t Imm = ConstNode->getSExtValue(); 668 // If the upper XLen-16 bits are not used, try to convert this to a simm12 669 // by sign extending bit 15. 670 if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) && 671 hasAllHUsers(Node)) 672 Imm = SignExtend64<16>(Imm); 673 // If the upper 32-bits are not used try to convert this into a simm32 by 674 // sign extending bit 32. 675 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node)) 676 Imm = SignExtend64<32>(Imm); 677 678 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget)); 679 return; 680 } 681 case ISD::ADD: { 682 // Try to select ADD + immediate used as memory addresses to 683 // (ADDI (ADD X, Imm-Lo12), Lo12) if it will allow the ADDI to be removed by 684 // doPeepholeLoadStoreADDI. 685 686 // LHS should be an immediate. 687 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 688 if (!N1C) 689 break; 690 691 int64_t Offset = N1C->getSExtValue(); 692 int64_t Lo12 = SignExtend64<12>(Offset); 693 694 // Don't do this if the lower 12 bits are 0 or we could use ADDI directly. 695 if (Lo12 == 0 || isInt<12>(Offset)) 696 break; 697 698 // Don't do this if we can use a pair of ADDIs. 699 if (isInt<12>(Offset / 2) && isInt<12>(Offset - Offset / 2)) 700 break; 701 702 bool AllPointerUses = true; 703 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 704 SDNode *User = *UI; 705 706 // Is this user a memory instruction that uses a register and immediate 707 // that has this ADD as its pointer. 708 unsigned BaseOpIdx, OffsetOpIdx; 709 if (!User->isMachineOpcode() || 710 !hasMemOffset(User, BaseOpIdx, OffsetOpIdx) || 711 UI.getOperandNo() != BaseOpIdx) { 712 AllPointerUses = false; 713 break; 714 } 715 716 // If the memory instruction already has an offset, make sure the combined 717 // offset is foldable. 718 int64_t MemOffs = 719 cast<ConstantSDNode>(User->getOperand(OffsetOpIdx))->getSExtValue(); 720 MemOffs += Lo12; 721 if (!isInt<12>(MemOffs)) { 722 AllPointerUses = false; 723 break; 724 } 725 } 726 727 if (!AllPointerUses) 728 break; 729 730 Offset -= Lo12; 731 // Restore sign bits for RV32. 732 if (!Subtarget->is64Bit()) 733 Offset = SignExtend64<32>(Offset); 734 735 // Emit (ADDI (ADD X, Hi), Lo) 736 SDNode *Imm = selectImm(CurDAG, DL, VT, Offset, *Subtarget); 737 SDNode *ADD = CurDAG->getMachineNode(RISCV::ADD, DL, VT, 738 Node->getOperand(0), SDValue(Imm, 0)); 739 SDNode *ADDI = 740 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, SDValue(ADD, 0), 741 CurDAG->getTargetConstant(Lo12, DL, VT)); 742 ReplaceNode(Node, ADDI); 743 return; 744 } 745 case ISD::SRL: { 746 // Optimize (srl (and X, C2), C) -> 747 // (srli (slli X, (XLen-C3), (XLen-C3) + C) 748 // Where C2 is a mask with C3 trailing ones. 749 // Taking into account that the C2 may have had lower bits unset by 750 // SimplifyDemandedBits. This avoids materializing the C2 immediate. 751 // This pattern occurs when type legalizing right shifts for types with 752 // less than XLen bits. 753 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 754 if (!N1C) 755 break; 756 SDValue N0 = Node->getOperand(0); 757 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 758 !isa<ConstantSDNode>(N0.getOperand(1))) 759 break; 760 unsigned ShAmt = N1C->getZExtValue(); 761 uint64_t Mask = N0.getConstantOperandVal(1); 762 Mask |= maskTrailingOnes<uint64_t>(ShAmt); 763 if (!isMask_64(Mask)) 764 break; 765 unsigned TrailingOnes = countTrailingOnes(Mask); 766 // 32 trailing ones should use srliw via tablegen pattern. 767 if (TrailingOnes == 32 || ShAmt >= TrailingOnes) 768 break; 769 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes; 770 SDNode *SLLI = 771 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 772 CurDAG->getTargetConstant(LShAmt, DL, VT)); 773 SDNode *SRLI = CurDAG->getMachineNode( 774 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 775 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 776 ReplaceNode(Node, SRLI); 777 return; 778 } 779 case ISD::SRA: { 780 // Optimize (sra (sext_inreg X, i16), C) -> 781 // (srai (slli X, (XLen-16), (XLen-16) + C) 782 // And (sra (sext_inreg X, i8), C) -> 783 // (srai (slli X, (XLen-8), (XLen-8) + C) 784 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal. 785 // This transform matches the code we get without Zbb. The shifts are more 786 // compressible, and this can help expose CSE opportunities in the sdiv by 787 // constant optimization. 788 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 789 if (!N1C) 790 break; 791 SDValue N0 = Node->getOperand(0); 792 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse()) 793 break; 794 unsigned ShAmt = N1C->getZExtValue(); 795 unsigned ExtSize = 796 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits(); 797 // ExtSize of 32 should use sraiw via tablegen pattern. 798 if (ExtSize >= 32 || ShAmt >= ExtSize) 799 break; 800 unsigned LShAmt = Subtarget->getXLen() - ExtSize; 801 SDNode *SLLI = 802 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 803 CurDAG->getTargetConstant(LShAmt, DL, VT)); 804 SDNode *SRAI = CurDAG->getMachineNode( 805 RISCV::SRAI, DL, VT, SDValue(SLLI, 0), 806 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 807 ReplaceNode(Node, SRAI); 808 return; 809 } 810 case ISD::AND: { 811 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 812 if (!N1C) 813 break; 814 815 SDValue N0 = Node->getOperand(0); 816 817 bool LeftShift = N0.getOpcode() == ISD::SHL; 818 if (!LeftShift && N0.getOpcode() != ISD::SRL) 819 break; 820 821 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 822 if (!C) 823 break; 824 uint64_t C2 = C->getZExtValue(); 825 unsigned XLen = Subtarget->getXLen(); 826 if (!C2 || C2 >= XLen) 827 break; 828 829 uint64_t C1 = N1C->getZExtValue(); 830 831 // Keep track of whether this is a c.andi. If we can't use c.andi, the 832 // shift pair might offer more compression opportunities. 833 // TODO: We could check for C extension here, but we don't have many lit 834 // tests with the C extension enabled so not checking gets better coverage. 835 // TODO: What if ANDI faster than shift? 836 bool IsCANDI = isInt<6>(N1C->getSExtValue()); 837 838 // Clear irrelevant bits in the mask. 839 if (LeftShift) 840 C1 &= maskTrailingZeros<uint64_t>(C2); 841 else 842 C1 &= maskTrailingOnes<uint64_t>(XLen - C2); 843 844 // Some transforms should only be done if the shift has a single use or 845 // the AND would become (srli (slli X, 32), 32) 846 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF); 847 848 SDValue X = N0.getOperand(0); 849 850 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask 851 // with c3 leading zeros. 852 if (!LeftShift && isMask_64(C1)) { 853 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 854 if (C2 < C3) { 855 // If the number of leading zeros is C2+32 this can be SRLIW. 856 if (C2 + 32 == C3) { 857 SDNode *SRLIW = 858 CurDAG->getMachineNode(RISCV::SRLIW, DL, XLenVT, X, 859 CurDAG->getTargetConstant(C2, DL, XLenVT)); 860 ReplaceNode(Node, SRLIW); 861 return; 862 } 863 864 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if 865 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1. 866 // 867 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type 868 // legalized and goes through DAG combine. 869 if (C2 >= 32 && (C3 - C2) == 1 && N0.hasOneUse() && 870 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 871 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) { 872 SDNode *SRAIW = 873 CurDAG->getMachineNode(RISCV::SRAIW, DL, XLenVT, X.getOperand(0), 874 CurDAG->getTargetConstant(31, DL, XLenVT)); 875 SDNode *SRLIW = CurDAG->getMachineNode( 876 RISCV::SRLIW, DL, XLenVT, SDValue(SRAIW, 0), 877 CurDAG->getTargetConstant(C3 - 32, DL, XLenVT)); 878 ReplaceNode(Node, SRLIW); 879 return; 880 } 881 882 // (srli (slli x, c3-c2), c3). 883 // Skip if we could use (zext.w (sraiw X, C2)). 884 bool Skip = Subtarget->hasStdExtZba() && C3 == 32 && 885 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 886 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32; 887 // Also Skip if we can use bexti. 888 Skip |= Subtarget->hasStdExtZbs() && C3 == XLen - 1; 889 if (OneUseOrZExtW && !Skip) { 890 SDNode *SLLI = CurDAG->getMachineNode( 891 RISCV::SLLI, DL, XLenVT, X, 892 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 893 SDNode *SRLI = 894 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 895 CurDAG->getTargetConstant(C3, DL, XLenVT)); 896 ReplaceNode(Node, SRLI); 897 return; 898 } 899 } 900 } 901 902 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask 903 // shifted by c2 bits with c3 leading zeros. 904 if (LeftShift && isShiftedMask_64(C1)) { 905 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 906 907 if (C2 + C3 < XLen && 908 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + C3)) << C2)) { 909 // Use slli.uw when possible. 910 if ((XLen - (C2 + C3)) == 32 && Subtarget->hasStdExtZba()) { 911 SDNode *SLLI_UW = 912 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, XLenVT, X, 913 CurDAG->getTargetConstant(C2, DL, XLenVT)); 914 ReplaceNode(Node, SLLI_UW); 915 return; 916 } 917 918 // (srli (slli c2+c3), c3) 919 if (OneUseOrZExtW && !IsCANDI) { 920 SDNode *SLLI = CurDAG->getMachineNode( 921 RISCV::SLLI, DL, XLenVT, X, 922 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 923 SDNode *SRLI = 924 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 925 CurDAG->getTargetConstant(C3, DL, XLenVT)); 926 ReplaceNode(Node, SRLI); 927 return; 928 } 929 } 930 } 931 932 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a 933 // shifted mask with c2 leading zeros and c3 trailing zeros. 934 if (!LeftShift && isShiftedMask_64(C1)) { 935 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 936 uint64_t C3 = countTrailingZeros(C1); 937 if (Leading == C2 && C2 + C3 < XLen && OneUseOrZExtW && !IsCANDI) { 938 unsigned SrliOpc = RISCV::SRLI; 939 // If the input is zexti32 we should use SRLIW. 940 if (X.getOpcode() == ISD::AND && isa<ConstantSDNode>(X.getOperand(1)) && 941 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) { 942 SrliOpc = RISCV::SRLIW; 943 X = X.getOperand(0); 944 } 945 SDNode *SRLI = CurDAG->getMachineNode( 946 SrliOpc, DL, XLenVT, X, 947 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 948 SDNode *SLLI = 949 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 950 CurDAG->getTargetConstant(C3, DL, XLenVT)); 951 ReplaceNode(Node, SLLI); 952 return; 953 } 954 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI. 955 if (Leading > 32 && (Leading - 32) == C2 && C2 + C3 < 32 && 956 OneUseOrZExtW && !IsCANDI) { 957 SDNode *SRLIW = CurDAG->getMachineNode( 958 RISCV::SRLIW, DL, XLenVT, X, 959 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 960 SDNode *SLLI = 961 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 962 CurDAG->getTargetConstant(C3, DL, XLenVT)); 963 ReplaceNode(Node, SLLI); 964 return; 965 } 966 } 967 968 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a 969 // shifted mask with no leading zeros and c3 trailing zeros. 970 if (LeftShift && isShiftedMask_64(C1)) { 971 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 972 uint64_t C3 = countTrailingZeros(C1); 973 if (Leading == 0 && C2 < C3 && OneUseOrZExtW && !IsCANDI) { 974 SDNode *SRLI = CurDAG->getMachineNode( 975 RISCV::SRLI, DL, XLenVT, X, 976 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 977 SDNode *SLLI = 978 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 979 CurDAG->getTargetConstant(C3, DL, XLenVT)); 980 ReplaceNode(Node, SLLI); 981 return; 982 } 983 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI. 984 if (C2 < C3 && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) { 985 SDNode *SRLIW = CurDAG->getMachineNode( 986 RISCV::SRLIW, DL, XLenVT, X, 987 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 988 SDNode *SLLI = 989 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 990 CurDAG->getTargetConstant(C3, DL, XLenVT)); 991 ReplaceNode(Node, SLLI); 992 return; 993 } 994 } 995 996 break; 997 } 998 case ISD::MUL: { 999 // Special case for calculating (mul (and X, C2), C1) where the full product 1000 // fits in XLen bits. We can shift X left by the number of leading zeros in 1001 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final 1002 // product has XLen trailing zeros, putting it in the output of MULHU. This 1003 // can avoid materializing a constant in a register for C2. 1004 1005 // RHS should be a constant. 1006 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1007 if (!N1C || !N1C->hasOneUse()) 1008 break; 1009 1010 // LHS should be an AND with constant. 1011 SDValue N0 = Node->getOperand(0); 1012 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) 1013 break; 1014 1015 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 1016 1017 // Constant should be a mask. 1018 if (!isMask_64(C2)) 1019 break; 1020 1021 // This should be the only use of the AND unless we will use 1022 // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND 1023 // constants. 1024 if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) 1025 break; 1026 1027 // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this 1028 // optimization. 1029 if (isInt<12>(C2) || 1030 (C2 == UINT64_C(0xFFFF) && 1031 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || 1032 (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) 1033 break; 1034 1035 // We need to shift left the AND input and C1 by a total of XLen bits. 1036 1037 // How far left do we need to shift the AND input? 1038 unsigned XLen = Subtarget->getXLen(); 1039 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); 1040 1041 // The constant gets shifted by the remaining amount unless that would 1042 // shift bits out. 1043 uint64_t C1 = N1C->getZExtValue(); 1044 unsigned ConstantShift = XLen - LeadingZeros; 1045 if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) 1046 break; 1047 1048 uint64_t ShiftedC1 = C1 << ConstantShift; 1049 // If this RV32, we need to sign extend the constant. 1050 if (XLen == 32) 1051 ShiftedC1 = SignExtend64<32>(ShiftedC1); 1052 1053 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). 1054 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget); 1055 SDNode *SLLI = 1056 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), 1057 CurDAG->getTargetConstant(LeadingZeros, DL, VT)); 1058 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, 1059 SDValue(SLLI, 0), SDValue(Imm, 0)); 1060 ReplaceNode(Node, MULHU); 1061 return; 1062 } 1063 case ISD::INTRINSIC_WO_CHAIN: { 1064 unsigned IntNo = Node->getConstantOperandVal(0); 1065 switch (IntNo) { 1066 // By default we do not custom select any intrinsic. 1067 default: 1068 break; 1069 case Intrinsic::riscv_vmsgeu: 1070 case Intrinsic::riscv_vmsge: { 1071 SDValue Src1 = Node->getOperand(1); 1072 SDValue Src2 = Node->getOperand(2); 1073 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu; 1074 bool IsCmpUnsignedZero = false; 1075 // Only custom select scalar second operand. 1076 if (Src2.getValueType() != XLenVT) 1077 break; 1078 // Small constants are handled with patterns. 1079 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1080 int64_t CVal = C->getSExtValue(); 1081 if (CVal >= -15 && CVal <= 16) { 1082 if (!IsUnsigned || CVal != 0) 1083 break; 1084 IsCmpUnsignedZero = true; 1085 } 1086 } 1087 MVT Src1VT = Src1.getSimpleValueType(); 1088 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode; 1089 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1090 default: 1091 llvm_unreachable("Unexpected LMUL!"); 1092 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 1093 case RISCVII::VLMUL::lmulenum: \ 1094 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1095 : RISCV::PseudoVMSLT_VX_##suffix; \ 1096 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \ 1097 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 1098 break; 1099 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1) 1100 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2) 1101 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4) 1102 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8) 1103 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16) 1104 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32) 1105 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64) 1106 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES 1107 } 1108 SDValue SEW = CurDAG->getTargetConstant( 1109 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1110 SDValue VL; 1111 selectVLOp(Node->getOperand(3), VL); 1112 1113 // If vmsgeu with 0 immediate, expand it to vmset. 1114 if (IsCmpUnsignedZero) { 1115 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW)); 1116 return; 1117 } 1118 1119 // Expand to 1120 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd 1121 SDValue Cmp = SDValue( 1122 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1123 0); 1124 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT, 1125 {Cmp, Cmp, VL, SEW})); 1126 return; 1127 } 1128 case Intrinsic::riscv_vmsgeu_mask: 1129 case Intrinsic::riscv_vmsge_mask: { 1130 SDValue Src1 = Node->getOperand(2); 1131 SDValue Src2 = Node->getOperand(3); 1132 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask; 1133 bool IsCmpUnsignedZero = false; 1134 // Only custom select scalar second operand. 1135 if (Src2.getValueType() != XLenVT) 1136 break; 1137 // Small constants are handled with patterns. 1138 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1139 int64_t CVal = C->getSExtValue(); 1140 if (CVal >= -15 && CVal <= 16) { 1141 if (!IsUnsigned || CVal != 0) 1142 break; 1143 IsCmpUnsignedZero = true; 1144 } 1145 } 1146 MVT Src1VT = Src1.getSimpleValueType(); 1147 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode, 1148 VMOROpcode; 1149 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1150 default: 1151 llvm_unreachable("Unexpected LMUL!"); 1152 #define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \ 1153 case RISCVII::VLMUL::lmulenum: \ 1154 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1155 : RISCV::PseudoVMSLT_VX_##suffix; \ 1156 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \ 1157 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \ 1158 break; 1159 CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1) 1160 CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2) 1161 CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4) 1162 CASE_VMSLT_OPCODES(LMUL_1, M1, B8) 1163 CASE_VMSLT_OPCODES(LMUL_2, M2, B16) 1164 CASE_VMSLT_OPCODES(LMUL_4, M4, B32) 1165 CASE_VMSLT_OPCODES(LMUL_8, M8, B64) 1166 #undef CASE_VMSLT_OPCODES 1167 } 1168 // Mask operations use the LMUL from the mask type. 1169 switch (RISCVTargetLowering::getLMUL(VT)) { 1170 default: 1171 llvm_unreachable("Unexpected LMUL!"); 1172 #define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \ 1173 case RISCVII::VLMUL::lmulenum: \ 1174 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \ 1175 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \ 1176 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \ 1177 break; 1178 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8) 1179 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4) 1180 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2) 1181 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, M1) 1182 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, M2) 1183 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, M4) 1184 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, M8) 1185 #undef CASE_VMXOR_VMANDN_VMOR_OPCODES 1186 } 1187 SDValue SEW = CurDAG->getTargetConstant( 1188 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1189 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT); 1190 SDValue VL; 1191 selectVLOp(Node->getOperand(5), VL); 1192 SDValue MaskedOff = Node->getOperand(1); 1193 SDValue Mask = Node->getOperand(4); 1194 1195 // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff. 1196 if (IsCmpUnsignedZero) { 1197 // We don't need vmor if the MaskedOff and the Mask are the same 1198 // value. 1199 if (Mask == MaskedOff) { 1200 ReplaceUses(Node, Mask.getNode()); 1201 return; 1202 } 1203 ReplaceNode(Node, 1204 CurDAG->getMachineNode(VMOROpcode, DL, VT, 1205 {Mask, MaskedOff, VL, MaskSEW})); 1206 return; 1207 } 1208 1209 // If the MaskedOff value and the Mask are the same value use 1210 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt 1211 // This avoids needing to copy v0 to vd before starting the next sequence. 1212 if (Mask == MaskedOff) { 1213 SDValue Cmp = SDValue( 1214 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1215 0); 1216 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT, 1217 {Mask, Cmp, VL, MaskSEW})); 1218 return; 1219 } 1220 1221 // Mask needs to be copied to V0. 1222 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, 1223 RISCV::V0, Mask, SDValue()); 1224 SDValue Glue = Chain.getValue(1); 1225 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); 1226 1227 // Otherwise use 1228 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0 1229 // The result is mask undisturbed. 1230 // We use the same instructions to emulate mask agnostic behavior, because 1231 // the agnostic result can be either undisturbed or all 1. 1232 SDValue Cmp = SDValue( 1233 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, 1234 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), 1235 0); 1236 // vmxor.mm vd, vd, v0 is used to update active value. 1237 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, 1238 {Cmp, Mask, VL, MaskSEW})); 1239 return; 1240 } 1241 case Intrinsic::riscv_vsetvli_opt: 1242 case Intrinsic::riscv_vsetvlimax_opt: 1243 return selectVSETVLI(Node); 1244 } 1245 break; 1246 } 1247 case ISD::INTRINSIC_W_CHAIN: { 1248 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1249 switch (IntNo) { 1250 // By default we do not custom select any intrinsic. 1251 default: 1252 break; 1253 case Intrinsic::riscv_vsetvli: 1254 case Intrinsic::riscv_vsetvlimax: 1255 return selectVSETVLI(Node); 1256 case Intrinsic::riscv_vlseg2: 1257 case Intrinsic::riscv_vlseg3: 1258 case Intrinsic::riscv_vlseg4: 1259 case Intrinsic::riscv_vlseg5: 1260 case Intrinsic::riscv_vlseg6: 1261 case Intrinsic::riscv_vlseg7: 1262 case Intrinsic::riscv_vlseg8: { 1263 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1264 return; 1265 } 1266 case Intrinsic::riscv_vlseg2_mask: 1267 case Intrinsic::riscv_vlseg3_mask: 1268 case Intrinsic::riscv_vlseg4_mask: 1269 case Intrinsic::riscv_vlseg5_mask: 1270 case Intrinsic::riscv_vlseg6_mask: 1271 case Intrinsic::riscv_vlseg7_mask: 1272 case Intrinsic::riscv_vlseg8_mask: { 1273 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1274 return; 1275 } 1276 case Intrinsic::riscv_vlsseg2: 1277 case Intrinsic::riscv_vlsseg3: 1278 case Intrinsic::riscv_vlsseg4: 1279 case Intrinsic::riscv_vlsseg5: 1280 case Intrinsic::riscv_vlsseg6: 1281 case Intrinsic::riscv_vlsseg7: 1282 case Intrinsic::riscv_vlsseg8: { 1283 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1284 return; 1285 } 1286 case Intrinsic::riscv_vlsseg2_mask: 1287 case Intrinsic::riscv_vlsseg3_mask: 1288 case Intrinsic::riscv_vlsseg4_mask: 1289 case Intrinsic::riscv_vlsseg5_mask: 1290 case Intrinsic::riscv_vlsseg6_mask: 1291 case Intrinsic::riscv_vlsseg7_mask: 1292 case Intrinsic::riscv_vlsseg8_mask: { 1293 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1294 return; 1295 } 1296 case Intrinsic::riscv_vloxseg2: 1297 case Intrinsic::riscv_vloxseg3: 1298 case Intrinsic::riscv_vloxseg4: 1299 case Intrinsic::riscv_vloxseg5: 1300 case Intrinsic::riscv_vloxseg6: 1301 case Intrinsic::riscv_vloxseg7: 1302 case Intrinsic::riscv_vloxseg8: 1303 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1304 return; 1305 case Intrinsic::riscv_vluxseg2: 1306 case Intrinsic::riscv_vluxseg3: 1307 case Intrinsic::riscv_vluxseg4: 1308 case Intrinsic::riscv_vluxseg5: 1309 case Intrinsic::riscv_vluxseg6: 1310 case Intrinsic::riscv_vluxseg7: 1311 case Intrinsic::riscv_vluxseg8: 1312 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1313 return; 1314 case Intrinsic::riscv_vloxseg2_mask: 1315 case Intrinsic::riscv_vloxseg3_mask: 1316 case Intrinsic::riscv_vloxseg4_mask: 1317 case Intrinsic::riscv_vloxseg5_mask: 1318 case Intrinsic::riscv_vloxseg6_mask: 1319 case Intrinsic::riscv_vloxseg7_mask: 1320 case Intrinsic::riscv_vloxseg8_mask: 1321 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1322 return; 1323 case Intrinsic::riscv_vluxseg2_mask: 1324 case Intrinsic::riscv_vluxseg3_mask: 1325 case Intrinsic::riscv_vluxseg4_mask: 1326 case Intrinsic::riscv_vluxseg5_mask: 1327 case Intrinsic::riscv_vluxseg6_mask: 1328 case Intrinsic::riscv_vluxseg7_mask: 1329 case Intrinsic::riscv_vluxseg8_mask: 1330 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1331 return; 1332 case Intrinsic::riscv_vlseg8ff: 1333 case Intrinsic::riscv_vlseg7ff: 1334 case Intrinsic::riscv_vlseg6ff: 1335 case Intrinsic::riscv_vlseg5ff: 1336 case Intrinsic::riscv_vlseg4ff: 1337 case Intrinsic::riscv_vlseg3ff: 1338 case Intrinsic::riscv_vlseg2ff: { 1339 selectVLSEGFF(Node, /*IsMasked*/ false); 1340 return; 1341 } 1342 case Intrinsic::riscv_vlseg8ff_mask: 1343 case Intrinsic::riscv_vlseg7ff_mask: 1344 case Intrinsic::riscv_vlseg6ff_mask: 1345 case Intrinsic::riscv_vlseg5ff_mask: 1346 case Intrinsic::riscv_vlseg4ff_mask: 1347 case Intrinsic::riscv_vlseg3ff_mask: 1348 case Intrinsic::riscv_vlseg2ff_mask: { 1349 selectVLSEGFF(Node, /*IsMasked*/ true); 1350 return; 1351 } 1352 case Intrinsic::riscv_vloxei: 1353 case Intrinsic::riscv_vloxei_mask: 1354 case Intrinsic::riscv_vluxei: 1355 case Intrinsic::riscv_vluxei_mask: { 1356 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask || 1357 IntNo == Intrinsic::riscv_vluxei_mask; 1358 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei || 1359 IntNo == Intrinsic::riscv_vloxei_mask; 1360 1361 MVT VT = Node->getSimpleValueType(0); 1362 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1363 1364 unsigned CurOp = 2; 1365 // Masked intrinsic only have TU version pseduo instructions. 1366 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1367 SmallVector<SDValue, 8> Operands; 1368 if (IsTU) 1369 Operands.push_back(Node->getOperand(CurOp++)); 1370 else 1371 // Skip the undef passthru operand for nomask TA version pseudo 1372 CurOp++; 1373 1374 MVT IndexVT; 1375 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1376 /*IsStridedOrIndexed*/ true, Operands, 1377 /*IsLoad=*/true, &IndexVT); 1378 1379 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1380 "Element count mismatch"); 1381 1382 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1383 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1384 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1385 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1386 report_fatal_error("The V extension does not support EEW=64 for index " 1387 "values when XLEN=32"); 1388 } 1389 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( 1390 IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 1391 static_cast<unsigned>(IndexLMUL)); 1392 MachineSDNode *Load = 1393 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1394 1395 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1396 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1397 1398 ReplaceNode(Node, Load); 1399 return; 1400 } 1401 case Intrinsic::riscv_vlm: 1402 case Intrinsic::riscv_vle: 1403 case Intrinsic::riscv_vle_mask: 1404 case Intrinsic::riscv_vlse: 1405 case Intrinsic::riscv_vlse_mask: { 1406 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask || 1407 IntNo == Intrinsic::riscv_vlse_mask; 1408 bool IsStrided = 1409 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask; 1410 1411 MVT VT = Node->getSimpleValueType(0); 1412 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1413 1414 unsigned CurOp = 2; 1415 // The riscv_vlm intrinsic are always tail agnostic and no passthru operand. 1416 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm; 1417 // Masked intrinsic only have TU version pseduo instructions. 1418 bool IsTU = 1419 HasPassthruOperand && 1420 ((!IsMasked && !Node->getOperand(CurOp).isUndef()) || IsMasked); 1421 SmallVector<SDValue, 8> Operands; 1422 if (IsTU) 1423 Operands.push_back(Node->getOperand(CurOp++)); 1424 else if (HasPassthruOperand) 1425 // Skip the undef passthru operand for nomask TA version pseudo 1426 CurOp++; 1427 1428 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1429 Operands, /*IsLoad=*/true); 1430 1431 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1432 const RISCV::VLEPseudo *P = 1433 RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 1434 static_cast<unsigned>(LMUL)); 1435 MachineSDNode *Load = 1436 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1437 1438 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1439 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1440 1441 ReplaceNode(Node, Load); 1442 return; 1443 } 1444 case Intrinsic::riscv_vleff: 1445 case Intrinsic::riscv_vleff_mask: { 1446 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask; 1447 1448 MVT VT = Node->getSimpleValueType(0); 1449 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1450 1451 unsigned CurOp = 2; 1452 // Masked intrinsic only have TU version pseduo instructions. 1453 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1454 SmallVector<SDValue, 7> Operands; 1455 if (IsTU) 1456 Operands.push_back(Node->getOperand(CurOp++)); 1457 else 1458 // Skip the undef passthru operand for nomask TA version pseudo 1459 CurOp++; 1460 1461 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1462 /*IsStridedOrIndexed*/ false, Operands, 1463 /*IsLoad=*/true); 1464 1465 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1466 const RISCV::VLEPseudo *P = 1467 RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 1468 Log2SEW, static_cast<unsigned>(LMUL)); 1469 MachineSDNode *Load = CurDAG->getMachineNode( 1470 P->Pseudo, DL, Node->getValueType(0), XLenVT, MVT::Other, Operands); 1471 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1472 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1473 1474 ReplaceNode(Node, Load); 1475 return; 1476 } 1477 } 1478 break; 1479 } 1480 case ISD::INTRINSIC_VOID: { 1481 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1482 switch (IntNo) { 1483 case Intrinsic::riscv_vsseg2: 1484 case Intrinsic::riscv_vsseg3: 1485 case Intrinsic::riscv_vsseg4: 1486 case Intrinsic::riscv_vsseg5: 1487 case Intrinsic::riscv_vsseg6: 1488 case Intrinsic::riscv_vsseg7: 1489 case Intrinsic::riscv_vsseg8: { 1490 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1491 return; 1492 } 1493 case Intrinsic::riscv_vsseg2_mask: 1494 case Intrinsic::riscv_vsseg3_mask: 1495 case Intrinsic::riscv_vsseg4_mask: 1496 case Intrinsic::riscv_vsseg5_mask: 1497 case Intrinsic::riscv_vsseg6_mask: 1498 case Intrinsic::riscv_vsseg7_mask: 1499 case Intrinsic::riscv_vsseg8_mask: { 1500 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1501 return; 1502 } 1503 case Intrinsic::riscv_vssseg2: 1504 case Intrinsic::riscv_vssseg3: 1505 case Intrinsic::riscv_vssseg4: 1506 case Intrinsic::riscv_vssseg5: 1507 case Intrinsic::riscv_vssseg6: 1508 case Intrinsic::riscv_vssseg7: 1509 case Intrinsic::riscv_vssseg8: { 1510 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1511 return; 1512 } 1513 case Intrinsic::riscv_vssseg2_mask: 1514 case Intrinsic::riscv_vssseg3_mask: 1515 case Intrinsic::riscv_vssseg4_mask: 1516 case Intrinsic::riscv_vssseg5_mask: 1517 case Intrinsic::riscv_vssseg6_mask: 1518 case Intrinsic::riscv_vssseg7_mask: 1519 case Intrinsic::riscv_vssseg8_mask: { 1520 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1521 return; 1522 } 1523 case Intrinsic::riscv_vsoxseg2: 1524 case Intrinsic::riscv_vsoxseg3: 1525 case Intrinsic::riscv_vsoxseg4: 1526 case Intrinsic::riscv_vsoxseg5: 1527 case Intrinsic::riscv_vsoxseg6: 1528 case Intrinsic::riscv_vsoxseg7: 1529 case Intrinsic::riscv_vsoxseg8: 1530 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1531 return; 1532 case Intrinsic::riscv_vsuxseg2: 1533 case Intrinsic::riscv_vsuxseg3: 1534 case Intrinsic::riscv_vsuxseg4: 1535 case Intrinsic::riscv_vsuxseg5: 1536 case Intrinsic::riscv_vsuxseg6: 1537 case Intrinsic::riscv_vsuxseg7: 1538 case Intrinsic::riscv_vsuxseg8: 1539 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1540 return; 1541 case Intrinsic::riscv_vsoxseg2_mask: 1542 case Intrinsic::riscv_vsoxseg3_mask: 1543 case Intrinsic::riscv_vsoxseg4_mask: 1544 case Intrinsic::riscv_vsoxseg5_mask: 1545 case Intrinsic::riscv_vsoxseg6_mask: 1546 case Intrinsic::riscv_vsoxseg7_mask: 1547 case Intrinsic::riscv_vsoxseg8_mask: 1548 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1549 return; 1550 case Intrinsic::riscv_vsuxseg2_mask: 1551 case Intrinsic::riscv_vsuxseg3_mask: 1552 case Intrinsic::riscv_vsuxseg4_mask: 1553 case Intrinsic::riscv_vsuxseg5_mask: 1554 case Intrinsic::riscv_vsuxseg6_mask: 1555 case Intrinsic::riscv_vsuxseg7_mask: 1556 case Intrinsic::riscv_vsuxseg8_mask: 1557 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1558 return; 1559 case Intrinsic::riscv_vsoxei: 1560 case Intrinsic::riscv_vsoxei_mask: 1561 case Intrinsic::riscv_vsuxei: 1562 case Intrinsic::riscv_vsuxei_mask: { 1563 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask || 1564 IntNo == Intrinsic::riscv_vsuxei_mask; 1565 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei || 1566 IntNo == Intrinsic::riscv_vsoxei_mask; 1567 1568 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1569 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1570 1571 unsigned CurOp = 2; 1572 SmallVector<SDValue, 8> Operands; 1573 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1574 1575 MVT IndexVT; 1576 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1577 /*IsStridedOrIndexed*/ true, Operands, 1578 /*IsLoad=*/false, &IndexVT); 1579 1580 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1581 "Element count mismatch"); 1582 1583 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1584 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1585 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1586 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1587 report_fatal_error("The V extension does not support EEW=64 for index " 1588 "values when XLEN=32"); 1589 } 1590 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( 1591 IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW, 1592 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL)); 1593 MachineSDNode *Store = 1594 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1595 1596 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1597 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1598 1599 ReplaceNode(Node, Store); 1600 return; 1601 } 1602 case Intrinsic::riscv_vsm: 1603 case Intrinsic::riscv_vse: 1604 case Intrinsic::riscv_vse_mask: 1605 case Intrinsic::riscv_vsse: 1606 case Intrinsic::riscv_vsse_mask: { 1607 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask || 1608 IntNo == Intrinsic::riscv_vsse_mask; 1609 bool IsStrided = 1610 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask; 1611 1612 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1613 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1614 1615 unsigned CurOp = 2; 1616 SmallVector<SDValue, 8> Operands; 1617 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1618 1619 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1620 Operands); 1621 1622 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1623 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( 1624 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 1625 MachineSDNode *Store = 1626 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1627 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1628 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1629 1630 ReplaceNode(Node, Store); 1631 return; 1632 } 1633 } 1634 break; 1635 } 1636 case ISD::BITCAST: { 1637 MVT SrcVT = Node->getOperand(0).getSimpleValueType(); 1638 // Just drop bitcasts between vectors if both are fixed or both are 1639 // scalable. 1640 if ((VT.isScalableVector() && SrcVT.isScalableVector()) || 1641 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) { 1642 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 1643 CurDAG->RemoveDeadNode(Node); 1644 return; 1645 } 1646 break; 1647 } 1648 case ISD::INSERT_SUBVECTOR: { 1649 SDValue V = Node->getOperand(0); 1650 SDValue SubV = Node->getOperand(1); 1651 SDLoc DL(SubV); 1652 auto Idx = Node->getConstantOperandVal(2); 1653 MVT SubVecVT = SubV.getSimpleValueType(); 1654 1655 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1656 MVT SubVecContainerVT = SubVecVT; 1657 // Establish the correct scalable-vector types for any fixed-length type. 1658 if (SubVecVT.isFixedLengthVector()) 1659 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT); 1660 if (VT.isFixedLengthVector()) 1661 VT = TLI.getContainerForFixedLengthVector(VT); 1662 1663 const auto *TRI = Subtarget->getRegisterInfo(); 1664 unsigned SubRegIdx; 1665 std::tie(SubRegIdx, Idx) = 1666 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1667 VT, SubVecContainerVT, Idx, TRI); 1668 1669 // If the Idx hasn't been completely eliminated then this is a subvector 1670 // insert which doesn't naturally align to a vector register. These must 1671 // be handled using instructions to manipulate the vector registers. 1672 if (Idx != 0) 1673 break; 1674 1675 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT); 1676 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 || 1677 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 || 1678 SubVecLMUL == RISCVII::VLMUL::LMUL_F8; 1679 (void)IsSubVecPartReg; // Silence unused variable warning without asserts. 1680 assert((!IsSubVecPartReg || V.isUndef()) && 1681 "Expecting lowering to have created legal INSERT_SUBVECTORs when " 1682 "the subvector is smaller than a full-sized register"); 1683 1684 // If we haven't set a SubRegIdx, then we must be going between 1685 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. 1686 if (SubRegIdx == RISCV::NoSubRegister) { 1687 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT); 1688 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1689 InRegClassID && 1690 "Unexpected subvector extraction"); 1691 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1692 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1693 DL, VT, SubV, RC); 1694 ReplaceNode(Node, NewNode); 1695 return; 1696 } 1697 1698 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV); 1699 ReplaceNode(Node, Insert.getNode()); 1700 return; 1701 } 1702 case ISD::EXTRACT_SUBVECTOR: { 1703 SDValue V = Node->getOperand(0); 1704 auto Idx = Node->getConstantOperandVal(1); 1705 MVT InVT = V.getSimpleValueType(); 1706 SDLoc DL(V); 1707 1708 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1709 MVT SubVecContainerVT = VT; 1710 // Establish the correct scalable-vector types for any fixed-length type. 1711 if (VT.isFixedLengthVector()) 1712 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT); 1713 if (InVT.isFixedLengthVector()) 1714 InVT = TLI.getContainerForFixedLengthVector(InVT); 1715 1716 const auto *TRI = Subtarget->getRegisterInfo(); 1717 unsigned SubRegIdx; 1718 std::tie(SubRegIdx, Idx) = 1719 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1720 InVT, SubVecContainerVT, Idx, TRI); 1721 1722 // If the Idx hasn't been completely eliminated then this is a subvector 1723 // extract which doesn't naturally align to a vector register. These must 1724 // be handled using instructions to manipulate the vector registers. 1725 if (Idx != 0) 1726 break; 1727 1728 // If we haven't set a SubRegIdx, then we must be going between 1729 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy. 1730 if (SubRegIdx == RISCV::NoSubRegister) { 1731 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT); 1732 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1733 InRegClassID && 1734 "Unexpected subvector extraction"); 1735 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1736 SDNode *NewNode = 1737 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC); 1738 ReplaceNode(Node, NewNode); 1739 return; 1740 } 1741 1742 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V); 1743 ReplaceNode(Node, Extract.getNode()); 1744 return; 1745 } 1746 case ISD::SPLAT_VECTOR: 1747 case RISCVISD::VMV_S_X_VL: 1748 case RISCVISD::VFMV_S_F_VL: 1749 case RISCVISD::VMV_V_X_VL: 1750 case RISCVISD::VFMV_V_F_VL: { 1751 // Try to match splat of a scalar load to a strided load with stride of x0. 1752 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL || 1753 Node->getOpcode() == RISCVISD::VFMV_S_F_VL; 1754 bool HasPassthruOperand = Node->getOpcode() != ISD::SPLAT_VECTOR; 1755 if (HasPassthruOperand && !Node->getOperand(0).isUndef()) 1756 break; 1757 SDValue Src = HasPassthruOperand ? Node->getOperand(1) : Node->getOperand(0); 1758 auto *Ld = dyn_cast<LoadSDNode>(Src); 1759 if (!Ld) 1760 break; 1761 EVT MemVT = Ld->getMemoryVT(); 1762 // The memory VT should be the same size as the element type. 1763 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize()) 1764 break; 1765 if (!IsProfitableToFold(Src, Node, Node) || 1766 !IsLegalToFold(Src, Node, Node, TM.getOptLevel())) 1767 break; 1768 1769 SDValue VL; 1770 if (Node->getOpcode() == ISD::SPLAT_VECTOR) 1771 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); 1772 else if (IsScalarMove) { 1773 // We could deal with more VL if we update the VSETVLI insert pass to 1774 // avoid introducing more VSETVLI. 1775 if (!isOneConstant(Node->getOperand(2))) 1776 break; 1777 selectVLOp(Node->getOperand(2), VL); 1778 } else 1779 selectVLOp(Node->getOperand(2), VL); 1780 1781 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1782 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 1783 1784 SDValue Operands[] = {Ld->getBasePtr(), 1785 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW, 1786 Ld->getChain()}; 1787 1788 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1789 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo( 1790 /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false, 1791 Log2SEW, static_cast<unsigned>(LMUL)); 1792 MachineSDNode *Load = 1793 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1794 1795 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()}); 1796 1797 ReplaceNode(Node, Load); 1798 return; 1799 } 1800 } 1801 1802 // Select the default instruction. 1803 SelectCode(Node); 1804 } 1805 1806 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand( 1807 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 1808 switch (ConstraintID) { 1809 case InlineAsm::Constraint_m: 1810 // We just support simple memory operands that have a single address 1811 // operand and need no special handling. 1812 OutOps.push_back(Op); 1813 return false; 1814 case InlineAsm::Constraint_A: 1815 OutOps.push_back(Op); 1816 return false; 1817 default: 1818 break; 1819 } 1820 1821 return true; 1822 } 1823 1824 // Select a frame index and an optional immediate offset from an ADD or OR. 1825 bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, 1826 SDValue &Offset) { 1827 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1828 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1829 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT()); 1830 return true; 1831 } 1832 1833 if (!CurDAG->isBaseWithConstantOffset(Addr)) 1834 return false; 1835 1836 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) { 1837 auto *CN = cast<ConstantSDNode>(Addr.getOperand(1)); 1838 if (isInt<12>(CN->getSExtValue())) { 1839 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), 1840 Subtarget->getXLenVT()); 1841 Offset = CurDAG->getTargetConstant(CN->getSExtValue(), SDLoc(Addr), 1842 Subtarget->getXLenVT()); 1843 return true; 1844 } 1845 } 1846 1847 return false; 1848 } 1849 1850 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) { 1851 // If this is FrameIndex, select it directly. Otherwise just let it get 1852 // selected to a register independently. 1853 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) 1854 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1855 else 1856 Base = Addr; 1857 return true; 1858 } 1859 1860 bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, 1861 SDValue &Offset) { 1862 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1863 auto *CN = cast<ConstantSDNode>(Addr.getOperand(1)); 1864 if (isInt<12>(CN->getSExtValue())) { 1865 Offset = CurDAG->getTargetConstant(CN->getSExtValue(), SDLoc(Addr), 1866 Subtarget->getXLenVT()); 1867 return SelectBaseAddr(Addr.getOperand(0), Base); 1868 } 1869 } 1870 1871 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT()); 1872 return SelectBaseAddr(Addr, Base); 1873 } 1874 1875 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, 1876 SDValue &ShAmt) { 1877 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift 1878 // amount. If there is an AND on the shift amount, we can bypass it if it 1879 // doesn't affect any of those bits. 1880 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 1881 const APInt &AndMask = N->getConstantOperandAPInt(1); 1882 1883 // Since the max shift amount is a power of 2 we can subtract 1 to make a 1884 // mask that covers the bits needed to represent all shift amounts. 1885 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!"); 1886 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); 1887 1888 if (ShMask.isSubsetOf(AndMask)) { 1889 ShAmt = N.getOperand(0); 1890 return true; 1891 } 1892 1893 // SimplifyDemandedBits may have optimized the mask so try restoring any 1894 // bits that are known zero. 1895 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0)); 1896 if (ShMask.isSubsetOf(AndMask | Known.Zero)) { 1897 ShAmt = N.getOperand(0); 1898 return true; 1899 } 1900 } else if (N.getOpcode() == ISD::SUB && 1901 isa<ConstantSDNode>(N.getOperand(0))) { 1902 uint64_t Imm = N.getConstantOperandVal(0); 1903 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to 1904 // generate a NEG instead of a SUB of a constant. 1905 if (Imm != 0 && Imm % ShiftWidth == 0) { 1906 SDLoc DL(N); 1907 EVT VT = N.getValueType(); 1908 SDValue Zero = 1909 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT); 1910 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB; 1911 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero, 1912 N.getOperand(1)); 1913 ShAmt = SDValue(Neg, 0); 1914 return true; 1915 } 1916 } 1917 1918 ShAmt = N; 1919 return true; 1920 } 1921 1922 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) { 1923 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG && 1924 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) { 1925 Val = N.getOperand(0); 1926 return true; 1927 } 1928 MVT VT = N.getSimpleValueType(); 1929 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) { 1930 Val = N; 1931 return true; 1932 } 1933 1934 return false; 1935 } 1936 1937 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) { 1938 if (N.getOpcode() == ISD::AND) { 1939 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1940 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) { 1941 Val = N.getOperand(0); 1942 return true; 1943 } 1944 } 1945 MVT VT = N.getSimpleValueType(); 1946 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32); 1947 if (CurDAG->MaskedValueIsZero(N, Mask)) { 1948 Val = N; 1949 return true; 1950 } 1951 1952 return false; 1953 } 1954 1955 // Return true if all users of this SDNode* only consume the lower \p Bits. 1956 // This can be used to form W instructions for add/sub/mul/shl even when the 1957 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if 1958 // SimplifyDemandedBits has made it so some users see a sext_inreg and some 1959 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave 1960 // the add/sub/mul/shl to become non-W instructions. By checking the users we 1961 // may be able to use a W instruction and CSE with the other instruction if 1962 // this has happened. We could try to detect that the CSE opportunity exists 1963 // before doing this, but that would be more complicated. 1964 // TODO: Does this need to look through AND/OR/XOR to their users to find more 1965 // opportunities. 1966 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const { 1967 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB || 1968 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL || 1969 Node->getOpcode() == ISD::SRL || 1970 Node->getOpcode() == ISD::SIGN_EXTEND_INREG || 1971 Node->getOpcode() == RISCVISD::GREV || 1972 Node->getOpcode() == RISCVISD::GORC || 1973 isa<ConstantSDNode>(Node)) && 1974 "Unexpected opcode"); 1975 1976 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 1977 SDNode *User = *UI; 1978 // Users of this node should have already been instruction selected 1979 if (!User->isMachineOpcode()) 1980 return false; 1981 1982 // TODO: Add more opcodes? 1983 switch (User->getMachineOpcode()) { 1984 default: 1985 return false; 1986 case RISCV::ADDW: 1987 case RISCV::ADDIW: 1988 case RISCV::SUBW: 1989 case RISCV::MULW: 1990 case RISCV::SLLW: 1991 case RISCV::SLLIW: 1992 case RISCV::SRAW: 1993 case RISCV::SRAIW: 1994 case RISCV::SRLW: 1995 case RISCV::SRLIW: 1996 case RISCV::DIVW: 1997 case RISCV::DIVUW: 1998 case RISCV::REMW: 1999 case RISCV::REMUW: 2000 case RISCV::ROLW: 2001 case RISCV::RORW: 2002 case RISCV::RORIW: 2003 case RISCV::CLZW: 2004 case RISCV::CTZW: 2005 case RISCV::CPOPW: 2006 case RISCV::SLLI_UW: 2007 case RISCV::FMV_W_X: 2008 case RISCV::FCVT_H_W: 2009 case RISCV::FCVT_H_WU: 2010 case RISCV::FCVT_S_W: 2011 case RISCV::FCVT_S_WU: 2012 case RISCV::FCVT_D_W: 2013 case RISCV::FCVT_D_WU: 2014 if (Bits < 32) 2015 return false; 2016 break; 2017 case RISCV::SLLI: 2018 // SLLI only uses the lower (XLen - ShAmt) bits. 2019 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1)) 2020 return false; 2021 break; 2022 case RISCV::ANDI: 2023 if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1)))) 2024 return false; 2025 break; 2026 case RISCV::SEXT_B: 2027 if (Bits < 8) 2028 return false; 2029 break; 2030 case RISCV::SEXT_H: 2031 case RISCV::FMV_H_X: 2032 case RISCV::ZEXT_H_RV32: 2033 case RISCV::ZEXT_H_RV64: 2034 if (Bits < 16) 2035 return false; 2036 break; 2037 case RISCV::ADD_UW: 2038 case RISCV::SH1ADD_UW: 2039 case RISCV::SH2ADD_UW: 2040 case RISCV::SH3ADD_UW: 2041 // The first operand to add.uw/shXadd.uw is implicitly zero extended from 2042 // 32 bits. 2043 if (UI.getOperandNo() != 0 || Bits < 32) 2044 return false; 2045 break; 2046 case RISCV::SB: 2047 if (UI.getOperandNo() != 0 || Bits < 8) 2048 return false; 2049 break; 2050 case RISCV::SH: 2051 if (UI.getOperandNo() != 0 || Bits < 16) 2052 return false; 2053 break; 2054 case RISCV::SW: 2055 if (UI.getOperandNo() != 0 || Bits < 32) 2056 return false; 2057 break; 2058 } 2059 } 2060 2061 return true; 2062 } 2063 2064 // Select VL as a 5 bit immediate or a value that will become a register. This 2065 // allows us to choose betwen VSETIVLI or VSETVLI later. 2066 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { 2067 auto *C = dyn_cast<ConstantSDNode>(N); 2068 if (C && isUInt<5>(C->getZExtValue())) { 2069 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N), 2070 N->getValueType(0)); 2071 } else if (C && C->isAllOnesValue()) { 2072 // Treat all ones as VLMax. 2073 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2074 N->getValueType(0)); 2075 } else if (isa<RegisterSDNode>(N) && 2076 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) { 2077 // All our VL operands use an operand that allows GPRNoX0 or an immediate 2078 // as the register class. Convert X0 to a special immediate to pass the 2079 // MachineVerifier. This is recognized specially by the vsetvli insertion 2080 // pass. 2081 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2082 N->getValueType(0)); 2083 } else { 2084 VL = N; 2085 } 2086 2087 return true; 2088 } 2089 2090 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { 2091 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef()) 2092 return false; 2093 SplatVal = N.getOperand(1); 2094 return true; 2095 } 2096 2097 using ValidateFn = bool (*)(int64_t); 2098 2099 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, 2100 SelectionDAG &DAG, 2101 const RISCVSubtarget &Subtarget, 2102 ValidateFn ValidateImm) { 2103 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2104 !isa<ConstantSDNode>(N.getOperand(1))) 2105 return false; 2106 2107 int64_t SplatImm = 2108 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2109 2110 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand 2111 // type is wider than the resulting vector element type: an implicit 2112 // truncation first takes place. Therefore, perform a manual 2113 // truncation/sign-extension in order to ignore any truncated bits and catch 2114 // any zero-extended immediate. 2115 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first 2116 // sign-extending to (XLenVT -1). 2117 MVT XLenVT = Subtarget.getXLenVT(); 2118 assert(XLenVT == N.getOperand(1).getSimpleValueType() && 2119 "Unexpected splat operand type"); 2120 MVT EltVT = N.getSimpleValueType().getVectorElementType(); 2121 if (EltVT.bitsLT(XLenVT)) 2122 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits()); 2123 2124 if (!ValidateImm(SplatImm)) 2125 return false; 2126 2127 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT); 2128 return true; 2129 } 2130 2131 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) { 2132 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget, 2133 [](int64_t Imm) { return isInt<5>(Imm); }); 2134 } 2135 2136 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) { 2137 return selectVSplatSimmHelper( 2138 N, SplatVal, *CurDAG, *Subtarget, 2139 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; }); 2140 } 2141 2142 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N, 2143 SDValue &SplatVal) { 2144 return selectVSplatSimmHelper( 2145 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) { 2146 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16); 2147 }); 2148 } 2149 2150 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) { 2151 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2152 !isa<ConstantSDNode>(N.getOperand(1))) 2153 return false; 2154 2155 int64_t SplatImm = 2156 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2157 2158 if (!isUInt<5>(SplatImm)) 2159 return false; 2160 2161 SplatVal = 2162 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT()); 2163 2164 return true; 2165 } 2166 2167 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width, 2168 SDValue &Imm) { 2169 if (auto *C = dyn_cast<ConstantSDNode>(N)) { 2170 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width); 2171 2172 if (!isInt<5>(ImmVal)) 2173 return false; 2174 2175 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT()); 2176 return true; 2177 } 2178 2179 return false; 2180 } 2181 2182 // Merge an ADDI into the offset of a load/store instruction where possible. 2183 // (load (addi base, off1), off2) -> (load base, off1+off2) 2184 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2) 2185 // (load (add base, (addi src, off1)), off2) 2186 // -> (load (add base, src), off1+off2) 2187 // (store val, (add base, (addi src, off1)), off2) 2188 // -> (store val, (add base, src), off1+off2) 2189 // This is possible when off1+off2 fits a 12-bit immediate. 2190 bool RISCVDAGToDAGISel::doPeepholeLoadStoreADDI(SDNode *N) { 2191 unsigned OffsetOpIdx, BaseOpIdx; 2192 if (!hasMemOffset(N, BaseOpIdx, OffsetOpIdx)) 2193 return false; 2194 2195 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx))) 2196 return false; 2197 2198 SDValue Base = N->getOperand(BaseOpIdx); 2199 2200 if (!Base.isMachineOpcode()) 2201 return false; 2202 2203 if (Base.getMachineOpcode() == RISCV::ADDI) { 2204 // If the base is an ADDI, we can merge it in to the load/store. 2205 } else if (Base.getMachineOpcode() == RISCV::ADDIW && 2206 isa<ConstantSDNode>(Base.getOperand(1)) && 2207 Base.getOperand(0).isMachineOpcode() && 2208 Base.getOperand(0).getMachineOpcode() == RISCV::LUI && 2209 isa<ConstantSDNode>(Base.getOperand(0).getOperand(0))) { 2210 // ADDIW can be merged if it's part of LUI+ADDIW constant materialization 2211 // and LUI+ADDI would have produced the same result. This is true for all 2212 // simm32 values except 0x7ffff800-0x7fffffff. 2213 int64_t Offset = 2214 SignExtend64<32>(Base.getOperand(0).getConstantOperandVal(0) << 12); 2215 Offset += cast<ConstantSDNode>(Base.getOperand(1))->getSExtValue(); 2216 if (!isInt<32>(Offset)) 2217 return false; 2218 } else 2219 return false; 2220 2221 SDValue ImmOperand = Base.getOperand(1); 2222 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx); 2223 2224 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) { 2225 int64_t Offset1 = Const->getSExtValue(); 2226 int64_t CombinedOffset = Offset1 + Offset2; 2227 if (!isInt<12>(CombinedOffset)) 2228 return false; 2229 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand), 2230 ImmOperand.getValueType()); 2231 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) { 2232 // If the off1 in (addi base, off1) is a global variable's address (its 2233 // low part, really), then we can rely on the alignment of that variable 2234 // to provide a margin of safety before off1 can overflow the 12 bits. 2235 // Check if off2 falls within that margin; if so off1+off2 can't overflow. 2236 const DataLayout &DL = CurDAG->getDataLayout(); 2237 Align Alignment = GA->getGlobal()->getPointerAlignment(DL); 2238 if (Offset2 != 0 && Alignment <= Offset2) 2239 return false; 2240 int64_t Offset1 = GA->getOffset(); 2241 int64_t CombinedOffset = Offset1 + Offset2; 2242 ImmOperand = CurDAG->getTargetGlobalAddress( 2243 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(), 2244 CombinedOffset, GA->getTargetFlags()); 2245 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) { 2246 // Ditto. 2247 Align Alignment = CP->getAlign(); 2248 if (Offset2 != 0 && Alignment <= Offset2) 2249 return false; 2250 int64_t Offset1 = CP->getOffset(); 2251 int64_t CombinedOffset = Offset1 + Offset2; 2252 ImmOperand = CurDAG->getTargetConstantPool( 2253 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(), 2254 CombinedOffset, CP->getTargetFlags()); 2255 } else { 2256 return false; 2257 } 2258 2259 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 2260 LLVM_DEBUG(Base->dump(CurDAG)); 2261 LLVM_DEBUG(dbgs() << "\nN: "); 2262 LLVM_DEBUG(N->dump(CurDAG)); 2263 LLVM_DEBUG(dbgs() << "\n"); 2264 2265 // Modify the offset operand of the load/store. 2266 if (BaseOpIdx == 0) { // Load 2267 N = CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand, 2268 N->getOperand(2)); 2269 } else { // Store 2270 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0), 2271 ImmOperand, N->getOperand(3)); 2272 } 2273 2274 return true; 2275 } 2276 2277 // Try to remove sext.w if the input is a W instruction or can be made into 2278 // a W instruction cheaply. 2279 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { 2280 // Look for the sext.w pattern, addiw rd, rs1, 0. 2281 if (N->getMachineOpcode() != RISCV::ADDIW || 2282 !isNullConstant(N->getOperand(1))) 2283 return false; 2284 2285 SDValue N0 = N->getOperand(0); 2286 if (!N0.isMachineOpcode()) 2287 return false; 2288 2289 switch (N0.getMachineOpcode()) { 2290 default: 2291 break; 2292 case RISCV::ADD: 2293 case RISCV::ADDI: 2294 case RISCV::SUB: 2295 case RISCV::MUL: 2296 case RISCV::SLLI: { 2297 // Convert sext.w+add/sub/mul to their W instructions. This will create 2298 // a new independent instruction. This improves latency. 2299 unsigned Opc; 2300 switch (N0.getMachineOpcode()) { 2301 default: 2302 llvm_unreachable("Unexpected opcode!"); 2303 case RISCV::ADD: Opc = RISCV::ADDW; break; 2304 case RISCV::ADDI: Opc = RISCV::ADDIW; break; 2305 case RISCV::SUB: Opc = RISCV::SUBW; break; 2306 case RISCV::MUL: Opc = RISCV::MULW; break; 2307 case RISCV::SLLI: Opc = RISCV::SLLIW; break; 2308 } 2309 2310 SDValue N00 = N0.getOperand(0); 2311 SDValue N01 = N0.getOperand(1); 2312 2313 // Shift amount needs to be uimm5. 2314 if (N0.getMachineOpcode() == RISCV::SLLI && 2315 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue())) 2316 break; 2317 2318 SDNode *Result = 2319 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), 2320 N00, N01); 2321 ReplaceUses(N, Result); 2322 return true; 2323 } 2324 case RISCV::ADDW: 2325 case RISCV::ADDIW: 2326 case RISCV::SUBW: 2327 case RISCV::MULW: 2328 case RISCV::SLLIW: 2329 case RISCV::GREVIW: 2330 case RISCV::GORCIW: 2331 // Result is already sign extended just remove the sext.w. 2332 // NOTE: We only handle the nodes that are selected with hasAllWUsers. 2333 ReplaceUses(N, N0.getNode()); 2334 return true; 2335 } 2336 2337 return false; 2338 } 2339 2340 // Optimize masked RVV pseudo instructions with a known all-ones mask to their 2341 // corresponding "unmasked" pseudo versions. The mask we're interested in will 2342 // take the form of a V0 physical register operand, with a glued 2343 // register-setting instruction. 2344 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) { 2345 const RISCV::RISCVMaskedPseudoInfo *I = 2346 RISCV::getMaskedPseudoInfo(N->getMachineOpcode()); 2347 if (!I) 2348 return false; 2349 2350 unsigned MaskOpIdx = I->MaskOpIdx; 2351 2352 // Check that we're using V0 as a mask register. 2353 if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) || 2354 cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0) 2355 return false; 2356 2357 // The glued user defines V0. 2358 const auto *Glued = N->getGluedNode(); 2359 2360 if (!Glued || Glued->getOpcode() != ISD::CopyToReg) 2361 return false; 2362 2363 // Check that we're defining V0 as a mask register. 2364 if (!isa<RegisterSDNode>(Glued->getOperand(1)) || 2365 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0) 2366 return false; 2367 2368 // Check the instruction defining V0; it needs to be a VMSET pseudo. 2369 SDValue MaskSetter = Glued->getOperand(2); 2370 2371 const auto IsVMSet = [](unsigned Opc) { 2372 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 || 2373 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 || 2374 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 || 2375 Opc == RISCV::PseudoVMSET_M_B8; 2376 }; 2377 2378 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has 2379 // undefined behaviour if it's the wrong bitwidth, so we could choose to 2380 // assume that it's all-ones? Same applies to its VL. 2381 if (!MaskSetter->isMachineOpcode() || !IsVMSet(MaskSetter.getMachineOpcode())) 2382 return false; 2383 2384 // Retrieve the tail policy operand index, if any. 2385 Optional<unsigned> TailPolicyOpIdx; 2386 const RISCVInstrInfo &TII = *Subtarget->getInstrInfo(); 2387 const MCInstrDesc &MaskedMCID = TII.get(N->getMachineOpcode()); 2388 2389 bool IsTA = true; 2390 if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) { 2391 // The last operand of the pseudo is the policy op, but we might have a 2392 // Glue operand last. We might also have a chain. 2393 TailPolicyOpIdx = N->getNumOperands() - 1; 2394 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Glue) 2395 (*TailPolicyOpIdx)--; 2396 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Other) 2397 (*TailPolicyOpIdx)--; 2398 2399 if (!(N->getConstantOperandVal(*TailPolicyOpIdx) & 2400 RISCVII::TAIL_AGNOSTIC)) { 2401 // Keep the true-masked instruction when there is no unmasked TU 2402 // instruction 2403 if (I->UnmaskedTUPseudo == I->MaskedPseudo && !N->getOperand(0).isUndef()) 2404 return false; 2405 // We can't use TA if the tie-operand is not IMPLICIT_DEF 2406 if (!N->getOperand(0).isUndef()) 2407 IsTA = false; 2408 } 2409 } 2410 2411 if (IsTA) { 2412 uint64_t TSFlags = TII.get(I->UnmaskedPseudo).TSFlags; 2413 2414 // Check that we're dropping the merge operand, the mask operand, and any 2415 // policy operand when we transform to this unmasked pseudo. 2416 assert(!RISCVII::hasMergeOp(TSFlags) && RISCVII::hasDummyMaskOp(TSFlags) && 2417 !RISCVII::hasVecPolicyOp(TSFlags) && 2418 "Unexpected pseudo to transform to"); 2419 (void)TSFlags; 2420 } else { 2421 uint64_t TSFlags = TII.get(I->UnmaskedTUPseudo).TSFlags; 2422 2423 // Check that we're dropping the mask operand, and any policy operand 2424 // when we transform to this unmasked tu pseudo. 2425 assert(RISCVII::hasMergeOp(TSFlags) && RISCVII::hasDummyMaskOp(TSFlags) && 2426 !RISCVII::hasVecPolicyOp(TSFlags) && 2427 "Unexpected pseudo to transform to"); 2428 (void)TSFlags; 2429 } 2430 2431 unsigned Opc = IsTA ? I->UnmaskedPseudo : I->UnmaskedTUPseudo; 2432 SmallVector<SDValue, 8> Ops; 2433 // Skip the merge operand at index 0 if IsTA 2434 for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) { 2435 // Skip the mask, the policy, and the Glue. 2436 SDValue Op = N->getOperand(I); 2437 if (I == MaskOpIdx || I == TailPolicyOpIdx || 2438 Op.getValueType() == MVT::Glue) 2439 continue; 2440 Ops.push_back(Op); 2441 } 2442 2443 // Transitively apply any node glued to our new node. 2444 if (auto *TGlued = Glued->getGluedNode()) 2445 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1)); 2446 2447 SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 2448 ReplaceUses(N, Result); 2449 2450 return true; 2451 } 2452 2453 // This pass converts a legalized DAG into a RISCV-specific DAG, ready 2454 // for instruction scheduling. 2455 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM, 2456 CodeGenOpt::Level OptLevel) { 2457 return new RISCVDAGToDAGISel(TM, OptLevel); 2458 } 2459