1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the RISCV target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVISelDAGToDAG.h" 14 #include "MCTargetDesc/RISCVMCTargetDesc.h" 15 #include "MCTargetDesc/RISCVMatInt.h" 16 #include "RISCVISelLowering.h" 17 #include "RISCVMachineFunctionInfo.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/IR/IntrinsicsRISCV.h" 20 #include "llvm/Support/Alignment.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/KnownBits.h" 23 #include "llvm/Support/MathExtras.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 #define DEBUG_TYPE "riscv-isel" 29 30 namespace llvm { 31 namespace RISCV { 32 #define GET_RISCVVSSEGTable_IMPL 33 #define GET_RISCVVLSEGTable_IMPL 34 #define GET_RISCVVLXSEGTable_IMPL 35 #define GET_RISCVVSXSEGTable_IMPL 36 #define GET_RISCVVLETable_IMPL 37 #define GET_RISCVVSETable_IMPL 38 #define GET_RISCVVLXTable_IMPL 39 #define GET_RISCVVSXTable_IMPL 40 #define GET_RISCVMaskedPseudosTable_IMPL 41 #include "RISCVGenSearchableTables.inc" 42 } // namespace RISCV 43 } // namespace llvm 44 45 void RISCVDAGToDAGISel::PreprocessISelDAG() { 46 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 47 E = CurDAG->allnodes_end(); 48 I != E;) { 49 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 50 51 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point 52 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden. 53 if (N->getOpcode() == ISD::SPLAT_VECTOR) { 54 MVT VT = N->getSimpleValueType(0); 55 unsigned Opc = 56 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL; 57 SDLoc DL(N); 58 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()); 59 SDValue Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), 60 N->getOperand(0), VL); 61 62 --I; 63 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 64 ++I; 65 CurDAG->DeleteNode(N); 66 continue; 67 } 68 69 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector 70 // load. Done after lowering and combining so that we have a chance to 71 // optimize this to VMV_V_X_VL when the upper bits aren't needed. 72 if (N->getOpcode() != RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) 73 continue; 74 75 assert(N->getNumOperands() == 4 && "Unexpected number of operands"); 76 MVT VT = N->getSimpleValueType(0); 77 SDValue Passthru = N->getOperand(0); 78 SDValue Lo = N->getOperand(1); 79 SDValue Hi = N->getOperand(2); 80 SDValue VL = N->getOperand(3); 81 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() && 82 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 && 83 "Unexpected VTs!"); 84 MachineFunction &MF = CurDAG->getMachineFunction(); 85 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>(); 86 SDLoc DL(N); 87 88 // We use the same frame index we use for moving two i32s into 64-bit FPR. 89 // This is an analogous operation. 90 int FI = FuncInfo->getMoveF64FrameIndex(MF); 91 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); 92 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 93 SDValue StackSlot = 94 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout())); 95 96 SDValue Chain = CurDAG->getEntryNode(); 97 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8)); 98 99 SDValue OffsetSlot = 100 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL); 101 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4), 102 Align(8)); 103 104 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 105 106 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other}); 107 SDValue IntID = 108 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64); 109 SDValue Ops[] = {Chain, 110 IntID, 111 Passthru, 112 StackSlot, 113 CurDAG->getRegister(RISCV::X0, MVT::i64), 114 VL}; 115 116 SDValue Result = CurDAG->getMemIntrinsicNode( 117 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MVT::i64, MPI, Align(8), 118 MachineMemOperand::MOLoad); 119 120 // We're about to replace all uses of the SPLAT_VECTOR_SPLIT_I64 with the 121 // vlse we created. This will cause general havok on the dag because 122 // anything below the conversion could be folded into other existing nodes. 123 // To avoid invalidating 'I', back it up to the convert node. 124 --I; 125 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 126 127 // Now that we did that, the node is dead. Increment the iterator to the 128 // next node to process, then delete N. 129 ++I; 130 CurDAG->DeleteNode(N); 131 } 132 } 133 134 void RISCVDAGToDAGISel::PostprocessISelDAG() { 135 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 136 137 bool MadeChange = false; 138 while (Position != CurDAG->allnodes_begin()) { 139 SDNode *N = &*--Position; 140 // Skip dead nodes and any non-machine opcodes. 141 if (N->use_empty() || !N->isMachineOpcode()) 142 continue; 143 144 MadeChange |= doPeepholeSExtW(N); 145 MadeChange |= doPeepholeLoadStoreADDI(N); 146 MadeChange |= doPeepholeMaskedRVV(N); 147 } 148 149 if (MadeChange) 150 CurDAG->RemoveDeadNodes(); 151 } 152 153 static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL, 154 const MVT VT, int64_t Imm, 155 const RISCVSubtarget &Subtarget) { 156 assert(VT == MVT::i64 && "Expecting MVT::i64"); 157 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering(); 158 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool( 159 ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT)); 160 SDValue Addr = TLI->getAddr(CP, *CurDAG); 161 SDValue Offset = CurDAG->getTargetConstant(0, DL, VT); 162 // Since there is no data race, the chain can be the entry node. 163 SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset, 164 CurDAG->getEntryNode()); 165 MachineFunction &MF = CurDAG->getMachineFunction(); 166 MachineMemOperand *MemOp = MF.getMachineMemOperand( 167 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 168 LLT(VT), CP->getAlign()); 169 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp}); 170 return Load; 171 } 172 173 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 174 int64_t Imm, const RISCVSubtarget &Subtarget) { 175 MVT XLenVT = Subtarget.getXLenVT(); 176 RISCVMatInt::InstSeq Seq = 177 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits()); 178 179 // If Imm is expensive to build, then we put it into constant pool. 180 if (Subtarget.useConstantPoolForLargeInts() && 181 Seq.size() > Subtarget.getMaxBuildIntsCost()) 182 return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget); 183 184 SDNode *Result = nullptr; 185 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT); 186 for (RISCVMatInt::Inst &Inst : Seq) { 187 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT); 188 if (Inst.Opc == RISCV::LUI) 189 Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm); 190 else if (Inst.Opc == RISCV::ADD_UW) 191 Result = CurDAG->getMachineNode(RISCV::ADD_UW, DL, XLenVT, SrcReg, 192 CurDAG->getRegister(RISCV::X0, XLenVT)); 193 else if (Inst.Opc == RISCV::SH1ADD || Inst.Opc == RISCV::SH2ADD || 194 Inst.Opc == RISCV::SH3ADD) 195 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SrcReg); 196 else 197 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm); 198 199 // Only the first instruction has X0 as its source. 200 SrcReg = SDValue(Result, 0); 201 } 202 203 return Result; 204 } 205 206 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 207 unsigned RegClassID, unsigned SubReg0) { 208 assert(Regs.size() >= 2 && Regs.size() <= 8); 209 210 SDLoc DL(Regs[0]); 211 SmallVector<SDValue, 8> Ops; 212 213 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32)); 214 215 for (unsigned I = 0; I < Regs.size(); ++I) { 216 Ops.push_back(Regs[I]); 217 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32)); 218 } 219 SDNode *N = 220 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 221 return SDValue(N, 0); 222 } 223 224 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 225 unsigned NF) { 226 static const unsigned RegClassIDs[] = { 227 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID, 228 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID, 229 RISCV::VRN8M1RegClassID}; 230 231 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0); 232 } 233 234 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 235 unsigned NF) { 236 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID, 237 RISCV::VRN3M2RegClassID, 238 RISCV::VRN4M2RegClassID}; 239 240 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0); 241 } 242 243 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 244 unsigned NF) { 245 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID, 246 RISCV::sub_vrm4_0); 247 } 248 249 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 250 unsigned NF, RISCVII::VLMUL LMUL) { 251 switch (LMUL) { 252 default: 253 llvm_unreachable("Invalid LMUL."); 254 case RISCVII::VLMUL::LMUL_F8: 255 case RISCVII::VLMUL::LMUL_F4: 256 case RISCVII::VLMUL::LMUL_F2: 257 case RISCVII::VLMUL::LMUL_1: 258 return createM1Tuple(CurDAG, Regs, NF); 259 case RISCVII::VLMUL::LMUL_2: 260 return createM2Tuple(CurDAG, Regs, NF); 261 case RISCVII::VLMUL::LMUL_4: 262 return createM4Tuple(CurDAG, Regs, NF); 263 } 264 } 265 266 void RISCVDAGToDAGISel::addVectorLoadStoreOperands( 267 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp, 268 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands, 269 bool IsLoad, MVT *IndexVT) { 270 SDValue Chain = Node->getOperand(0); 271 SDValue Glue; 272 273 SDValue Base; 274 SelectBaseAddr(Node->getOperand(CurOp++), Base); 275 Operands.push_back(Base); // Base pointer. 276 277 if (IsStridedOrIndexed) { 278 Operands.push_back(Node->getOperand(CurOp++)); // Index. 279 if (IndexVT) 280 *IndexVT = Operands.back()->getSimpleValueType(0); 281 } 282 283 if (IsMasked) { 284 // Mask needs to be copied to V0. 285 SDValue Mask = Node->getOperand(CurOp++); 286 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); 287 Glue = Chain.getValue(1); 288 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); 289 } 290 SDValue VL; 291 selectVLOp(Node->getOperand(CurOp++), VL); 292 Operands.push_back(VL); 293 294 MVT XLenVT = Subtarget->getXLenVT(); 295 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 296 Operands.push_back(SEWOp); 297 298 // Masked load has the tail policy argument. 299 if (IsMasked && IsLoad) { 300 // Policy must be a constant. 301 uint64_t Policy = Node->getConstantOperandVal(CurOp++); 302 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT); 303 Operands.push_back(PolicyOp); 304 } 305 306 Operands.push_back(Chain); // Chain. 307 if (Glue) 308 Operands.push_back(Glue); 309 } 310 311 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked, 312 bool IsStrided) { 313 SDLoc DL(Node); 314 unsigned NF = Node->getNumValues() - 1; 315 MVT VT = Node->getSimpleValueType(0); 316 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 317 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 318 319 unsigned CurOp = 2; 320 SmallVector<SDValue, 8> Operands; 321 if (IsMasked) { 322 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 323 Node->op_begin() + CurOp + NF); 324 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 325 Operands.push_back(MaskedOff); 326 CurOp += NF; 327 } 328 329 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 330 Operands, /*IsLoad=*/true); 331 332 const RISCV::VLSEGPseudo *P = 333 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW, 334 static_cast<unsigned>(LMUL)); 335 MachineSDNode *Load = 336 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 337 338 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 339 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 340 341 SDValue SuperReg = SDValue(Load, 0); 342 for (unsigned I = 0; I < NF; ++I) { 343 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 344 ReplaceUses(SDValue(Node, I), 345 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 346 } 347 348 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 349 CurDAG->RemoveDeadNode(Node); 350 } 351 352 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) { 353 SDLoc DL(Node); 354 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. 355 MVT VT = Node->getSimpleValueType(0); 356 MVT XLenVT = Subtarget->getXLenVT(); 357 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 358 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 359 360 unsigned CurOp = 2; 361 SmallVector<SDValue, 7> Operands; 362 if (IsMasked) { 363 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 364 Node->op_begin() + CurOp + NF); 365 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 366 Operands.push_back(MaskedOff); 367 CurOp += NF; 368 } 369 370 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 371 /*IsStridedOrIndexed*/ false, Operands, 372 /*IsLoad=*/true); 373 374 const RISCV::VLSEGPseudo *P = 375 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true, 376 Log2SEW, static_cast<unsigned>(LMUL)); 377 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, 378 MVT::Other, MVT::Glue, Operands); 379 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, 380 /*Glue*/ SDValue(Load, 2)); 381 382 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 383 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 384 385 SDValue SuperReg = SDValue(Load, 0); 386 for (unsigned I = 0; I < NF; ++I) { 387 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 388 ReplaceUses(SDValue(Node, I), 389 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 390 } 391 392 ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL 393 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain 394 CurDAG->RemoveDeadNode(Node); 395 } 396 397 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked, 398 bool IsOrdered) { 399 SDLoc DL(Node); 400 unsigned NF = Node->getNumValues() - 1; 401 MVT VT = Node->getSimpleValueType(0); 402 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 403 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 404 405 unsigned CurOp = 2; 406 SmallVector<SDValue, 8> Operands; 407 if (IsMasked) { 408 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 409 Node->op_begin() + CurOp + NF); 410 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 411 Operands.push_back(MaskedOff); 412 CurOp += NF; 413 } 414 415 MVT IndexVT; 416 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 417 /*IsStridedOrIndexed*/ true, Operands, 418 /*IsLoad=*/true, &IndexVT); 419 420 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 421 "Element count mismatch"); 422 423 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 424 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 425 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 426 report_fatal_error("The V extension does not support EEW=64 for index " 427 "values when XLEN=32"); 428 } 429 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo( 430 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 431 static_cast<unsigned>(IndexLMUL)); 432 MachineSDNode *Load = 433 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 434 435 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 436 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 437 438 SDValue SuperReg = SDValue(Load, 0); 439 for (unsigned I = 0; I < NF; ++I) { 440 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 441 ReplaceUses(SDValue(Node, I), 442 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 443 } 444 445 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 446 CurDAG->RemoveDeadNode(Node); 447 } 448 449 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked, 450 bool IsStrided) { 451 SDLoc DL(Node); 452 unsigned NF = Node->getNumOperands() - 4; 453 if (IsStrided) 454 NF--; 455 if (IsMasked) 456 NF--; 457 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 458 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 459 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 460 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 461 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 462 463 SmallVector<SDValue, 8> Operands; 464 Operands.push_back(StoreVal); 465 unsigned CurOp = 2 + NF; 466 467 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 468 Operands); 469 470 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo( 471 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 472 MachineSDNode *Store = 473 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 474 475 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 476 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 477 478 ReplaceNode(Node, Store); 479 } 480 481 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked, 482 bool IsOrdered) { 483 SDLoc DL(Node); 484 unsigned NF = Node->getNumOperands() - 5; 485 if (IsMasked) 486 --NF; 487 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 488 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 489 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 490 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 491 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 492 493 SmallVector<SDValue, 8> Operands; 494 Operands.push_back(StoreVal); 495 unsigned CurOp = 2 + NF; 496 497 MVT IndexVT; 498 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 499 /*IsStridedOrIndexed*/ true, Operands, 500 /*IsLoad=*/false, &IndexVT); 501 502 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 503 "Element count mismatch"); 504 505 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 506 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 507 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 508 report_fatal_error("The V extension does not support EEW=64 for index " 509 "values when XLEN=32"); 510 } 511 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo( 512 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 513 static_cast<unsigned>(IndexLMUL)); 514 MachineSDNode *Store = 515 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 516 517 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 518 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 519 520 ReplaceNode(Node, Store); 521 } 522 523 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) { 524 if (!Subtarget->hasVInstructions()) 525 return; 526 527 assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN || 528 Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) && 529 "Unexpected opcode"); 530 531 SDLoc DL(Node); 532 MVT XLenVT = Subtarget->getXLenVT(); 533 534 bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN; 535 unsigned IntNoOffset = HasChain ? 1 : 0; 536 unsigned IntNo = Node->getConstantOperandVal(IntNoOffset); 537 538 assert((IntNo == Intrinsic::riscv_vsetvli || 539 IntNo == Intrinsic::riscv_vsetvlimax || 540 IntNo == Intrinsic::riscv_vsetvli_opt || 541 IntNo == Intrinsic::riscv_vsetvlimax_opt) && 542 "Unexpected vsetvli intrinsic"); 543 544 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax || 545 IntNo == Intrinsic::riscv_vsetvlimax_opt; 546 unsigned Offset = IntNoOffset + (VLMax ? 1 : 2); 547 548 assert(Node->getNumOperands() == Offset + 2 && 549 "Unexpected number of operands"); 550 551 unsigned SEW = 552 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7); 553 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>( 554 Node->getConstantOperandVal(Offset + 1) & 0x7); 555 556 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true, 557 /*MaskAgnostic*/ false); 558 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT); 559 560 SmallVector<EVT, 2> VTs = {XLenVT}; 561 if (HasChain) 562 VTs.push_back(MVT::Other); 563 564 SDValue VLOperand; 565 unsigned Opcode = RISCV::PseudoVSETVLI; 566 if (VLMax) { 567 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT); 568 Opcode = RISCV::PseudoVSETVLIX0; 569 } else { 570 VLOperand = Node->getOperand(IntNoOffset + 1); 571 572 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { 573 uint64_t AVL = C->getZExtValue(); 574 if (isUInt<5>(AVL)) { 575 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); 576 SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp}; 577 if (HasChain) 578 Ops.push_back(Node->getOperand(0)); 579 ReplaceNode( 580 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops)); 581 return; 582 } 583 } 584 } 585 586 SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp}; 587 if (HasChain) 588 Ops.push_back(Node->getOperand(0)); 589 590 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops)); 591 } 592 593 void RISCVDAGToDAGISel::Select(SDNode *Node) { 594 // If we have a custom node, we have already selected. 595 if (Node->isMachineOpcode()) { 596 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 597 Node->setNodeId(-1); 598 return; 599 } 600 601 // Instruction Selection not handled by the auto-generated tablegen selection 602 // should be handled here. 603 unsigned Opcode = Node->getOpcode(); 604 MVT XLenVT = Subtarget->getXLenVT(); 605 SDLoc DL(Node); 606 MVT VT = Node->getSimpleValueType(0); 607 608 switch (Opcode) { 609 case ISD::Constant: { 610 auto *ConstNode = cast<ConstantSDNode>(Node); 611 if (VT == XLenVT && ConstNode->isZero()) { 612 SDValue New = 613 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT); 614 ReplaceNode(Node, New.getNode()); 615 return; 616 } 617 int64_t Imm = ConstNode->getSExtValue(); 618 // If the upper XLen-16 bits are not used, try to convert this to a simm12 619 // by sign extending bit 15. 620 if (isUInt<16>(Imm) && isInt<12>(SignExtend64(Imm, 16)) && 621 hasAllHUsers(Node)) 622 Imm = SignExtend64(Imm, 16); 623 // If the upper 32-bits are not used try to convert this into a simm32 by 624 // sign extending bit 32. 625 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node)) 626 Imm = SignExtend64(Imm, 32); 627 628 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget)); 629 return; 630 } 631 case ISD::FrameIndex: { 632 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT); 633 int FI = cast<FrameIndexSDNode>(Node)->getIndex(); 634 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT); 635 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm)); 636 return; 637 } 638 case ISD::SRL: { 639 // Optimize (srl (and X, C2), C) -> 640 // (srli (slli X, (XLen-C3), (XLen-C3) + C) 641 // Where C2 is a mask with C3 trailing ones. 642 // Taking into account that the C2 may have had lower bits unset by 643 // SimplifyDemandedBits. This avoids materializing the C2 immediate. 644 // This pattern occurs when type legalizing right shifts for types with 645 // less than XLen bits. 646 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 647 if (!N1C) 648 break; 649 SDValue N0 = Node->getOperand(0); 650 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 651 !isa<ConstantSDNode>(N0.getOperand(1))) 652 break; 653 unsigned ShAmt = N1C->getZExtValue(); 654 uint64_t Mask = N0.getConstantOperandVal(1); 655 Mask |= maskTrailingOnes<uint64_t>(ShAmt); 656 if (!isMask_64(Mask)) 657 break; 658 unsigned TrailingOnes = countTrailingOnes(Mask); 659 // 32 trailing ones should use srliw via tablegen pattern. 660 if (TrailingOnes == 32 || ShAmt >= TrailingOnes) 661 break; 662 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes; 663 SDNode *SLLI = 664 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 665 CurDAG->getTargetConstant(LShAmt, DL, VT)); 666 SDNode *SRLI = CurDAG->getMachineNode( 667 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 668 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 669 ReplaceNode(Node, SRLI); 670 return; 671 } 672 case ISD::SRA: { 673 // Optimize (sra (sext_inreg X, i16), C) -> 674 // (srai (slli X, (XLen-16), (XLen-16) + C) 675 // And (sra (sext_inreg X, i8), C) -> 676 // (srai (slli X, (XLen-8), (XLen-8) + C) 677 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal. 678 // This transform matches the code we get without Zbb. The shifts are more 679 // compressible, and this can help expose CSE opportunities in the sdiv by 680 // constant optimization. 681 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 682 if (!N1C) 683 break; 684 SDValue N0 = Node->getOperand(0); 685 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse()) 686 break; 687 unsigned ShAmt = N1C->getZExtValue(); 688 unsigned ExtSize = 689 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits(); 690 // ExtSize of 32 should use sraiw via tablegen pattern. 691 if (ExtSize >= 32 || ShAmt >= ExtSize) 692 break; 693 unsigned LShAmt = Subtarget->getXLen() - ExtSize; 694 SDNode *SLLI = 695 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 696 CurDAG->getTargetConstant(LShAmt, DL, VT)); 697 SDNode *SRAI = CurDAG->getMachineNode( 698 RISCV::SRAI, DL, VT, SDValue(SLLI, 0), 699 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 700 ReplaceNode(Node, SRAI); 701 return; 702 } 703 case ISD::AND: { 704 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 705 if (!N1C) 706 break; 707 708 SDValue N0 = Node->getOperand(0); 709 710 bool LeftShift = N0.getOpcode() == ISD::SHL; 711 if (!LeftShift && N0.getOpcode() != ISD::SRL) 712 break; 713 714 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 715 if (!C) 716 break; 717 uint64_t C2 = C->getZExtValue(); 718 unsigned XLen = Subtarget->getXLen(); 719 if (!C2 || C2 >= XLen) 720 break; 721 722 uint64_t C1 = N1C->getZExtValue(); 723 724 // Keep track of whether this is an andi. 725 bool IsANDI = isInt<12>(N1C->getSExtValue()); 726 727 // Clear irrelevant bits in the mask. 728 if (LeftShift) 729 C1 &= maskTrailingZeros<uint64_t>(C2); 730 else 731 C1 &= maskTrailingOnes<uint64_t>(XLen - C2); 732 733 // Some transforms should only be done if the shift has a single use or 734 // the AND would become (srli (slli X, 32), 32) 735 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF); 736 737 SDValue X = N0.getOperand(0); 738 739 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask 740 // with c3 leading zeros. 741 if (!LeftShift && isMask_64(C1)) { 742 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 743 if (C2 < C3) { 744 // If the number of leading zeros is C2+32 this can be SRLIW. 745 if (C2 + 32 == C3) { 746 SDNode *SRLIW = 747 CurDAG->getMachineNode(RISCV::SRLIW, DL, XLenVT, X, 748 CurDAG->getTargetConstant(C2, DL, XLenVT)); 749 ReplaceNode(Node, SRLIW); 750 return; 751 } 752 753 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if 754 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1. 755 // 756 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type 757 // legalized and goes through DAG combine. 758 if (C2 >= 32 && (C3 - C2) == 1 && N0.hasOneUse() && 759 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 760 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) { 761 SDNode *SRAIW = 762 CurDAG->getMachineNode(RISCV::SRAIW, DL, XLenVT, X.getOperand(0), 763 CurDAG->getTargetConstant(31, DL, XLenVT)); 764 SDNode *SRLIW = CurDAG->getMachineNode( 765 RISCV::SRLIW, DL, XLenVT, SDValue(SRAIW, 0), 766 CurDAG->getTargetConstant(C3 - 32, DL, XLenVT)); 767 ReplaceNode(Node, SRLIW); 768 return; 769 } 770 771 // (srli (slli x, c3-c2), c3). 772 if (OneUseOrZExtW && !IsANDI) { 773 SDNode *SLLI = CurDAG->getMachineNode( 774 RISCV::SLLI, DL, XLenVT, X, 775 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 776 SDNode *SRLI = 777 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 778 CurDAG->getTargetConstant(C3, DL, XLenVT)); 779 ReplaceNode(Node, SRLI); 780 return; 781 } 782 } 783 } 784 785 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask 786 // shifted by c2 bits with c3 leading zeros. 787 if (LeftShift && isShiftedMask_64(C1)) { 788 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 789 790 if (C2 + C3 < XLen && 791 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + C3)) << C2)) { 792 // Use slli.uw when possible. 793 if ((XLen - (C2 + C3)) == 32 && Subtarget->hasStdExtZba()) { 794 SDNode *SLLI_UW = 795 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, XLenVT, X, 796 CurDAG->getTargetConstant(C2, DL, XLenVT)); 797 ReplaceNode(Node, SLLI_UW); 798 return; 799 } 800 801 // (srli (slli c2+c3), c3) 802 if (OneUseOrZExtW && !IsANDI) { 803 SDNode *SLLI = CurDAG->getMachineNode( 804 RISCV::SLLI, DL, XLenVT, X, 805 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 806 SDNode *SRLI = 807 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 808 CurDAG->getTargetConstant(C3, DL, XLenVT)); 809 ReplaceNode(Node, SRLI); 810 return; 811 } 812 } 813 } 814 815 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a 816 // shifted mask with c2 leading zeros and c3 trailing zeros. 817 if (!LeftShift && isShiftedMask_64(C1)) { 818 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 819 uint64_t C3 = countTrailingZeros(C1); 820 if (Leading == C2 && C2 + C3 < XLen && OneUseOrZExtW && !IsANDI) { 821 SDNode *SRLI = CurDAG->getMachineNode( 822 RISCV::SRLI, DL, XLenVT, X, 823 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 824 SDNode *SLLI = 825 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 826 CurDAG->getTargetConstant(C3, DL, XLenVT)); 827 ReplaceNode(Node, SLLI); 828 return; 829 } 830 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI. 831 if (Leading > 32 && (Leading - 32) == C2 && C2 + C3 < 32 && 832 OneUseOrZExtW && !IsANDI) { 833 SDNode *SRLIW = CurDAG->getMachineNode( 834 RISCV::SRLIW, DL, XLenVT, X, 835 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 836 SDNode *SLLI = 837 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 838 CurDAG->getTargetConstant(C3, DL, XLenVT)); 839 ReplaceNode(Node, SLLI); 840 return; 841 } 842 } 843 844 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a 845 // shifted mask with no leading zeros and c3 trailing zeros. 846 if (LeftShift && isShiftedMask_64(C1)) { 847 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 848 uint64_t C3 = countTrailingZeros(C1); 849 if (Leading == 0 && C2 < C3 && OneUseOrZExtW && !IsANDI) { 850 SDNode *SRLI = CurDAG->getMachineNode( 851 RISCV::SRLI, DL, XLenVT, X, 852 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 853 SDNode *SLLI = 854 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 855 CurDAG->getTargetConstant(C3, DL, XLenVT)); 856 ReplaceNode(Node, SLLI); 857 return; 858 } 859 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI. 860 if (C2 < C3 && Leading + C2 == 32 && OneUseOrZExtW && !IsANDI) { 861 SDNode *SRLIW = CurDAG->getMachineNode( 862 RISCV::SRLIW, DL, XLenVT, X, 863 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 864 SDNode *SLLI = 865 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 866 CurDAG->getTargetConstant(C3, DL, XLenVT)); 867 ReplaceNode(Node, SLLI); 868 return; 869 } 870 } 871 872 break; 873 } 874 case ISD::MUL: { 875 // Special case for calculating (mul (and X, C2), C1) where the full product 876 // fits in XLen bits. We can shift X left by the number of leading zeros in 877 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final 878 // product has XLen trailing zeros, putting it in the output of MULHU. This 879 // can avoid materializing a constant in a register for C2. 880 881 // RHS should be a constant. 882 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 883 if (!N1C || !N1C->hasOneUse()) 884 break; 885 886 // LHS should be an AND with constant. 887 SDValue N0 = Node->getOperand(0); 888 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) 889 break; 890 891 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 892 893 // Constant should be a mask. 894 if (!isMask_64(C2)) 895 break; 896 897 // This should be the only use of the AND unless we will use 898 // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND 899 // constants. 900 if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) 901 break; 902 903 // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this 904 // optimization. 905 if (isInt<12>(C2) || 906 (C2 == UINT64_C(0xFFFF) && 907 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || 908 (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) 909 break; 910 911 // We need to shift left the AND input and C1 by a total of XLen bits. 912 913 // How far left do we need to shift the AND input? 914 unsigned XLen = Subtarget->getXLen(); 915 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); 916 917 // The constant gets shifted by the remaining amount unless that would 918 // shift bits out. 919 uint64_t C1 = N1C->getZExtValue(); 920 unsigned ConstantShift = XLen - LeadingZeros; 921 if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) 922 break; 923 924 uint64_t ShiftedC1 = C1 << ConstantShift; 925 // If this RV32, we need to sign extend the constant. 926 if (XLen == 32) 927 ShiftedC1 = SignExtend64(ShiftedC1, 32); 928 929 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). 930 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget); 931 SDNode *SLLI = 932 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), 933 CurDAG->getTargetConstant(LeadingZeros, DL, VT)); 934 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, 935 SDValue(SLLI, 0), SDValue(Imm, 0)); 936 ReplaceNode(Node, MULHU); 937 return; 938 } 939 case ISD::INTRINSIC_WO_CHAIN: { 940 unsigned IntNo = Node->getConstantOperandVal(0); 941 switch (IntNo) { 942 // By default we do not custom select any intrinsic. 943 default: 944 break; 945 case Intrinsic::riscv_vmsgeu: 946 case Intrinsic::riscv_vmsge: { 947 SDValue Src1 = Node->getOperand(1); 948 SDValue Src2 = Node->getOperand(2); 949 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu; 950 bool IsCmpUnsignedZero = false; 951 // Only custom select scalar second operand. 952 if (Src2.getValueType() != XLenVT) 953 break; 954 // Small constants are handled with patterns. 955 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 956 int64_t CVal = C->getSExtValue(); 957 if (CVal >= -15 && CVal <= 16) { 958 if (!IsUnsigned || CVal != 0) 959 break; 960 IsCmpUnsignedZero = true; 961 } 962 } 963 MVT Src1VT = Src1.getSimpleValueType(); 964 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode; 965 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 966 default: 967 llvm_unreachable("Unexpected LMUL!"); 968 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 969 case RISCVII::VLMUL::lmulenum: \ 970 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 971 : RISCV::PseudoVMSLT_VX_##suffix; \ 972 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \ 973 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 974 break; 975 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1) 976 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2) 977 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4) 978 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8) 979 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16) 980 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32) 981 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64) 982 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES 983 } 984 SDValue SEW = CurDAG->getTargetConstant( 985 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 986 SDValue VL; 987 selectVLOp(Node->getOperand(3), VL); 988 989 // If vmsgeu with 0 immediate, expand it to vmset. 990 if (IsCmpUnsignedZero) { 991 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW)); 992 return; 993 } 994 995 // Expand to 996 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd 997 SDValue Cmp = SDValue( 998 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 999 0); 1000 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT, 1001 {Cmp, Cmp, VL, SEW})); 1002 return; 1003 } 1004 case Intrinsic::riscv_vmsgeu_mask: 1005 case Intrinsic::riscv_vmsge_mask: { 1006 SDValue Src1 = Node->getOperand(2); 1007 SDValue Src2 = Node->getOperand(3); 1008 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask; 1009 bool IsCmpUnsignedZero = false; 1010 // Only custom select scalar second operand. 1011 if (Src2.getValueType() != XLenVT) 1012 break; 1013 // Small constants are handled with patterns. 1014 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1015 int64_t CVal = C->getSExtValue(); 1016 if (CVal >= -15 && CVal <= 16) { 1017 if (!IsUnsigned || CVal != 0) 1018 break; 1019 IsCmpUnsignedZero = true; 1020 } 1021 } 1022 MVT Src1VT = Src1.getSimpleValueType(); 1023 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode, 1024 VMSetOpcode, VMANDOpcode; 1025 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1026 default: 1027 llvm_unreachable("Unexpected LMUL!"); 1028 #define CASE_VMSLT_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 1029 case RISCVII::VLMUL::lmulenum: \ 1030 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1031 : RISCV::PseudoVMSLT_VX_##suffix; \ 1032 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \ 1033 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \ 1034 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 1035 break; 1036 CASE_VMSLT_VMSET_OPCODES(LMUL_F8, MF8, B1) 1037 CASE_VMSLT_VMSET_OPCODES(LMUL_F4, MF4, B2) 1038 CASE_VMSLT_VMSET_OPCODES(LMUL_F2, MF2, B4) 1039 CASE_VMSLT_VMSET_OPCODES(LMUL_1, M1, B8) 1040 CASE_VMSLT_VMSET_OPCODES(LMUL_2, M2, B16) 1041 CASE_VMSLT_VMSET_OPCODES(LMUL_4, M4, B32) 1042 CASE_VMSLT_VMSET_OPCODES(LMUL_8, M8, B64) 1043 #undef CASE_VMSLT_VMSET_OPCODES 1044 } 1045 // Mask operations use the LMUL from the mask type. 1046 switch (RISCVTargetLowering::getLMUL(VT)) { 1047 default: 1048 llvm_unreachable("Unexpected LMUL!"); 1049 #define CASE_VMXOR_VMANDN_VMAND_OPCODES(lmulenum, suffix) \ 1050 case RISCVII::VLMUL::lmulenum: \ 1051 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \ 1052 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \ 1053 VMANDOpcode = RISCV::PseudoVMAND_MM_##suffix; \ 1054 break; 1055 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F8, MF8) 1056 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F4, MF4) 1057 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F2, MF2) 1058 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_1, M1) 1059 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_2, M2) 1060 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_4, M4) 1061 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_8, M8) 1062 #undef CASE_VMXOR_VMANDN_VMAND_OPCODES 1063 } 1064 SDValue SEW = CurDAG->getTargetConstant( 1065 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1066 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT); 1067 SDValue VL; 1068 selectVLOp(Node->getOperand(5), VL); 1069 SDValue MaskedOff = Node->getOperand(1); 1070 SDValue Mask = Node->getOperand(4); 1071 1072 // If vmsgeu_mask with 0 immediate, expand it to {vmset, vmand}. 1073 if (IsCmpUnsignedZero) { 1074 SDValue VMSet = 1075 SDValue(CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW), 0); 1076 ReplaceNode(Node, CurDAG->getMachineNode(VMANDOpcode, DL, VT, 1077 {Mask, VMSet, VL, MaskSEW})); 1078 return; 1079 } 1080 1081 // If the MaskedOff value and the Mask are the same value use 1082 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt 1083 // This avoids needing to copy v0 to vd before starting the next sequence. 1084 if (Mask == MaskedOff) { 1085 SDValue Cmp = SDValue( 1086 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1087 0); 1088 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT, 1089 {Mask, Cmp, VL, MaskSEW})); 1090 return; 1091 } 1092 1093 // Mask needs to be copied to V0. 1094 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, 1095 RISCV::V0, Mask, SDValue()); 1096 SDValue Glue = Chain.getValue(1); 1097 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); 1098 1099 // Otherwise use 1100 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0 1101 SDValue Cmp = SDValue( 1102 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, 1103 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), 1104 0); 1105 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, 1106 {Cmp, Mask, VL, MaskSEW})); 1107 return; 1108 } 1109 case Intrinsic::riscv_vsetvli_opt: 1110 case Intrinsic::riscv_vsetvlimax_opt: 1111 return selectVSETVLI(Node); 1112 } 1113 break; 1114 } 1115 case ISD::INTRINSIC_W_CHAIN: { 1116 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1117 switch (IntNo) { 1118 // By default we do not custom select any intrinsic. 1119 default: 1120 break; 1121 case Intrinsic::riscv_vsetvli: 1122 case Intrinsic::riscv_vsetvlimax: 1123 return selectVSETVLI(Node); 1124 case Intrinsic::riscv_vlseg2: 1125 case Intrinsic::riscv_vlseg3: 1126 case Intrinsic::riscv_vlseg4: 1127 case Intrinsic::riscv_vlseg5: 1128 case Intrinsic::riscv_vlseg6: 1129 case Intrinsic::riscv_vlseg7: 1130 case Intrinsic::riscv_vlseg8: { 1131 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1132 return; 1133 } 1134 case Intrinsic::riscv_vlseg2_mask: 1135 case Intrinsic::riscv_vlseg3_mask: 1136 case Intrinsic::riscv_vlseg4_mask: 1137 case Intrinsic::riscv_vlseg5_mask: 1138 case Intrinsic::riscv_vlseg6_mask: 1139 case Intrinsic::riscv_vlseg7_mask: 1140 case Intrinsic::riscv_vlseg8_mask: { 1141 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1142 return; 1143 } 1144 case Intrinsic::riscv_vlsseg2: 1145 case Intrinsic::riscv_vlsseg3: 1146 case Intrinsic::riscv_vlsseg4: 1147 case Intrinsic::riscv_vlsseg5: 1148 case Intrinsic::riscv_vlsseg6: 1149 case Intrinsic::riscv_vlsseg7: 1150 case Intrinsic::riscv_vlsseg8: { 1151 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1152 return; 1153 } 1154 case Intrinsic::riscv_vlsseg2_mask: 1155 case Intrinsic::riscv_vlsseg3_mask: 1156 case Intrinsic::riscv_vlsseg4_mask: 1157 case Intrinsic::riscv_vlsseg5_mask: 1158 case Intrinsic::riscv_vlsseg6_mask: 1159 case Intrinsic::riscv_vlsseg7_mask: 1160 case Intrinsic::riscv_vlsseg8_mask: { 1161 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1162 return; 1163 } 1164 case Intrinsic::riscv_vloxseg2: 1165 case Intrinsic::riscv_vloxseg3: 1166 case Intrinsic::riscv_vloxseg4: 1167 case Intrinsic::riscv_vloxseg5: 1168 case Intrinsic::riscv_vloxseg6: 1169 case Intrinsic::riscv_vloxseg7: 1170 case Intrinsic::riscv_vloxseg8: 1171 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1172 return; 1173 case Intrinsic::riscv_vluxseg2: 1174 case Intrinsic::riscv_vluxseg3: 1175 case Intrinsic::riscv_vluxseg4: 1176 case Intrinsic::riscv_vluxseg5: 1177 case Intrinsic::riscv_vluxseg6: 1178 case Intrinsic::riscv_vluxseg7: 1179 case Intrinsic::riscv_vluxseg8: 1180 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1181 return; 1182 case Intrinsic::riscv_vloxseg2_mask: 1183 case Intrinsic::riscv_vloxseg3_mask: 1184 case Intrinsic::riscv_vloxseg4_mask: 1185 case Intrinsic::riscv_vloxseg5_mask: 1186 case Intrinsic::riscv_vloxseg6_mask: 1187 case Intrinsic::riscv_vloxseg7_mask: 1188 case Intrinsic::riscv_vloxseg8_mask: 1189 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1190 return; 1191 case Intrinsic::riscv_vluxseg2_mask: 1192 case Intrinsic::riscv_vluxseg3_mask: 1193 case Intrinsic::riscv_vluxseg4_mask: 1194 case Intrinsic::riscv_vluxseg5_mask: 1195 case Intrinsic::riscv_vluxseg6_mask: 1196 case Intrinsic::riscv_vluxseg7_mask: 1197 case Intrinsic::riscv_vluxseg8_mask: 1198 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1199 return; 1200 case Intrinsic::riscv_vlseg8ff: 1201 case Intrinsic::riscv_vlseg7ff: 1202 case Intrinsic::riscv_vlseg6ff: 1203 case Intrinsic::riscv_vlseg5ff: 1204 case Intrinsic::riscv_vlseg4ff: 1205 case Intrinsic::riscv_vlseg3ff: 1206 case Intrinsic::riscv_vlseg2ff: { 1207 selectVLSEGFF(Node, /*IsMasked*/ false); 1208 return; 1209 } 1210 case Intrinsic::riscv_vlseg8ff_mask: 1211 case Intrinsic::riscv_vlseg7ff_mask: 1212 case Intrinsic::riscv_vlseg6ff_mask: 1213 case Intrinsic::riscv_vlseg5ff_mask: 1214 case Intrinsic::riscv_vlseg4ff_mask: 1215 case Intrinsic::riscv_vlseg3ff_mask: 1216 case Intrinsic::riscv_vlseg2ff_mask: { 1217 selectVLSEGFF(Node, /*IsMasked*/ true); 1218 return; 1219 } 1220 case Intrinsic::riscv_vloxei: 1221 case Intrinsic::riscv_vloxei_mask: 1222 case Intrinsic::riscv_vluxei: 1223 case Intrinsic::riscv_vluxei_mask: { 1224 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask || 1225 IntNo == Intrinsic::riscv_vluxei_mask; 1226 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei || 1227 IntNo == Intrinsic::riscv_vloxei_mask; 1228 1229 MVT VT = Node->getSimpleValueType(0); 1230 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1231 1232 unsigned CurOp = 2; 1233 // Masked intrinsic only have TU version pseduo instructions. 1234 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1235 SmallVector<SDValue, 8> Operands; 1236 if (IsTU) 1237 Operands.push_back(Node->getOperand(CurOp++)); 1238 else 1239 // Skip the undef passthru operand for nomask TA version pseudo 1240 CurOp++; 1241 1242 MVT IndexVT; 1243 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1244 /*IsStridedOrIndexed*/ true, Operands, 1245 /*IsLoad=*/true, &IndexVT); 1246 1247 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1248 "Element count mismatch"); 1249 1250 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1251 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1252 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1253 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1254 report_fatal_error("The V extension does not support EEW=64 for index " 1255 "values when XLEN=32"); 1256 } 1257 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( 1258 IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 1259 static_cast<unsigned>(IndexLMUL)); 1260 MachineSDNode *Load = 1261 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1262 1263 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1264 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1265 1266 ReplaceNode(Node, Load); 1267 return; 1268 } 1269 case Intrinsic::riscv_vlm: 1270 case Intrinsic::riscv_vle: 1271 case Intrinsic::riscv_vle_mask: 1272 case Intrinsic::riscv_vlse: 1273 case Intrinsic::riscv_vlse_mask: { 1274 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask || 1275 IntNo == Intrinsic::riscv_vlse_mask; 1276 bool IsStrided = 1277 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask; 1278 1279 MVT VT = Node->getSimpleValueType(0); 1280 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1281 1282 unsigned CurOp = 2; 1283 // The riscv_vlm intrinsic are always tail agnostic and no passthru operand. 1284 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm; 1285 // Masked intrinsic only have TU version pseduo instructions. 1286 bool IsTU = 1287 HasPassthruOperand && 1288 ((!IsMasked && !Node->getOperand(CurOp).isUndef()) || IsMasked); 1289 SmallVector<SDValue, 8> Operands; 1290 if (IsTU) 1291 Operands.push_back(Node->getOperand(CurOp++)); 1292 else if (HasPassthruOperand) 1293 // Skip the undef passthru operand for nomask TA version pseudo 1294 CurOp++; 1295 1296 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1297 Operands, /*IsLoad=*/true); 1298 1299 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1300 const RISCV::VLEPseudo *P = 1301 RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 1302 static_cast<unsigned>(LMUL)); 1303 MachineSDNode *Load = 1304 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1305 1306 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1307 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1308 1309 ReplaceNode(Node, Load); 1310 return; 1311 } 1312 case Intrinsic::riscv_vleff: 1313 case Intrinsic::riscv_vleff_mask: { 1314 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask; 1315 1316 MVT VT = Node->getSimpleValueType(0); 1317 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1318 1319 unsigned CurOp = 2; 1320 // Masked intrinsic only have TU version pseduo instructions. 1321 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1322 SmallVector<SDValue, 7> Operands; 1323 if (IsTU) 1324 Operands.push_back(Node->getOperand(CurOp++)); 1325 else 1326 // Skip the undef passthru operand for nomask TA version pseudo 1327 CurOp++; 1328 1329 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1330 /*IsStridedOrIndexed*/ false, Operands, 1331 /*IsLoad=*/true); 1332 1333 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1334 const RISCV::VLEPseudo *P = 1335 RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 1336 Log2SEW, static_cast<unsigned>(LMUL)); 1337 MachineSDNode *Load = 1338 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), 1339 MVT::Other, MVT::Glue, Operands); 1340 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, 1341 /*Glue*/ SDValue(Load, 2)); 1342 1343 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1344 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1345 1346 ReplaceUses(SDValue(Node, 0), SDValue(Load, 0)); 1347 ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL 1348 ReplaceUses(SDValue(Node, 2), SDValue(Load, 1)); // Chain 1349 CurDAG->RemoveDeadNode(Node); 1350 return; 1351 } 1352 } 1353 break; 1354 } 1355 case ISD::INTRINSIC_VOID: { 1356 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1357 switch (IntNo) { 1358 case Intrinsic::riscv_vsseg2: 1359 case Intrinsic::riscv_vsseg3: 1360 case Intrinsic::riscv_vsseg4: 1361 case Intrinsic::riscv_vsseg5: 1362 case Intrinsic::riscv_vsseg6: 1363 case Intrinsic::riscv_vsseg7: 1364 case Intrinsic::riscv_vsseg8: { 1365 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1366 return; 1367 } 1368 case Intrinsic::riscv_vsseg2_mask: 1369 case Intrinsic::riscv_vsseg3_mask: 1370 case Intrinsic::riscv_vsseg4_mask: 1371 case Intrinsic::riscv_vsseg5_mask: 1372 case Intrinsic::riscv_vsseg6_mask: 1373 case Intrinsic::riscv_vsseg7_mask: 1374 case Intrinsic::riscv_vsseg8_mask: { 1375 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1376 return; 1377 } 1378 case Intrinsic::riscv_vssseg2: 1379 case Intrinsic::riscv_vssseg3: 1380 case Intrinsic::riscv_vssseg4: 1381 case Intrinsic::riscv_vssseg5: 1382 case Intrinsic::riscv_vssseg6: 1383 case Intrinsic::riscv_vssseg7: 1384 case Intrinsic::riscv_vssseg8: { 1385 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1386 return; 1387 } 1388 case Intrinsic::riscv_vssseg2_mask: 1389 case Intrinsic::riscv_vssseg3_mask: 1390 case Intrinsic::riscv_vssseg4_mask: 1391 case Intrinsic::riscv_vssseg5_mask: 1392 case Intrinsic::riscv_vssseg6_mask: 1393 case Intrinsic::riscv_vssseg7_mask: 1394 case Intrinsic::riscv_vssseg8_mask: { 1395 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1396 return; 1397 } 1398 case Intrinsic::riscv_vsoxseg2: 1399 case Intrinsic::riscv_vsoxseg3: 1400 case Intrinsic::riscv_vsoxseg4: 1401 case Intrinsic::riscv_vsoxseg5: 1402 case Intrinsic::riscv_vsoxseg6: 1403 case Intrinsic::riscv_vsoxseg7: 1404 case Intrinsic::riscv_vsoxseg8: 1405 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1406 return; 1407 case Intrinsic::riscv_vsuxseg2: 1408 case Intrinsic::riscv_vsuxseg3: 1409 case Intrinsic::riscv_vsuxseg4: 1410 case Intrinsic::riscv_vsuxseg5: 1411 case Intrinsic::riscv_vsuxseg6: 1412 case Intrinsic::riscv_vsuxseg7: 1413 case Intrinsic::riscv_vsuxseg8: 1414 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1415 return; 1416 case Intrinsic::riscv_vsoxseg2_mask: 1417 case Intrinsic::riscv_vsoxseg3_mask: 1418 case Intrinsic::riscv_vsoxseg4_mask: 1419 case Intrinsic::riscv_vsoxseg5_mask: 1420 case Intrinsic::riscv_vsoxseg6_mask: 1421 case Intrinsic::riscv_vsoxseg7_mask: 1422 case Intrinsic::riscv_vsoxseg8_mask: 1423 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1424 return; 1425 case Intrinsic::riscv_vsuxseg2_mask: 1426 case Intrinsic::riscv_vsuxseg3_mask: 1427 case Intrinsic::riscv_vsuxseg4_mask: 1428 case Intrinsic::riscv_vsuxseg5_mask: 1429 case Intrinsic::riscv_vsuxseg6_mask: 1430 case Intrinsic::riscv_vsuxseg7_mask: 1431 case Intrinsic::riscv_vsuxseg8_mask: 1432 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1433 return; 1434 case Intrinsic::riscv_vsoxei: 1435 case Intrinsic::riscv_vsoxei_mask: 1436 case Intrinsic::riscv_vsuxei: 1437 case Intrinsic::riscv_vsuxei_mask: { 1438 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask || 1439 IntNo == Intrinsic::riscv_vsuxei_mask; 1440 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei || 1441 IntNo == Intrinsic::riscv_vsoxei_mask; 1442 1443 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1444 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1445 1446 unsigned CurOp = 2; 1447 SmallVector<SDValue, 8> Operands; 1448 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1449 1450 MVT IndexVT; 1451 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1452 /*IsStridedOrIndexed*/ true, Operands, 1453 /*IsLoad=*/false, &IndexVT); 1454 1455 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1456 "Element count mismatch"); 1457 1458 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1459 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1460 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1461 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1462 report_fatal_error("The V extension does not support EEW=64 for index " 1463 "values when XLEN=32"); 1464 } 1465 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( 1466 IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW, 1467 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL)); 1468 MachineSDNode *Store = 1469 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1470 1471 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1472 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1473 1474 ReplaceNode(Node, Store); 1475 return; 1476 } 1477 case Intrinsic::riscv_vsm: 1478 case Intrinsic::riscv_vse: 1479 case Intrinsic::riscv_vse_mask: 1480 case Intrinsic::riscv_vsse: 1481 case Intrinsic::riscv_vsse_mask: { 1482 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask || 1483 IntNo == Intrinsic::riscv_vsse_mask; 1484 bool IsStrided = 1485 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask; 1486 1487 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1488 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1489 1490 unsigned CurOp = 2; 1491 SmallVector<SDValue, 8> Operands; 1492 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1493 1494 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1495 Operands); 1496 1497 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1498 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( 1499 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 1500 MachineSDNode *Store = 1501 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1502 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1503 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1504 1505 ReplaceNode(Node, Store); 1506 return; 1507 } 1508 } 1509 break; 1510 } 1511 case ISD::BITCAST: { 1512 MVT SrcVT = Node->getOperand(0).getSimpleValueType(); 1513 // Just drop bitcasts between vectors if both are fixed or both are 1514 // scalable. 1515 if ((VT.isScalableVector() && SrcVT.isScalableVector()) || 1516 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) { 1517 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 1518 CurDAG->RemoveDeadNode(Node); 1519 return; 1520 } 1521 break; 1522 } 1523 case ISD::INSERT_SUBVECTOR: { 1524 SDValue V = Node->getOperand(0); 1525 SDValue SubV = Node->getOperand(1); 1526 SDLoc DL(SubV); 1527 auto Idx = Node->getConstantOperandVal(2); 1528 MVT SubVecVT = SubV.getSimpleValueType(); 1529 1530 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1531 MVT SubVecContainerVT = SubVecVT; 1532 // Establish the correct scalable-vector types for any fixed-length type. 1533 if (SubVecVT.isFixedLengthVector()) 1534 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT); 1535 if (VT.isFixedLengthVector()) 1536 VT = TLI.getContainerForFixedLengthVector(VT); 1537 1538 const auto *TRI = Subtarget->getRegisterInfo(); 1539 unsigned SubRegIdx; 1540 std::tie(SubRegIdx, Idx) = 1541 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1542 VT, SubVecContainerVT, Idx, TRI); 1543 1544 // If the Idx hasn't been completely eliminated then this is a subvector 1545 // insert which doesn't naturally align to a vector register. These must 1546 // be handled using instructions to manipulate the vector registers. 1547 if (Idx != 0) 1548 break; 1549 1550 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT); 1551 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 || 1552 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 || 1553 SubVecLMUL == RISCVII::VLMUL::LMUL_F8; 1554 (void)IsSubVecPartReg; // Silence unused variable warning without asserts. 1555 assert((!IsSubVecPartReg || V.isUndef()) && 1556 "Expecting lowering to have created legal INSERT_SUBVECTORs when " 1557 "the subvector is smaller than a full-sized register"); 1558 1559 // If we haven't set a SubRegIdx, then we must be going between 1560 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. 1561 if (SubRegIdx == RISCV::NoSubRegister) { 1562 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT); 1563 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1564 InRegClassID && 1565 "Unexpected subvector extraction"); 1566 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1567 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1568 DL, VT, SubV, RC); 1569 ReplaceNode(Node, NewNode); 1570 return; 1571 } 1572 1573 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV); 1574 ReplaceNode(Node, Insert.getNode()); 1575 return; 1576 } 1577 case ISD::EXTRACT_SUBVECTOR: { 1578 SDValue V = Node->getOperand(0); 1579 auto Idx = Node->getConstantOperandVal(1); 1580 MVT InVT = V.getSimpleValueType(); 1581 SDLoc DL(V); 1582 1583 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1584 MVT SubVecContainerVT = VT; 1585 // Establish the correct scalable-vector types for any fixed-length type. 1586 if (VT.isFixedLengthVector()) 1587 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT); 1588 if (InVT.isFixedLengthVector()) 1589 InVT = TLI.getContainerForFixedLengthVector(InVT); 1590 1591 const auto *TRI = Subtarget->getRegisterInfo(); 1592 unsigned SubRegIdx; 1593 std::tie(SubRegIdx, Idx) = 1594 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1595 InVT, SubVecContainerVT, Idx, TRI); 1596 1597 // If the Idx hasn't been completely eliminated then this is a subvector 1598 // extract which doesn't naturally align to a vector register. These must 1599 // be handled using instructions to manipulate the vector registers. 1600 if (Idx != 0) 1601 break; 1602 1603 // If we haven't set a SubRegIdx, then we must be going between 1604 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy. 1605 if (SubRegIdx == RISCV::NoSubRegister) { 1606 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT); 1607 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1608 InRegClassID && 1609 "Unexpected subvector extraction"); 1610 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1611 SDNode *NewNode = 1612 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC); 1613 ReplaceNode(Node, NewNode); 1614 return; 1615 } 1616 1617 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V); 1618 ReplaceNode(Node, Extract.getNode()); 1619 return; 1620 } 1621 case ISD::SPLAT_VECTOR: 1622 case RISCVISD::VMV_S_X_VL: 1623 case RISCVISD::VFMV_S_F_VL: 1624 case RISCVISD::VMV_V_X_VL: 1625 case RISCVISD::VFMV_V_F_VL: { 1626 // Try to match splat of a scalar load to a strided load with stride of x0. 1627 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL || 1628 Node->getOpcode() == RISCVISD::VFMV_S_F_VL; 1629 bool HasPassthruOperand = Node->getOpcode() != ISD::SPLAT_VECTOR; 1630 if (HasPassthruOperand && !IsScalarMove && !Node->getOperand(0).isUndef()) 1631 break; 1632 SDValue Src = HasPassthruOperand ? Node->getOperand(1) : Node->getOperand(0); 1633 auto *Ld = dyn_cast<LoadSDNode>(Src); 1634 if (!Ld) 1635 break; 1636 EVT MemVT = Ld->getMemoryVT(); 1637 // The memory VT should be the same size as the element type. 1638 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize()) 1639 break; 1640 if (!IsProfitableToFold(Src, Node, Node) || 1641 !IsLegalToFold(Src, Node, Node, TM.getOptLevel())) 1642 break; 1643 1644 SDValue VL; 1645 if (Node->getOpcode() == ISD::SPLAT_VECTOR) 1646 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); 1647 else if (IsScalarMove) { 1648 // We could deal with more VL if we update the VSETVLI insert pass to 1649 // avoid introducing more VSETVLI. 1650 if (!isOneConstant(Node->getOperand(2))) 1651 break; 1652 selectVLOp(Node->getOperand(2), VL); 1653 } else 1654 selectVLOp(Node->getOperand(2), VL); 1655 1656 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1657 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 1658 1659 SDValue Operands[] = {Ld->getBasePtr(), 1660 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW, 1661 Ld->getChain()}; 1662 1663 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1664 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo( 1665 /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false, 1666 Log2SEW, static_cast<unsigned>(LMUL)); 1667 MachineSDNode *Load = 1668 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1669 1670 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()}); 1671 1672 ReplaceNode(Node, Load); 1673 return; 1674 } 1675 } 1676 1677 // Select the default instruction. 1678 SelectCode(Node); 1679 } 1680 1681 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand( 1682 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 1683 switch (ConstraintID) { 1684 case InlineAsm::Constraint_m: 1685 // We just support simple memory operands that have a single address 1686 // operand and need no special handling. 1687 OutOps.push_back(Op); 1688 return false; 1689 case InlineAsm::Constraint_A: 1690 OutOps.push_back(Op); 1691 return false; 1692 default: 1693 break; 1694 } 1695 1696 return true; 1697 } 1698 1699 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) { 1700 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1701 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1702 return true; 1703 } 1704 return false; 1705 } 1706 1707 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) { 1708 // If this is FrameIndex, select it directly. Otherwise just let it get 1709 // selected to a register independently. 1710 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) 1711 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1712 else 1713 Base = Addr; 1714 return true; 1715 } 1716 1717 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, 1718 SDValue &ShAmt) { 1719 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift 1720 // amount. If there is an AND on the shift amount, we can bypass it if it 1721 // doesn't affect any of those bits. 1722 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 1723 const APInt &AndMask = N->getConstantOperandAPInt(1); 1724 1725 // Since the max shift amount is a power of 2 we can subtract 1 to make a 1726 // mask that covers the bits needed to represent all shift amounts. 1727 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!"); 1728 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); 1729 1730 if (ShMask.isSubsetOf(AndMask)) { 1731 ShAmt = N.getOperand(0); 1732 return true; 1733 } 1734 1735 // SimplifyDemandedBits may have optimized the mask so try restoring any 1736 // bits that are known zero. 1737 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0)); 1738 if (ShMask.isSubsetOf(AndMask | Known.Zero)) { 1739 ShAmt = N.getOperand(0); 1740 return true; 1741 } 1742 } else if (N.getOpcode() == ISD::SUB && 1743 isa<ConstantSDNode>(N.getOperand(0))) { 1744 uint64_t Imm = N.getConstantOperandVal(0); 1745 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to 1746 // generate a NEG instead of a SUB of a constant. 1747 if (Imm != 0 && Imm % ShiftWidth == 0) { 1748 SDLoc DL(N); 1749 EVT VT = N.getValueType(); 1750 SDValue Zero = 1751 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT); 1752 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB; 1753 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero, 1754 N.getOperand(1)); 1755 ShAmt = SDValue(Neg, 0); 1756 return true; 1757 } 1758 } 1759 1760 ShAmt = N; 1761 return true; 1762 } 1763 1764 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) { 1765 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG && 1766 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) { 1767 Val = N.getOperand(0); 1768 return true; 1769 } 1770 MVT VT = N.getSimpleValueType(); 1771 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) { 1772 Val = N; 1773 return true; 1774 } 1775 1776 return false; 1777 } 1778 1779 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) { 1780 if (N.getOpcode() == ISD::AND) { 1781 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1782 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) { 1783 Val = N.getOperand(0); 1784 return true; 1785 } 1786 } 1787 MVT VT = N.getSimpleValueType(); 1788 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32); 1789 if (CurDAG->MaskedValueIsZero(N, Mask)) { 1790 Val = N; 1791 return true; 1792 } 1793 1794 return false; 1795 } 1796 1797 // Return true if all users of this SDNode* only consume the lower \p Bits. 1798 // This can be used to form W instructions for add/sub/mul/shl even when the 1799 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if 1800 // SimplifyDemandedBits has made it so some users see a sext_inreg and some 1801 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave 1802 // the add/sub/mul/shl to become non-W instructions. By checking the users we 1803 // may be able to use a W instruction and CSE with the other instruction if 1804 // this has happened. We could try to detect that the CSE opportunity exists 1805 // before doing this, but that would be more complicated. 1806 // TODO: Does this need to look through AND/OR/XOR to their users to find more 1807 // opportunities. 1808 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const { 1809 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB || 1810 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL || 1811 Node->getOpcode() == ISD::SRL || 1812 Node->getOpcode() == ISD::SIGN_EXTEND_INREG || 1813 isa<ConstantSDNode>(Node)) && 1814 "Unexpected opcode"); 1815 1816 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 1817 SDNode *User = *UI; 1818 // Users of this node should have already been instruction selected 1819 if (!User->isMachineOpcode()) 1820 return false; 1821 1822 // TODO: Add more opcodes? 1823 switch (User->getMachineOpcode()) { 1824 default: 1825 return false; 1826 case RISCV::ADDW: 1827 case RISCV::ADDIW: 1828 case RISCV::SUBW: 1829 case RISCV::MULW: 1830 case RISCV::SLLW: 1831 case RISCV::SLLIW: 1832 case RISCV::SRAW: 1833 case RISCV::SRAIW: 1834 case RISCV::SRLW: 1835 case RISCV::SRLIW: 1836 case RISCV::DIVW: 1837 case RISCV::DIVUW: 1838 case RISCV::REMW: 1839 case RISCV::REMUW: 1840 case RISCV::ROLW: 1841 case RISCV::RORW: 1842 case RISCV::RORIW: 1843 case RISCV::CLZW: 1844 case RISCV::CTZW: 1845 case RISCV::CPOPW: 1846 case RISCV::SLLI_UW: 1847 case RISCV::FCVT_H_W: 1848 case RISCV::FCVT_H_WU: 1849 case RISCV::FCVT_S_W: 1850 case RISCV::FCVT_S_WU: 1851 case RISCV::FCVT_D_W: 1852 case RISCV::FCVT_D_WU: 1853 if (Bits < 32) 1854 return false; 1855 break; 1856 case RISCV::SLLI: 1857 // SLLI only uses the lower (XLen - ShAmt) bits. 1858 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1)) 1859 return false; 1860 break; 1861 case RISCV::ANDI: 1862 if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1)))) 1863 return false; 1864 break; 1865 case RISCV::SEXT_B: 1866 if (Bits < 8) 1867 return false; 1868 break; 1869 case RISCV::SEXT_H: 1870 case RISCV::ZEXT_H_RV32: 1871 case RISCV::ZEXT_H_RV64: 1872 if (Bits < 16) 1873 return false; 1874 break; 1875 case RISCV::ADD_UW: 1876 case RISCV::SH1ADD_UW: 1877 case RISCV::SH2ADD_UW: 1878 case RISCV::SH3ADD_UW: 1879 // The first operand to add.uw/shXadd.uw is implicitly zero extended from 1880 // 32 bits. 1881 if (UI.getOperandNo() != 0 || Bits < 32) 1882 return false; 1883 break; 1884 case RISCV::SB: 1885 if (UI.getOperandNo() != 0 || Bits < 8) 1886 return false; 1887 break; 1888 case RISCV::SH: 1889 if (UI.getOperandNo() != 0 || Bits < 16) 1890 return false; 1891 break; 1892 case RISCV::SW: 1893 if (UI.getOperandNo() != 0 || Bits < 32) 1894 return false; 1895 break; 1896 } 1897 } 1898 1899 return true; 1900 } 1901 1902 // Select VL as a 5 bit immediate or a value that will become a register. This 1903 // allows us to choose betwen VSETIVLI or VSETVLI later. 1904 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { 1905 auto *C = dyn_cast<ConstantSDNode>(N); 1906 if (C && isUInt<5>(C->getZExtValue())) { 1907 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N), 1908 N->getValueType(0)); 1909 } else if (C && C->isAllOnesValue()) { 1910 // Treat all ones as VLMax. 1911 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 1912 N->getValueType(0)); 1913 } else if (isa<RegisterSDNode>(N) && 1914 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) { 1915 // All our VL operands use an operand that allows GPRNoX0 or an immediate 1916 // as the register class. Convert X0 to a special immediate to pass the 1917 // MachineVerifier. This is recognized specially by the vsetvli insertion 1918 // pass. 1919 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 1920 N->getValueType(0)); 1921 } else { 1922 VL = N; 1923 } 1924 1925 return true; 1926 } 1927 1928 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { 1929 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef()) 1930 return false; 1931 SplatVal = N.getOperand(1); 1932 return true; 1933 } 1934 1935 using ValidateFn = bool (*)(int64_t); 1936 1937 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, 1938 SelectionDAG &DAG, 1939 const RISCVSubtarget &Subtarget, 1940 ValidateFn ValidateImm) { 1941 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 1942 !isa<ConstantSDNode>(N.getOperand(1))) 1943 return false; 1944 1945 int64_t SplatImm = 1946 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 1947 1948 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand 1949 // type is wider than the resulting vector element type: an implicit 1950 // truncation first takes place. Therefore, perform a manual 1951 // truncation/sign-extension in order to ignore any truncated bits and catch 1952 // any zero-extended immediate. 1953 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first 1954 // sign-extending to (XLenVT -1). 1955 MVT XLenVT = Subtarget.getXLenVT(); 1956 assert(XLenVT == N.getOperand(1).getSimpleValueType() && 1957 "Unexpected splat operand type"); 1958 MVT EltVT = N.getSimpleValueType().getVectorElementType(); 1959 if (EltVT.bitsLT(XLenVT)) 1960 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits()); 1961 1962 if (!ValidateImm(SplatImm)) 1963 return false; 1964 1965 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT); 1966 return true; 1967 } 1968 1969 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) { 1970 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget, 1971 [](int64_t Imm) { return isInt<5>(Imm); }); 1972 } 1973 1974 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) { 1975 return selectVSplatSimmHelper( 1976 N, SplatVal, *CurDAG, *Subtarget, 1977 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; }); 1978 } 1979 1980 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N, 1981 SDValue &SplatVal) { 1982 return selectVSplatSimmHelper( 1983 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) { 1984 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16); 1985 }); 1986 } 1987 1988 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) { 1989 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 1990 !isa<ConstantSDNode>(N.getOperand(1))) 1991 return false; 1992 1993 int64_t SplatImm = 1994 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 1995 1996 if (!isUInt<5>(SplatImm)) 1997 return false; 1998 1999 SplatVal = 2000 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT()); 2001 2002 return true; 2003 } 2004 2005 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width, 2006 SDValue &Imm) { 2007 if (auto *C = dyn_cast<ConstantSDNode>(N)) { 2008 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width); 2009 2010 if (!isInt<5>(ImmVal)) 2011 return false; 2012 2013 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT()); 2014 return true; 2015 } 2016 2017 return false; 2018 } 2019 2020 // Merge an ADDI into the offset of a load/store instruction where possible. 2021 // (load (addi base, off1), off2) -> (load base, off1+off2) 2022 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2) 2023 // This is possible when off1+off2 fits a 12-bit immediate. 2024 bool RISCVDAGToDAGISel::doPeepholeLoadStoreADDI(SDNode *N) { 2025 int OffsetOpIdx; 2026 int BaseOpIdx; 2027 2028 // Only attempt this optimisation for I-type loads and S-type stores. 2029 switch (N->getMachineOpcode()) { 2030 default: 2031 return false; 2032 case RISCV::LB: 2033 case RISCV::LH: 2034 case RISCV::LW: 2035 case RISCV::LBU: 2036 case RISCV::LHU: 2037 case RISCV::LWU: 2038 case RISCV::LD: 2039 case RISCV::FLH: 2040 case RISCV::FLW: 2041 case RISCV::FLD: 2042 BaseOpIdx = 0; 2043 OffsetOpIdx = 1; 2044 break; 2045 case RISCV::SB: 2046 case RISCV::SH: 2047 case RISCV::SW: 2048 case RISCV::SD: 2049 case RISCV::FSH: 2050 case RISCV::FSW: 2051 case RISCV::FSD: 2052 BaseOpIdx = 1; 2053 OffsetOpIdx = 2; 2054 break; 2055 } 2056 2057 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx))) 2058 return false; 2059 2060 SDValue Base = N->getOperand(BaseOpIdx); 2061 2062 // If the base is an ADDI, we can merge it in to the load/store. 2063 if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI) 2064 return false; 2065 2066 SDValue ImmOperand = Base.getOperand(1); 2067 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx); 2068 2069 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) { 2070 int64_t Offset1 = Const->getSExtValue(); 2071 int64_t CombinedOffset = Offset1 + Offset2; 2072 if (!isInt<12>(CombinedOffset)) 2073 return false; 2074 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand), 2075 ImmOperand.getValueType()); 2076 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) { 2077 // If the off1 in (addi base, off1) is a global variable's address (its 2078 // low part, really), then we can rely on the alignment of that variable 2079 // to provide a margin of safety before off1 can overflow the 12 bits. 2080 // Check if off2 falls within that margin; if so off1+off2 can't overflow. 2081 const DataLayout &DL = CurDAG->getDataLayout(); 2082 Align Alignment = GA->getGlobal()->getPointerAlignment(DL); 2083 if (Offset2 != 0 && Alignment <= Offset2) 2084 return false; 2085 int64_t Offset1 = GA->getOffset(); 2086 int64_t CombinedOffset = Offset1 + Offset2; 2087 ImmOperand = CurDAG->getTargetGlobalAddress( 2088 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(), 2089 CombinedOffset, GA->getTargetFlags()); 2090 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) { 2091 // Ditto. 2092 Align Alignment = CP->getAlign(); 2093 if (Offset2 != 0 && Alignment <= Offset2) 2094 return false; 2095 int64_t Offset1 = CP->getOffset(); 2096 int64_t CombinedOffset = Offset1 + Offset2; 2097 ImmOperand = CurDAG->getTargetConstantPool( 2098 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(), 2099 CombinedOffset, CP->getTargetFlags()); 2100 } else { 2101 return false; 2102 } 2103 2104 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 2105 LLVM_DEBUG(Base->dump(CurDAG)); 2106 LLVM_DEBUG(dbgs() << "\nN: "); 2107 LLVM_DEBUG(N->dump(CurDAG)); 2108 LLVM_DEBUG(dbgs() << "\n"); 2109 2110 // Modify the offset operand of the load/store. 2111 if (BaseOpIdx == 0) // Load 2112 CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand, 2113 N->getOperand(2)); 2114 else // Store 2115 CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0), 2116 ImmOperand, N->getOperand(3)); 2117 2118 return true; 2119 } 2120 2121 // Try to remove sext.w if the input is a W instruction or can be made into 2122 // a W instruction cheaply. 2123 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { 2124 // Look for the sext.w pattern, addiw rd, rs1, 0. 2125 if (N->getMachineOpcode() != RISCV::ADDIW || 2126 !isNullConstant(N->getOperand(1))) 2127 return false; 2128 2129 SDValue N0 = N->getOperand(0); 2130 if (!N0.isMachineOpcode()) 2131 return false; 2132 2133 switch (N0.getMachineOpcode()) { 2134 default: 2135 break; 2136 case RISCV::ADD: 2137 case RISCV::ADDI: 2138 case RISCV::SUB: 2139 case RISCV::MUL: 2140 case RISCV::SLLI: { 2141 // Convert sext.w+add/sub/mul to their W instructions. This will create 2142 // a new independent instruction. This improves latency. 2143 unsigned Opc; 2144 switch (N0.getMachineOpcode()) { 2145 default: 2146 llvm_unreachable("Unexpected opcode!"); 2147 case RISCV::ADD: Opc = RISCV::ADDW; break; 2148 case RISCV::ADDI: Opc = RISCV::ADDIW; break; 2149 case RISCV::SUB: Opc = RISCV::SUBW; break; 2150 case RISCV::MUL: Opc = RISCV::MULW; break; 2151 case RISCV::SLLI: Opc = RISCV::SLLIW; break; 2152 } 2153 2154 SDValue N00 = N0.getOperand(0); 2155 SDValue N01 = N0.getOperand(1); 2156 2157 // Shift amount needs to be uimm5. 2158 if (N0.getMachineOpcode() == RISCV::SLLI && 2159 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue())) 2160 break; 2161 2162 SDNode *Result = 2163 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), 2164 N00, N01); 2165 ReplaceUses(N, Result); 2166 return true; 2167 } 2168 case RISCV::ADDW: 2169 case RISCV::ADDIW: 2170 case RISCV::SUBW: 2171 case RISCV::MULW: 2172 case RISCV::SLLIW: 2173 // Result is already sign extended just remove the sext.w. 2174 // NOTE: We only handle the nodes that are selected with hasAllWUsers. 2175 ReplaceUses(N, N0.getNode()); 2176 return true; 2177 } 2178 2179 return false; 2180 } 2181 2182 // Optimize masked RVV pseudo instructions with a known all-ones mask to their 2183 // corresponding "unmasked" pseudo versions. The mask we're interested in will 2184 // take the form of a V0 physical register operand, with a glued 2185 // register-setting instruction. 2186 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) { 2187 const RISCV::RISCVMaskedPseudoInfo *I = 2188 RISCV::getMaskedPseudoInfo(N->getMachineOpcode()); 2189 if (!I) 2190 return false; 2191 2192 unsigned MaskOpIdx = I->MaskOpIdx; 2193 2194 // Check that we're using V0 as a mask register. 2195 if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) || 2196 cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0) 2197 return false; 2198 2199 // The glued user defines V0. 2200 const auto *Glued = N->getGluedNode(); 2201 2202 if (!Glued || Glued->getOpcode() != ISD::CopyToReg) 2203 return false; 2204 2205 // Check that we're defining V0 as a mask register. 2206 if (!isa<RegisterSDNode>(Glued->getOperand(1)) || 2207 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0) 2208 return false; 2209 2210 // Check the instruction defining V0; it needs to be a VMSET pseudo. 2211 SDValue MaskSetter = Glued->getOperand(2); 2212 2213 const auto IsVMSet = [](unsigned Opc) { 2214 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 || 2215 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 || 2216 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 || 2217 Opc == RISCV::PseudoVMSET_M_B8; 2218 }; 2219 2220 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has 2221 // undefined behaviour if it's the wrong bitwidth, so we could choose to 2222 // assume that it's all-ones? Same applies to its VL. 2223 if (!MaskSetter->isMachineOpcode() || !IsVMSet(MaskSetter.getMachineOpcode())) 2224 return false; 2225 2226 // Retrieve the tail policy operand index, if any. 2227 Optional<unsigned> TailPolicyOpIdx; 2228 const RISCVInstrInfo *TII = static_cast<const RISCVInstrInfo *>( 2229 CurDAG->getSubtarget().getInstrInfo()); 2230 2231 const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode()); 2232 2233 if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) { 2234 // The last operand of the pseudo is the policy op, but we're expecting a 2235 // Glue operand last. We may also have a chain. 2236 TailPolicyOpIdx = N->getNumOperands() - 1; 2237 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Glue) 2238 (*TailPolicyOpIdx)--; 2239 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Other) 2240 (*TailPolicyOpIdx)--; 2241 2242 // If the policy isn't TAIL_AGNOSTIC we can't perform this optimization. 2243 if (N->getConstantOperandVal(*TailPolicyOpIdx) != RISCVII::TAIL_AGNOSTIC) 2244 return false; 2245 } 2246 2247 const MCInstrDesc &UnmaskedMCID = TII->get(I->UnmaskedPseudo); 2248 2249 // Check that we're dropping the merge operand, the mask operand, and any 2250 // policy operand when we transform to this unmasked pseudo. 2251 assert(!RISCVII::hasMergeOp(UnmaskedMCID.TSFlags) && 2252 RISCVII::hasDummyMaskOp(UnmaskedMCID.TSFlags) && 2253 !RISCVII::hasVecPolicyOp(UnmaskedMCID.TSFlags) && 2254 "Unexpected pseudo to transform to"); 2255 (void)UnmaskedMCID; 2256 2257 SmallVector<SDValue, 8> Ops; 2258 // Skip the merge operand at index 0. 2259 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) { 2260 // Skip the mask, the policy, and the Glue. 2261 SDValue Op = N->getOperand(I); 2262 if (I == MaskOpIdx || I == TailPolicyOpIdx || 2263 Op.getValueType() == MVT::Glue) 2264 continue; 2265 Ops.push_back(Op); 2266 } 2267 2268 // Transitively apply any node glued to our new node. 2269 if (auto *TGlued = Glued->getGluedNode()) 2270 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1)); 2271 2272 SDNode *Result = 2273 CurDAG->getMachineNode(I->UnmaskedPseudo, SDLoc(N), N->getVTList(), Ops); 2274 ReplaceUses(N, Result); 2275 2276 return true; 2277 } 2278 2279 // This pass converts a legalized DAG into a RISCV-specific DAG, ready 2280 // for instruction scheduling. 2281 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) { 2282 return new RISCVDAGToDAGISel(TM); 2283 } 2284