1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the RISCV target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVISelDAGToDAG.h" 14 #include "MCTargetDesc/RISCVMCTargetDesc.h" 15 #include "MCTargetDesc/RISCVMatInt.h" 16 #include "RISCVISelLowering.h" 17 #include "RISCVMachineFunctionInfo.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/IR/IntrinsicsRISCV.h" 20 #include "llvm/Support/Alignment.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/KnownBits.h" 23 #include "llvm/Support/MathExtras.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 #define DEBUG_TYPE "riscv-isel" 29 30 namespace llvm { 31 namespace RISCV { 32 #define GET_RISCVVSSEGTable_IMPL 33 #define GET_RISCVVLSEGTable_IMPL 34 #define GET_RISCVVLXSEGTable_IMPL 35 #define GET_RISCVVSXSEGTable_IMPL 36 #define GET_RISCVVLETable_IMPL 37 #define GET_RISCVVSETable_IMPL 38 #define GET_RISCVVLXTable_IMPL 39 #define GET_RISCVVSXTable_IMPL 40 #include "RISCVGenSearchableTables.inc" 41 } // namespace RISCV 42 } // namespace llvm 43 44 void RISCVDAGToDAGISel::PreprocessISelDAG() { 45 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 46 E = CurDAG->allnodes_end(); 47 I != E;) { 48 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 49 50 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector 51 // load. Done after lowering and combining so that we have a chance to 52 // optimize this to VMV_V_X_VL when the upper bits aren't needed. 53 if (N->getOpcode() != RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) 54 continue; 55 56 assert(N->getNumOperands() == 3 && "Unexpected number of operands"); 57 MVT VT = N->getSimpleValueType(0); 58 SDValue Lo = N->getOperand(0); 59 SDValue Hi = N->getOperand(1); 60 SDValue VL = N->getOperand(2); 61 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() && 62 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 && 63 "Unexpected VTs!"); 64 MachineFunction &MF = CurDAG->getMachineFunction(); 65 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>(); 66 SDLoc DL(N); 67 68 // We use the same frame index we use for moving two i32s into 64-bit FPR. 69 // This is an analogous operation. 70 int FI = FuncInfo->getMoveF64FrameIndex(MF); 71 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); 72 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 73 SDValue StackSlot = 74 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout())); 75 76 SDValue Chain = CurDAG->getEntryNode(); 77 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8)); 78 79 SDValue OffsetSlot = 80 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL); 81 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4), 82 Align(8)); 83 84 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 85 86 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other}); 87 SDValue IntID = 88 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64); 89 SDValue Ops[] = {Chain, 90 IntID, 91 CurDAG->getUNDEF(VT), 92 StackSlot, 93 CurDAG->getRegister(RISCV::X0, MVT::i64), 94 VL}; 95 96 SDValue Result = CurDAG->getMemIntrinsicNode( 97 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MVT::i64, MPI, Align(8), 98 MachineMemOperand::MOLoad); 99 100 // We're about to replace all uses of the SPLAT_VECTOR_SPLIT_I64 with the 101 // vlse we created. This will cause general havok on the dag because 102 // anything below the conversion could be folded into other existing nodes. 103 // To avoid invalidating 'I', back it up to the convert node. 104 --I; 105 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 106 107 // Now that we did that, the node is dead. Increment the iterator to the 108 // next node to process, then delete N. 109 ++I; 110 CurDAG->DeleteNode(N); 111 } 112 } 113 114 void RISCVDAGToDAGISel::PostprocessISelDAG() { 115 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 116 117 bool MadeChange = false; 118 while (Position != CurDAG->allnodes_begin()) { 119 SDNode *N = &*--Position; 120 // Skip dead nodes and any non-machine opcodes. 121 if (N->use_empty() || !N->isMachineOpcode()) 122 continue; 123 124 MadeChange |= doPeepholeSExtW(N); 125 MadeChange |= doPeepholeLoadStoreADDI(N); 126 } 127 128 if (MadeChange) 129 CurDAG->RemoveDeadNodes(); 130 } 131 132 static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL, 133 const MVT VT, int64_t Imm, 134 const RISCVSubtarget &Subtarget) { 135 assert(VT == MVT::i64 && "Expecting MVT::i64"); 136 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering(); 137 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool( 138 ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT)); 139 SDValue Addr = TLI->getAddr(CP, *CurDAG); 140 SDValue Offset = CurDAG->getTargetConstant(0, DL, VT); 141 // Since there is no data race, the chain can be the entry node. 142 SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset, 143 CurDAG->getEntryNode()); 144 MachineFunction &MF = CurDAG->getMachineFunction(); 145 MachineMemOperand *MemOp = MF.getMachineMemOperand( 146 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 147 LLT(VT), CP->getAlign()); 148 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp}); 149 return Load; 150 } 151 152 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 153 int64_t Imm, const RISCVSubtarget &Subtarget) { 154 MVT XLenVT = Subtarget.getXLenVT(); 155 RISCVMatInt::InstSeq Seq = 156 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits()); 157 158 // If Imm is expensive to build, then we put it into constant pool. 159 if (Subtarget.useConstantPoolForLargeInts() && 160 Seq.size() > Subtarget.getMaxBuildIntsCost()) 161 return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget); 162 163 SDNode *Result = nullptr; 164 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT); 165 for (RISCVMatInt::Inst &Inst : Seq) { 166 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT); 167 if (Inst.Opc == RISCV::LUI) 168 Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm); 169 else if (Inst.Opc == RISCV::ADD_UW) 170 Result = CurDAG->getMachineNode(RISCV::ADD_UW, DL, XLenVT, SrcReg, 171 CurDAG->getRegister(RISCV::X0, XLenVT)); 172 else if (Inst.Opc == RISCV::SH1ADD || Inst.Opc == RISCV::SH2ADD || 173 Inst.Opc == RISCV::SH3ADD) 174 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SrcReg); 175 else 176 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm); 177 178 // Only the first instruction has X0 as its source. 179 SrcReg = SDValue(Result, 0); 180 } 181 182 return Result; 183 } 184 185 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 186 unsigned RegClassID, unsigned SubReg0) { 187 assert(Regs.size() >= 2 && Regs.size() <= 8); 188 189 SDLoc DL(Regs[0]); 190 SmallVector<SDValue, 8> Ops; 191 192 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32)); 193 194 for (unsigned I = 0; I < Regs.size(); ++I) { 195 Ops.push_back(Regs[I]); 196 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32)); 197 } 198 SDNode *N = 199 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 200 return SDValue(N, 0); 201 } 202 203 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 204 unsigned NF) { 205 static const unsigned RegClassIDs[] = { 206 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID, 207 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID, 208 RISCV::VRN8M1RegClassID}; 209 210 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0); 211 } 212 213 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 214 unsigned NF) { 215 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID, 216 RISCV::VRN3M2RegClassID, 217 RISCV::VRN4M2RegClassID}; 218 219 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0); 220 } 221 222 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 223 unsigned NF) { 224 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID, 225 RISCV::sub_vrm4_0); 226 } 227 228 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 229 unsigned NF, RISCVII::VLMUL LMUL) { 230 switch (LMUL) { 231 default: 232 llvm_unreachable("Invalid LMUL."); 233 case RISCVII::VLMUL::LMUL_F8: 234 case RISCVII::VLMUL::LMUL_F4: 235 case RISCVII::VLMUL::LMUL_F2: 236 case RISCVII::VLMUL::LMUL_1: 237 return createM1Tuple(CurDAG, Regs, NF); 238 case RISCVII::VLMUL::LMUL_2: 239 return createM2Tuple(CurDAG, Regs, NF); 240 case RISCVII::VLMUL::LMUL_4: 241 return createM4Tuple(CurDAG, Regs, NF); 242 } 243 } 244 245 void RISCVDAGToDAGISel::addVectorLoadStoreOperands( 246 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp, 247 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands, 248 bool IsLoad, MVT *IndexVT) { 249 SDValue Chain = Node->getOperand(0); 250 SDValue Glue; 251 252 SDValue Base; 253 SelectBaseAddr(Node->getOperand(CurOp++), Base); 254 Operands.push_back(Base); // Base pointer. 255 256 if (IsStridedOrIndexed) { 257 Operands.push_back(Node->getOperand(CurOp++)); // Index. 258 if (IndexVT) 259 *IndexVT = Operands.back()->getSimpleValueType(0); 260 } 261 262 if (IsMasked) { 263 // Mask needs to be copied to V0. 264 SDValue Mask = Node->getOperand(CurOp++); 265 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); 266 Glue = Chain.getValue(1); 267 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); 268 } 269 SDValue VL; 270 selectVLOp(Node->getOperand(CurOp++), VL); 271 Operands.push_back(VL); 272 273 MVT XLenVT = Subtarget->getXLenVT(); 274 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 275 Operands.push_back(SEWOp); 276 277 // Masked load has the tail policy argument. 278 if (IsMasked && IsLoad) { 279 // Policy must be a constant. 280 uint64_t Policy = Node->getConstantOperandVal(CurOp++); 281 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT); 282 Operands.push_back(PolicyOp); 283 } 284 285 Operands.push_back(Chain); // Chain. 286 if (Glue) 287 Operands.push_back(Glue); 288 } 289 290 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked, 291 bool IsStrided) { 292 SDLoc DL(Node); 293 unsigned NF = Node->getNumValues() - 1; 294 MVT VT = Node->getSimpleValueType(0); 295 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 296 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 297 298 unsigned CurOp = 2; 299 SmallVector<SDValue, 8> Operands; 300 if (IsMasked) { 301 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 302 Node->op_begin() + CurOp + NF); 303 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 304 Operands.push_back(MaskedOff); 305 CurOp += NF; 306 } 307 308 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 309 Operands, /*IsLoad=*/true); 310 311 const RISCV::VLSEGPseudo *P = 312 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW, 313 static_cast<unsigned>(LMUL)); 314 MachineSDNode *Load = 315 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 316 317 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 318 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 319 320 SDValue SuperReg = SDValue(Load, 0); 321 for (unsigned I = 0; I < NF; ++I) { 322 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 323 ReplaceUses(SDValue(Node, I), 324 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 325 } 326 327 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 328 CurDAG->RemoveDeadNode(Node); 329 } 330 331 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) { 332 SDLoc DL(Node); 333 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. 334 MVT VT = Node->getSimpleValueType(0); 335 MVT XLenVT = Subtarget->getXLenVT(); 336 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 337 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 338 339 unsigned CurOp = 2; 340 SmallVector<SDValue, 7> Operands; 341 if (IsMasked) { 342 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 343 Node->op_begin() + CurOp + NF); 344 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 345 Operands.push_back(MaskedOff); 346 CurOp += NF; 347 } 348 349 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 350 /*IsStridedOrIndexed*/ false, Operands, 351 /*IsLoad=*/true); 352 353 const RISCV::VLSEGPseudo *P = 354 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true, 355 Log2SEW, static_cast<unsigned>(LMUL)); 356 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, 357 MVT::Other, MVT::Glue, Operands); 358 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, 359 /*Glue*/ SDValue(Load, 2)); 360 361 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 362 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 363 364 SDValue SuperReg = SDValue(Load, 0); 365 for (unsigned I = 0; I < NF; ++I) { 366 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 367 ReplaceUses(SDValue(Node, I), 368 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 369 } 370 371 ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL 372 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain 373 CurDAG->RemoveDeadNode(Node); 374 } 375 376 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked, 377 bool IsOrdered) { 378 SDLoc DL(Node); 379 unsigned NF = Node->getNumValues() - 1; 380 MVT VT = Node->getSimpleValueType(0); 381 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 382 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 383 384 unsigned CurOp = 2; 385 SmallVector<SDValue, 8> Operands; 386 if (IsMasked) { 387 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 388 Node->op_begin() + CurOp + NF); 389 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 390 Operands.push_back(MaskedOff); 391 CurOp += NF; 392 } 393 394 MVT IndexVT; 395 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 396 /*IsStridedOrIndexed*/ true, Operands, 397 /*IsLoad=*/true, &IndexVT); 398 399 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 400 "Element count mismatch"); 401 402 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 403 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 404 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 405 report_fatal_error("The V extension does not support EEW=64 for index " 406 "values when XLEN=32"); 407 } 408 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo( 409 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 410 static_cast<unsigned>(IndexLMUL)); 411 MachineSDNode *Load = 412 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 413 414 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 415 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 416 417 SDValue SuperReg = SDValue(Load, 0); 418 for (unsigned I = 0; I < NF; ++I) { 419 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 420 ReplaceUses(SDValue(Node, I), 421 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 422 } 423 424 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 425 CurDAG->RemoveDeadNode(Node); 426 } 427 428 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked, 429 bool IsStrided) { 430 SDLoc DL(Node); 431 unsigned NF = Node->getNumOperands() - 4; 432 if (IsStrided) 433 NF--; 434 if (IsMasked) 435 NF--; 436 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 437 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 438 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 439 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 440 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 441 442 SmallVector<SDValue, 8> Operands; 443 Operands.push_back(StoreVal); 444 unsigned CurOp = 2 + NF; 445 446 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 447 Operands); 448 449 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo( 450 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 451 MachineSDNode *Store = 452 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 453 454 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 455 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 456 457 ReplaceNode(Node, Store); 458 } 459 460 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked, 461 bool IsOrdered) { 462 SDLoc DL(Node); 463 unsigned NF = Node->getNumOperands() - 5; 464 if (IsMasked) 465 --NF; 466 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 467 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 468 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 469 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 470 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 471 472 SmallVector<SDValue, 8> Operands; 473 Operands.push_back(StoreVal); 474 unsigned CurOp = 2 + NF; 475 476 MVT IndexVT; 477 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 478 /*IsStridedOrIndexed*/ true, Operands, 479 /*IsLoad=*/false, &IndexVT); 480 481 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 482 "Element count mismatch"); 483 484 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 485 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 486 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 487 report_fatal_error("The V extension does not support EEW=64 for index " 488 "values when XLEN=32"); 489 } 490 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo( 491 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 492 static_cast<unsigned>(IndexLMUL)); 493 MachineSDNode *Store = 494 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 495 496 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 497 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 498 499 ReplaceNode(Node, Store); 500 } 501 502 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) { 503 if (!Subtarget->hasVInstructions()) 504 return; 505 506 assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN || 507 Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) && 508 "Unexpected opcode"); 509 510 SDLoc DL(Node); 511 MVT XLenVT = Subtarget->getXLenVT(); 512 513 bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN; 514 unsigned IntNoOffset = HasChain ? 1 : 0; 515 unsigned IntNo = Node->getConstantOperandVal(IntNoOffset); 516 517 assert((IntNo == Intrinsic::riscv_vsetvli || 518 IntNo == Intrinsic::riscv_vsetvlimax || 519 IntNo == Intrinsic::riscv_vsetvli_opt || 520 IntNo == Intrinsic::riscv_vsetvlimax_opt) && 521 "Unexpected vsetvli intrinsic"); 522 523 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax || 524 IntNo == Intrinsic::riscv_vsetvlimax_opt; 525 unsigned Offset = IntNoOffset + (VLMax ? 1 : 2); 526 527 assert(Node->getNumOperands() == Offset + 2 && 528 "Unexpected number of operands"); 529 530 unsigned SEW = 531 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7); 532 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>( 533 Node->getConstantOperandVal(Offset + 1) & 0x7); 534 535 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true, 536 /*MaskAgnostic*/ false); 537 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT); 538 539 SmallVector<EVT, 2> VTs = {XLenVT}; 540 if (HasChain) 541 VTs.push_back(MVT::Other); 542 543 SDValue VLOperand; 544 unsigned Opcode = RISCV::PseudoVSETVLI; 545 if (VLMax) { 546 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT); 547 Opcode = RISCV::PseudoVSETVLIX0; 548 } else { 549 VLOperand = Node->getOperand(IntNoOffset + 1); 550 551 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { 552 uint64_t AVL = C->getZExtValue(); 553 if (isUInt<5>(AVL)) { 554 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); 555 SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp}; 556 if (HasChain) 557 Ops.push_back(Node->getOperand(0)); 558 ReplaceNode( 559 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops)); 560 return; 561 } 562 } 563 } 564 565 SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp}; 566 if (HasChain) 567 Ops.push_back(Node->getOperand(0)); 568 569 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops)); 570 } 571 572 void RISCVDAGToDAGISel::Select(SDNode *Node) { 573 // If we have a custom node, we have already selected. 574 if (Node->isMachineOpcode()) { 575 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 576 Node->setNodeId(-1); 577 return; 578 } 579 580 // Instruction Selection not handled by the auto-generated tablegen selection 581 // should be handled here. 582 unsigned Opcode = Node->getOpcode(); 583 MVT XLenVT = Subtarget->getXLenVT(); 584 SDLoc DL(Node); 585 MVT VT = Node->getSimpleValueType(0); 586 587 switch (Opcode) { 588 case ISD::Constant: { 589 auto *ConstNode = cast<ConstantSDNode>(Node); 590 if (VT == XLenVT && ConstNode->isZero()) { 591 SDValue New = 592 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT); 593 ReplaceNode(Node, New.getNode()); 594 return; 595 } 596 int64_t Imm = ConstNode->getSExtValue(); 597 // If the upper XLen-16 bits are not used, try to convert this to a simm12 598 // by sign extending bit 15. 599 if (isUInt<16>(Imm) && isInt<12>(SignExtend64(Imm, 16)) && 600 hasAllHUsers(Node)) 601 Imm = SignExtend64(Imm, 16); 602 // If the upper 32-bits are not used try to convert this into a simm32 by 603 // sign extending bit 32. 604 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node)) 605 Imm = SignExtend64(Imm, 32); 606 607 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget)); 608 return; 609 } 610 case ISD::FrameIndex: { 611 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT); 612 int FI = cast<FrameIndexSDNode>(Node)->getIndex(); 613 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT); 614 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm)); 615 return; 616 } 617 case ISD::SRL: { 618 // Optimize (srl (and X, C2), C) -> 619 // (srli (slli X, (XLen-C3), (XLen-C3) + C) 620 // Where C2 is a mask with C3 trailing ones. 621 // Taking into account that the C2 may have had lower bits unset by 622 // SimplifyDemandedBits. This avoids materializing the C2 immediate. 623 // This pattern occurs when type legalizing right shifts for types with 624 // less than XLen bits. 625 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 626 if (!N1C) 627 break; 628 SDValue N0 = Node->getOperand(0); 629 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 630 !isa<ConstantSDNode>(N0.getOperand(1))) 631 break; 632 unsigned ShAmt = N1C->getZExtValue(); 633 uint64_t Mask = N0.getConstantOperandVal(1); 634 Mask |= maskTrailingOnes<uint64_t>(ShAmt); 635 if (!isMask_64(Mask)) 636 break; 637 unsigned TrailingOnes = countTrailingOnes(Mask); 638 // 32 trailing ones should use srliw via tablegen pattern. 639 if (TrailingOnes == 32 || ShAmt >= TrailingOnes) 640 break; 641 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes; 642 SDNode *SLLI = 643 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 644 CurDAG->getTargetConstant(LShAmt, DL, VT)); 645 SDNode *SRLI = CurDAG->getMachineNode( 646 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 647 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 648 ReplaceNode(Node, SRLI); 649 return; 650 } 651 case ISD::SRA: { 652 // Optimize (sra (sext_inreg X, i16), C) -> 653 // (srai (slli X, (XLen-16), (XLen-16) + C) 654 // And (sra (sext_inreg X, i8), C) -> 655 // (srai (slli X, (XLen-8), (XLen-8) + C) 656 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal. 657 // This transform matches the code we get without Zbb. The shifts are more 658 // compressible, and this can help expose CSE opportunities in the sdiv by 659 // constant optimization. 660 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 661 if (!N1C) 662 break; 663 SDValue N0 = Node->getOperand(0); 664 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse()) 665 break; 666 unsigned ShAmt = N1C->getZExtValue(); 667 unsigned ExtSize = 668 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits(); 669 // ExtSize of 32 should use sraiw via tablegen pattern. 670 if (ExtSize >= 32 || ShAmt >= ExtSize) 671 break; 672 unsigned LShAmt = Subtarget->getXLen() - ExtSize; 673 SDNode *SLLI = 674 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 675 CurDAG->getTargetConstant(LShAmt, DL, VT)); 676 SDNode *SRAI = CurDAG->getMachineNode( 677 RISCV::SRAI, DL, VT, SDValue(SLLI, 0), 678 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 679 ReplaceNode(Node, SRAI); 680 return; 681 } 682 case ISD::AND: { 683 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 684 if (!N1C) 685 break; 686 687 SDValue N0 = Node->getOperand(0); 688 689 bool LeftShift = N0.getOpcode() == ISD::SHL; 690 if (!LeftShift && N0.getOpcode() != ISD::SRL) 691 break; 692 693 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 694 if (!C) 695 break; 696 uint64_t C2 = C->getZExtValue(); 697 unsigned XLen = Subtarget->getXLen(); 698 if (!C2 || C2 >= XLen) 699 break; 700 701 uint64_t C1 = N1C->getZExtValue(); 702 703 // Keep track of whether this is an andi. 704 bool IsANDI = isInt<12>(N1C->getSExtValue()); 705 706 // Clear irrelevant bits in the mask. 707 if (LeftShift) 708 C1 &= maskTrailingZeros<uint64_t>(C2); 709 else 710 C1 &= maskTrailingOnes<uint64_t>(XLen - C2); 711 712 // Some transforms should only be done if the shift has a single use or 713 // the AND would become (srli (slli X, 32), 32) 714 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF); 715 716 SDValue X = N0.getOperand(0); 717 718 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask 719 // with c3 leading zeros. 720 if (!LeftShift && isMask_64(C1)) { 721 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 722 if (C2 < C3) { 723 // If the number of leading zeros is C2+32 this can be SRLIW. 724 if (C2 + 32 == C3) { 725 SDNode *SRLIW = 726 CurDAG->getMachineNode(RISCV::SRLIW, DL, XLenVT, X, 727 CurDAG->getTargetConstant(C2, DL, XLenVT)); 728 ReplaceNode(Node, SRLIW); 729 return; 730 } 731 732 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if 733 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1. 734 // 735 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type 736 // legalized and goes through DAG combine. 737 if (C2 >= 32 && (C3 - C2) == 1 && N0.hasOneUse() && 738 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 739 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) { 740 SDNode *SRAIW = 741 CurDAG->getMachineNode(RISCV::SRAIW, DL, XLenVT, X.getOperand(0), 742 CurDAG->getTargetConstant(31, DL, XLenVT)); 743 SDNode *SRLIW = CurDAG->getMachineNode( 744 RISCV::SRLIW, DL, XLenVT, SDValue(SRAIW, 0), 745 CurDAG->getTargetConstant(C3 - 32, DL, XLenVT)); 746 ReplaceNode(Node, SRLIW); 747 return; 748 } 749 750 // (srli (slli x, c3-c2), c3). 751 if (OneUseOrZExtW && !IsANDI) { 752 SDNode *SLLI = CurDAG->getMachineNode( 753 RISCV::SLLI, DL, XLenVT, X, 754 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 755 SDNode *SRLI = 756 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 757 CurDAG->getTargetConstant(C3, DL, XLenVT)); 758 ReplaceNode(Node, SRLI); 759 return; 760 } 761 } 762 } 763 764 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask 765 // shifted by c2 bits with c3 leading zeros. 766 if (LeftShift && isShiftedMask_64(C1)) { 767 uint64_t C3 = XLen - (64 - countLeadingZeros(C1)); 768 769 if (C2 + C3 < XLen && 770 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + C3)) << C2)) { 771 // Use slli.uw when possible. 772 if ((XLen - (C2 + C3)) == 32 && Subtarget->hasStdExtZba()) { 773 SDNode *SLLI_UW = 774 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, XLenVT, X, 775 CurDAG->getTargetConstant(C2, DL, XLenVT)); 776 ReplaceNode(Node, SLLI_UW); 777 return; 778 } 779 780 // (srli (slli c2+c3), c3) 781 if (OneUseOrZExtW && !IsANDI) { 782 SDNode *SLLI = CurDAG->getMachineNode( 783 RISCV::SLLI, DL, XLenVT, X, 784 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 785 SDNode *SRLI = 786 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0), 787 CurDAG->getTargetConstant(C3, DL, XLenVT)); 788 ReplaceNode(Node, SRLI); 789 return; 790 } 791 } 792 } 793 794 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a 795 // shifted mask with c2 leading zeros and c3 trailing zeros. 796 if (!LeftShift && isShiftedMask_64(C1)) { 797 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 798 uint64_t C3 = countTrailingZeros(C1); 799 if (Leading == C2 && C2 + C3 < XLen && OneUseOrZExtW && !IsANDI) { 800 SDNode *SRLI = CurDAG->getMachineNode( 801 RISCV::SRLI, DL, XLenVT, X, 802 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 803 SDNode *SLLI = 804 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 805 CurDAG->getTargetConstant(C3, DL, XLenVT)); 806 ReplaceNode(Node, SLLI); 807 return; 808 } 809 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI. 810 if (Leading > 32 && (Leading - 32) == C2 && C2 + C3 < 32 && 811 OneUseOrZExtW && !IsANDI) { 812 SDNode *SRLIW = CurDAG->getMachineNode( 813 RISCV::SRLIW, DL, XLenVT, X, 814 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT)); 815 SDNode *SLLI = 816 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 817 CurDAG->getTargetConstant(C3, DL, XLenVT)); 818 ReplaceNode(Node, SLLI); 819 return; 820 } 821 } 822 823 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a 824 // shifted mask with no leading zeros and c3 trailing zeros. 825 if (LeftShift && isShiftedMask_64(C1)) { 826 uint64_t Leading = XLen - (64 - countLeadingZeros(C1)); 827 uint64_t C3 = countTrailingZeros(C1); 828 if (Leading == 0 && C2 < C3 && OneUseOrZExtW && !IsANDI) { 829 SDNode *SRLI = CurDAG->getMachineNode( 830 RISCV::SRLI, DL, XLenVT, X, 831 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 832 SDNode *SLLI = 833 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLI, 0), 834 CurDAG->getTargetConstant(C3, DL, XLenVT)); 835 ReplaceNode(Node, SLLI); 836 return; 837 } 838 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI. 839 if (C2 < C3 && Leading + C2 == 32 && OneUseOrZExtW && !IsANDI) { 840 SDNode *SRLIW = CurDAG->getMachineNode( 841 RISCV::SRLIW, DL, XLenVT, X, 842 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT)); 843 SDNode *SLLI = 844 CurDAG->getMachineNode(RISCV::SLLI, DL, XLenVT, SDValue(SRLIW, 0), 845 CurDAG->getTargetConstant(C3, DL, XLenVT)); 846 ReplaceNode(Node, SLLI); 847 return; 848 } 849 } 850 851 break; 852 } 853 case ISD::MUL: { 854 // Special case for calculating (mul (and X, C2), C1) where the full product 855 // fits in XLen bits. We can shift X left by the number of leading zeros in 856 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final 857 // product has XLen trailing zeros, putting it in the output of MULHU. This 858 // can avoid materializing a constant in a register for C2. 859 860 // RHS should be a constant. 861 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 862 if (!N1C || !N1C->hasOneUse()) 863 break; 864 865 // LHS should be an AND with constant. 866 SDValue N0 = Node->getOperand(0); 867 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) 868 break; 869 870 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 871 872 // Constant should be a mask. 873 if (!isMask_64(C2)) 874 break; 875 876 // This should be the only use of the AND unless we will use 877 // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND 878 // constants. 879 if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) 880 break; 881 882 // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this 883 // optimization. 884 if (isInt<12>(C2) || 885 (C2 == UINT64_C(0xFFFF) && 886 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || 887 (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) 888 break; 889 890 // We need to shift left the AND input and C1 by a total of XLen bits. 891 892 // How far left do we need to shift the AND input? 893 unsigned XLen = Subtarget->getXLen(); 894 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); 895 896 // The constant gets shifted by the remaining amount unless that would 897 // shift bits out. 898 uint64_t C1 = N1C->getZExtValue(); 899 unsigned ConstantShift = XLen - LeadingZeros; 900 if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) 901 break; 902 903 uint64_t ShiftedC1 = C1 << ConstantShift; 904 // If this RV32, we need to sign extend the constant. 905 if (XLen == 32) 906 ShiftedC1 = SignExtend64(ShiftedC1, 32); 907 908 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). 909 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget); 910 SDNode *SLLI = 911 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), 912 CurDAG->getTargetConstant(LeadingZeros, DL, VT)); 913 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, 914 SDValue(SLLI, 0), SDValue(Imm, 0)); 915 ReplaceNode(Node, MULHU); 916 return; 917 } 918 case ISD::INTRINSIC_WO_CHAIN: { 919 unsigned IntNo = Node->getConstantOperandVal(0); 920 switch (IntNo) { 921 // By default we do not custom select any intrinsic. 922 default: 923 break; 924 case Intrinsic::riscv_vmsgeu: 925 case Intrinsic::riscv_vmsge: { 926 SDValue Src1 = Node->getOperand(1); 927 SDValue Src2 = Node->getOperand(2); 928 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu; 929 bool IsCmpUnsignedZero = false; 930 // Only custom select scalar second operand. 931 if (Src2.getValueType() != XLenVT) 932 break; 933 // Small constants are handled with patterns. 934 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 935 int64_t CVal = C->getSExtValue(); 936 if (CVal >= -15 && CVal <= 16) { 937 if (!IsUnsigned || CVal != 0) 938 break; 939 IsCmpUnsignedZero = true; 940 } 941 } 942 MVT Src1VT = Src1.getSimpleValueType(); 943 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode; 944 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 945 default: 946 llvm_unreachable("Unexpected LMUL!"); 947 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 948 case RISCVII::VLMUL::lmulenum: \ 949 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 950 : RISCV::PseudoVMSLT_VX_##suffix; \ 951 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \ 952 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 953 break; 954 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1) 955 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2) 956 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4) 957 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8) 958 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16) 959 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32) 960 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64) 961 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES 962 } 963 SDValue SEW = CurDAG->getTargetConstant( 964 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 965 SDValue VL; 966 selectVLOp(Node->getOperand(3), VL); 967 968 // If vmsgeu with 0 immediate, expand it to vmset. 969 if (IsCmpUnsignedZero) { 970 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW)); 971 return; 972 } 973 974 // Expand to 975 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd 976 SDValue Cmp = SDValue( 977 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 978 0); 979 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT, 980 {Cmp, Cmp, VL, SEW})); 981 return; 982 } 983 case Intrinsic::riscv_vmsgeu_mask: 984 case Intrinsic::riscv_vmsge_mask: { 985 SDValue Src1 = Node->getOperand(2); 986 SDValue Src2 = Node->getOperand(3); 987 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask; 988 bool IsCmpUnsignedZero = false; 989 // Only custom select scalar second operand. 990 if (Src2.getValueType() != XLenVT) 991 break; 992 // Small constants are handled with patterns. 993 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 994 int64_t CVal = C->getSExtValue(); 995 if (CVal >= -15 && CVal <= 16) { 996 if (!IsUnsigned || CVal != 0) 997 break; 998 IsCmpUnsignedZero = true; 999 } 1000 } 1001 MVT Src1VT = Src1.getSimpleValueType(); 1002 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode, 1003 VMSetOpcode, VMANDOpcode; 1004 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1005 default: 1006 llvm_unreachable("Unexpected LMUL!"); 1007 #define CASE_VMSLT_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 1008 case RISCVII::VLMUL::lmulenum: \ 1009 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1010 : RISCV::PseudoVMSLT_VX_##suffix; \ 1011 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \ 1012 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \ 1013 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 1014 break; 1015 CASE_VMSLT_VMSET_OPCODES(LMUL_F8, MF8, B1) 1016 CASE_VMSLT_VMSET_OPCODES(LMUL_F4, MF4, B2) 1017 CASE_VMSLT_VMSET_OPCODES(LMUL_F2, MF2, B4) 1018 CASE_VMSLT_VMSET_OPCODES(LMUL_1, M1, B8) 1019 CASE_VMSLT_VMSET_OPCODES(LMUL_2, M2, B16) 1020 CASE_VMSLT_VMSET_OPCODES(LMUL_4, M4, B32) 1021 CASE_VMSLT_VMSET_OPCODES(LMUL_8, M8, B64) 1022 #undef CASE_VMSLT_VMSET_OPCODES 1023 } 1024 // Mask operations use the LMUL from the mask type. 1025 switch (RISCVTargetLowering::getLMUL(VT)) { 1026 default: 1027 llvm_unreachable("Unexpected LMUL!"); 1028 #define CASE_VMXOR_VMANDN_VMAND_OPCODES(lmulenum, suffix) \ 1029 case RISCVII::VLMUL::lmulenum: \ 1030 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \ 1031 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \ 1032 VMANDOpcode = RISCV::PseudoVMAND_MM_##suffix; \ 1033 break; 1034 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F8, MF8) 1035 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F4, MF4) 1036 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_F2, MF2) 1037 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_1, M1) 1038 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_2, M2) 1039 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_4, M4) 1040 CASE_VMXOR_VMANDN_VMAND_OPCODES(LMUL_8, M8) 1041 #undef CASE_VMXOR_VMANDN_VMAND_OPCODES 1042 } 1043 SDValue SEW = CurDAG->getTargetConstant( 1044 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1045 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT); 1046 SDValue VL; 1047 selectVLOp(Node->getOperand(5), VL); 1048 SDValue MaskedOff = Node->getOperand(1); 1049 SDValue Mask = Node->getOperand(4); 1050 1051 // If vmsgeu_mask with 0 immediate, expand it to {vmset, vmand}. 1052 if (IsCmpUnsignedZero) { 1053 SDValue VMSet = 1054 SDValue(CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW), 0); 1055 ReplaceNode(Node, CurDAG->getMachineNode(VMANDOpcode, DL, VT, 1056 {Mask, VMSet, VL, MaskSEW})); 1057 return; 1058 } 1059 1060 // If the MaskedOff value and the Mask are the same value use 1061 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt 1062 // This avoids needing to copy v0 to vd before starting the next sequence. 1063 if (Mask == MaskedOff) { 1064 SDValue Cmp = SDValue( 1065 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1066 0); 1067 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT, 1068 {Mask, Cmp, VL, MaskSEW})); 1069 return; 1070 } 1071 1072 // Mask needs to be copied to V0. 1073 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, 1074 RISCV::V0, Mask, SDValue()); 1075 SDValue Glue = Chain.getValue(1); 1076 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); 1077 1078 // Otherwise use 1079 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0 1080 SDValue Cmp = SDValue( 1081 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, 1082 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), 1083 0); 1084 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, 1085 {Cmp, Mask, VL, MaskSEW})); 1086 return; 1087 } 1088 case Intrinsic::riscv_vsetvli_opt: 1089 case Intrinsic::riscv_vsetvlimax_opt: 1090 return selectVSETVLI(Node); 1091 } 1092 break; 1093 } 1094 case ISD::INTRINSIC_W_CHAIN: { 1095 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1096 switch (IntNo) { 1097 // By default we do not custom select any intrinsic. 1098 default: 1099 break; 1100 case Intrinsic::riscv_vsetvli: 1101 case Intrinsic::riscv_vsetvlimax: 1102 return selectVSETVLI(Node); 1103 case Intrinsic::riscv_vlseg2: 1104 case Intrinsic::riscv_vlseg3: 1105 case Intrinsic::riscv_vlseg4: 1106 case Intrinsic::riscv_vlseg5: 1107 case Intrinsic::riscv_vlseg6: 1108 case Intrinsic::riscv_vlseg7: 1109 case Intrinsic::riscv_vlseg8: { 1110 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1111 return; 1112 } 1113 case Intrinsic::riscv_vlseg2_mask: 1114 case Intrinsic::riscv_vlseg3_mask: 1115 case Intrinsic::riscv_vlseg4_mask: 1116 case Intrinsic::riscv_vlseg5_mask: 1117 case Intrinsic::riscv_vlseg6_mask: 1118 case Intrinsic::riscv_vlseg7_mask: 1119 case Intrinsic::riscv_vlseg8_mask: { 1120 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1121 return; 1122 } 1123 case Intrinsic::riscv_vlsseg2: 1124 case Intrinsic::riscv_vlsseg3: 1125 case Intrinsic::riscv_vlsseg4: 1126 case Intrinsic::riscv_vlsseg5: 1127 case Intrinsic::riscv_vlsseg6: 1128 case Intrinsic::riscv_vlsseg7: 1129 case Intrinsic::riscv_vlsseg8: { 1130 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1131 return; 1132 } 1133 case Intrinsic::riscv_vlsseg2_mask: 1134 case Intrinsic::riscv_vlsseg3_mask: 1135 case Intrinsic::riscv_vlsseg4_mask: 1136 case Intrinsic::riscv_vlsseg5_mask: 1137 case Intrinsic::riscv_vlsseg6_mask: 1138 case Intrinsic::riscv_vlsseg7_mask: 1139 case Intrinsic::riscv_vlsseg8_mask: { 1140 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1141 return; 1142 } 1143 case Intrinsic::riscv_vloxseg2: 1144 case Intrinsic::riscv_vloxseg3: 1145 case Intrinsic::riscv_vloxseg4: 1146 case Intrinsic::riscv_vloxseg5: 1147 case Intrinsic::riscv_vloxseg6: 1148 case Intrinsic::riscv_vloxseg7: 1149 case Intrinsic::riscv_vloxseg8: 1150 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1151 return; 1152 case Intrinsic::riscv_vluxseg2: 1153 case Intrinsic::riscv_vluxseg3: 1154 case Intrinsic::riscv_vluxseg4: 1155 case Intrinsic::riscv_vluxseg5: 1156 case Intrinsic::riscv_vluxseg6: 1157 case Intrinsic::riscv_vluxseg7: 1158 case Intrinsic::riscv_vluxseg8: 1159 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1160 return; 1161 case Intrinsic::riscv_vloxseg2_mask: 1162 case Intrinsic::riscv_vloxseg3_mask: 1163 case Intrinsic::riscv_vloxseg4_mask: 1164 case Intrinsic::riscv_vloxseg5_mask: 1165 case Intrinsic::riscv_vloxseg6_mask: 1166 case Intrinsic::riscv_vloxseg7_mask: 1167 case Intrinsic::riscv_vloxseg8_mask: 1168 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1169 return; 1170 case Intrinsic::riscv_vluxseg2_mask: 1171 case Intrinsic::riscv_vluxseg3_mask: 1172 case Intrinsic::riscv_vluxseg4_mask: 1173 case Intrinsic::riscv_vluxseg5_mask: 1174 case Intrinsic::riscv_vluxseg6_mask: 1175 case Intrinsic::riscv_vluxseg7_mask: 1176 case Intrinsic::riscv_vluxseg8_mask: 1177 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1178 return; 1179 case Intrinsic::riscv_vlseg8ff: 1180 case Intrinsic::riscv_vlseg7ff: 1181 case Intrinsic::riscv_vlseg6ff: 1182 case Intrinsic::riscv_vlseg5ff: 1183 case Intrinsic::riscv_vlseg4ff: 1184 case Intrinsic::riscv_vlseg3ff: 1185 case Intrinsic::riscv_vlseg2ff: { 1186 selectVLSEGFF(Node, /*IsMasked*/ false); 1187 return; 1188 } 1189 case Intrinsic::riscv_vlseg8ff_mask: 1190 case Intrinsic::riscv_vlseg7ff_mask: 1191 case Intrinsic::riscv_vlseg6ff_mask: 1192 case Intrinsic::riscv_vlseg5ff_mask: 1193 case Intrinsic::riscv_vlseg4ff_mask: 1194 case Intrinsic::riscv_vlseg3ff_mask: 1195 case Intrinsic::riscv_vlseg2ff_mask: { 1196 selectVLSEGFF(Node, /*IsMasked*/ true); 1197 return; 1198 } 1199 case Intrinsic::riscv_vloxei: 1200 case Intrinsic::riscv_vloxei_mask: 1201 case Intrinsic::riscv_vluxei: 1202 case Intrinsic::riscv_vluxei_mask: { 1203 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask || 1204 IntNo == Intrinsic::riscv_vluxei_mask; 1205 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei || 1206 IntNo == Intrinsic::riscv_vloxei_mask; 1207 1208 MVT VT = Node->getSimpleValueType(0); 1209 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1210 1211 unsigned CurOp = 2; 1212 // Masked intrinsic only have TU version pseduo instructions. 1213 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1214 SmallVector<SDValue, 8> Operands; 1215 if (IsTU) 1216 Operands.push_back(Node->getOperand(CurOp++)); 1217 else 1218 // Skip the undef passthru operand for nomask TA version pseudo 1219 CurOp++; 1220 1221 MVT IndexVT; 1222 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1223 /*IsStridedOrIndexed*/ true, Operands, 1224 /*IsLoad=*/true, &IndexVT); 1225 1226 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1227 "Element count mismatch"); 1228 1229 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1230 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1231 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1232 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1233 report_fatal_error("The V extension does not support EEW=64 for index " 1234 "values when XLEN=32"); 1235 } 1236 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( 1237 IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 1238 static_cast<unsigned>(IndexLMUL)); 1239 MachineSDNode *Load = 1240 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1241 1242 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1243 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1244 1245 ReplaceNode(Node, Load); 1246 return; 1247 } 1248 case Intrinsic::riscv_vlm: 1249 case Intrinsic::riscv_vle: 1250 case Intrinsic::riscv_vle_mask: 1251 case Intrinsic::riscv_vlse: 1252 case Intrinsic::riscv_vlse_mask: { 1253 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask || 1254 IntNo == Intrinsic::riscv_vlse_mask; 1255 bool IsStrided = 1256 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask; 1257 1258 MVT VT = Node->getSimpleValueType(0); 1259 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1260 1261 unsigned CurOp = 2; 1262 // The riscv_vlm intrinsic are always tail agnostic and no passthru operand. 1263 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm; 1264 // Masked intrinsic only have TU version pseduo instructions. 1265 bool IsTU = 1266 HasPassthruOperand && 1267 ((!IsMasked && !Node->getOperand(CurOp).isUndef()) || IsMasked); 1268 SmallVector<SDValue, 8> Operands; 1269 if (IsTU) 1270 Operands.push_back(Node->getOperand(CurOp++)); 1271 else if (HasPassthruOperand) 1272 // Skip the undef passthru operand for nomask TA version pseudo 1273 CurOp++; 1274 1275 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1276 Operands, /*IsLoad=*/true); 1277 1278 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1279 const RISCV::VLEPseudo *P = 1280 RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 1281 static_cast<unsigned>(LMUL)); 1282 MachineSDNode *Load = 1283 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1284 1285 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1286 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1287 1288 ReplaceNode(Node, Load); 1289 return; 1290 } 1291 case Intrinsic::riscv_vleff: 1292 case Intrinsic::riscv_vleff_mask: { 1293 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask; 1294 1295 MVT VT = Node->getSimpleValueType(0); 1296 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1297 1298 unsigned CurOp = 2; 1299 // Masked intrinsic only have TU version pseduo instructions. 1300 bool IsTU = IsMasked || (!IsMasked && !Node->getOperand(CurOp).isUndef()); 1301 SmallVector<SDValue, 7> Operands; 1302 if (IsTU) 1303 Operands.push_back(Node->getOperand(CurOp++)); 1304 else 1305 // Skip the undef passthru operand for nomask TA version pseudo 1306 CurOp++; 1307 1308 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1309 /*IsStridedOrIndexed*/ false, Operands, 1310 /*IsLoad=*/true); 1311 1312 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1313 const RISCV::VLEPseudo *P = 1314 RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 1315 Log2SEW, static_cast<unsigned>(LMUL)); 1316 MachineSDNode *Load = 1317 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), 1318 MVT::Other, MVT::Glue, Operands); 1319 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, 1320 /*Glue*/ SDValue(Load, 2)); 1321 1322 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1323 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1324 1325 ReplaceUses(SDValue(Node, 0), SDValue(Load, 0)); 1326 ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL 1327 ReplaceUses(SDValue(Node, 2), SDValue(Load, 1)); // Chain 1328 CurDAG->RemoveDeadNode(Node); 1329 return; 1330 } 1331 } 1332 break; 1333 } 1334 case ISD::INTRINSIC_VOID: { 1335 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1336 switch (IntNo) { 1337 case Intrinsic::riscv_vsseg2: 1338 case Intrinsic::riscv_vsseg3: 1339 case Intrinsic::riscv_vsseg4: 1340 case Intrinsic::riscv_vsseg5: 1341 case Intrinsic::riscv_vsseg6: 1342 case Intrinsic::riscv_vsseg7: 1343 case Intrinsic::riscv_vsseg8: { 1344 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1345 return; 1346 } 1347 case Intrinsic::riscv_vsseg2_mask: 1348 case Intrinsic::riscv_vsseg3_mask: 1349 case Intrinsic::riscv_vsseg4_mask: 1350 case Intrinsic::riscv_vsseg5_mask: 1351 case Intrinsic::riscv_vsseg6_mask: 1352 case Intrinsic::riscv_vsseg7_mask: 1353 case Intrinsic::riscv_vsseg8_mask: { 1354 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1355 return; 1356 } 1357 case Intrinsic::riscv_vssseg2: 1358 case Intrinsic::riscv_vssseg3: 1359 case Intrinsic::riscv_vssseg4: 1360 case Intrinsic::riscv_vssseg5: 1361 case Intrinsic::riscv_vssseg6: 1362 case Intrinsic::riscv_vssseg7: 1363 case Intrinsic::riscv_vssseg8: { 1364 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1365 return; 1366 } 1367 case Intrinsic::riscv_vssseg2_mask: 1368 case Intrinsic::riscv_vssseg3_mask: 1369 case Intrinsic::riscv_vssseg4_mask: 1370 case Intrinsic::riscv_vssseg5_mask: 1371 case Intrinsic::riscv_vssseg6_mask: 1372 case Intrinsic::riscv_vssseg7_mask: 1373 case Intrinsic::riscv_vssseg8_mask: { 1374 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1375 return; 1376 } 1377 case Intrinsic::riscv_vsoxseg2: 1378 case Intrinsic::riscv_vsoxseg3: 1379 case Intrinsic::riscv_vsoxseg4: 1380 case Intrinsic::riscv_vsoxseg5: 1381 case Intrinsic::riscv_vsoxseg6: 1382 case Intrinsic::riscv_vsoxseg7: 1383 case Intrinsic::riscv_vsoxseg8: 1384 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1385 return; 1386 case Intrinsic::riscv_vsuxseg2: 1387 case Intrinsic::riscv_vsuxseg3: 1388 case Intrinsic::riscv_vsuxseg4: 1389 case Intrinsic::riscv_vsuxseg5: 1390 case Intrinsic::riscv_vsuxseg6: 1391 case Intrinsic::riscv_vsuxseg7: 1392 case Intrinsic::riscv_vsuxseg8: 1393 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1394 return; 1395 case Intrinsic::riscv_vsoxseg2_mask: 1396 case Intrinsic::riscv_vsoxseg3_mask: 1397 case Intrinsic::riscv_vsoxseg4_mask: 1398 case Intrinsic::riscv_vsoxseg5_mask: 1399 case Intrinsic::riscv_vsoxseg6_mask: 1400 case Intrinsic::riscv_vsoxseg7_mask: 1401 case Intrinsic::riscv_vsoxseg8_mask: 1402 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1403 return; 1404 case Intrinsic::riscv_vsuxseg2_mask: 1405 case Intrinsic::riscv_vsuxseg3_mask: 1406 case Intrinsic::riscv_vsuxseg4_mask: 1407 case Intrinsic::riscv_vsuxseg5_mask: 1408 case Intrinsic::riscv_vsuxseg6_mask: 1409 case Intrinsic::riscv_vsuxseg7_mask: 1410 case Intrinsic::riscv_vsuxseg8_mask: 1411 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1412 return; 1413 case Intrinsic::riscv_vsoxei: 1414 case Intrinsic::riscv_vsoxei_mask: 1415 case Intrinsic::riscv_vsuxei: 1416 case Intrinsic::riscv_vsuxei_mask: { 1417 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask || 1418 IntNo == Intrinsic::riscv_vsuxei_mask; 1419 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei || 1420 IntNo == Intrinsic::riscv_vsoxei_mask; 1421 1422 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1423 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1424 1425 unsigned CurOp = 2; 1426 SmallVector<SDValue, 8> Operands; 1427 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1428 1429 MVT IndexVT; 1430 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1431 /*IsStridedOrIndexed*/ true, Operands, 1432 /*IsLoad=*/false, &IndexVT); 1433 1434 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1435 "Element count mismatch"); 1436 1437 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1438 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1439 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1440 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1441 report_fatal_error("The V extension does not support EEW=64 for index " 1442 "values when XLEN=32"); 1443 } 1444 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( 1445 IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW, 1446 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL)); 1447 MachineSDNode *Store = 1448 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1449 1450 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1451 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1452 1453 ReplaceNode(Node, Store); 1454 return; 1455 } 1456 case Intrinsic::riscv_vsm: 1457 case Intrinsic::riscv_vse: 1458 case Intrinsic::riscv_vse_mask: 1459 case Intrinsic::riscv_vsse: 1460 case Intrinsic::riscv_vsse_mask: { 1461 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask || 1462 IntNo == Intrinsic::riscv_vsse_mask; 1463 bool IsStrided = 1464 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask; 1465 1466 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1467 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1468 1469 unsigned CurOp = 2; 1470 SmallVector<SDValue, 8> Operands; 1471 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1472 1473 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1474 Operands); 1475 1476 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1477 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( 1478 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 1479 MachineSDNode *Store = 1480 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1481 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1482 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1483 1484 ReplaceNode(Node, Store); 1485 return; 1486 } 1487 } 1488 break; 1489 } 1490 case ISD::BITCAST: { 1491 MVT SrcVT = Node->getOperand(0).getSimpleValueType(); 1492 // Just drop bitcasts between vectors if both are fixed or both are 1493 // scalable. 1494 if ((VT.isScalableVector() && SrcVT.isScalableVector()) || 1495 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) { 1496 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 1497 CurDAG->RemoveDeadNode(Node); 1498 return; 1499 } 1500 break; 1501 } 1502 case ISD::INSERT_SUBVECTOR: { 1503 SDValue V = Node->getOperand(0); 1504 SDValue SubV = Node->getOperand(1); 1505 SDLoc DL(SubV); 1506 auto Idx = Node->getConstantOperandVal(2); 1507 MVT SubVecVT = SubV.getSimpleValueType(); 1508 1509 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1510 MVT SubVecContainerVT = SubVecVT; 1511 // Establish the correct scalable-vector types for any fixed-length type. 1512 if (SubVecVT.isFixedLengthVector()) 1513 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT); 1514 if (VT.isFixedLengthVector()) 1515 VT = TLI.getContainerForFixedLengthVector(VT); 1516 1517 const auto *TRI = Subtarget->getRegisterInfo(); 1518 unsigned SubRegIdx; 1519 std::tie(SubRegIdx, Idx) = 1520 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1521 VT, SubVecContainerVT, Idx, TRI); 1522 1523 // If the Idx hasn't been completely eliminated then this is a subvector 1524 // insert which doesn't naturally align to a vector register. These must 1525 // be handled using instructions to manipulate the vector registers. 1526 if (Idx != 0) 1527 break; 1528 1529 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT); 1530 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 || 1531 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 || 1532 SubVecLMUL == RISCVII::VLMUL::LMUL_F8; 1533 (void)IsSubVecPartReg; // Silence unused variable warning without asserts. 1534 assert((!IsSubVecPartReg || V.isUndef()) && 1535 "Expecting lowering to have created legal INSERT_SUBVECTORs when " 1536 "the subvector is smaller than a full-sized register"); 1537 1538 // If we haven't set a SubRegIdx, then we must be going between 1539 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. 1540 if (SubRegIdx == RISCV::NoSubRegister) { 1541 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT); 1542 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1543 InRegClassID && 1544 "Unexpected subvector extraction"); 1545 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1546 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1547 DL, VT, SubV, RC); 1548 ReplaceNode(Node, NewNode); 1549 return; 1550 } 1551 1552 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV); 1553 ReplaceNode(Node, Insert.getNode()); 1554 return; 1555 } 1556 case ISD::EXTRACT_SUBVECTOR: { 1557 SDValue V = Node->getOperand(0); 1558 auto Idx = Node->getConstantOperandVal(1); 1559 MVT InVT = V.getSimpleValueType(); 1560 SDLoc DL(V); 1561 1562 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1563 MVT SubVecContainerVT = VT; 1564 // Establish the correct scalable-vector types for any fixed-length type. 1565 if (VT.isFixedLengthVector()) 1566 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT); 1567 if (InVT.isFixedLengthVector()) 1568 InVT = TLI.getContainerForFixedLengthVector(InVT); 1569 1570 const auto *TRI = Subtarget->getRegisterInfo(); 1571 unsigned SubRegIdx; 1572 std::tie(SubRegIdx, Idx) = 1573 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1574 InVT, SubVecContainerVT, Idx, TRI); 1575 1576 // If the Idx hasn't been completely eliminated then this is a subvector 1577 // extract which doesn't naturally align to a vector register. These must 1578 // be handled using instructions to manipulate the vector registers. 1579 if (Idx != 0) 1580 break; 1581 1582 // If we haven't set a SubRegIdx, then we must be going between 1583 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy. 1584 if (SubRegIdx == RISCV::NoSubRegister) { 1585 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT); 1586 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1587 InRegClassID && 1588 "Unexpected subvector extraction"); 1589 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1590 SDNode *NewNode = 1591 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC); 1592 ReplaceNode(Node, NewNode); 1593 return; 1594 } 1595 1596 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V); 1597 ReplaceNode(Node, Extract.getNode()); 1598 return; 1599 } 1600 case ISD::SPLAT_VECTOR: 1601 case RISCVISD::VMV_S_X_VL: 1602 case RISCVISD::VFMV_S_F_VL: 1603 case RISCVISD::VMV_V_X_VL: 1604 case RISCVISD::VFMV_V_F_VL: { 1605 // Try to match splat of a scalar load to a strided load with stride of x0. 1606 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL || 1607 Node->getOpcode() == RISCVISD::VFMV_S_F_VL; 1608 if (IsScalarMove && !Node->getOperand(0).isUndef()) 1609 break; 1610 SDValue Src = IsScalarMove ? Node->getOperand(1) : Node->getOperand(0); 1611 auto *Ld = dyn_cast<LoadSDNode>(Src); 1612 if (!Ld) 1613 break; 1614 EVT MemVT = Ld->getMemoryVT(); 1615 // The memory VT should be the same size as the element type. 1616 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize()) 1617 break; 1618 if (!IsProfitableToFold(Src, Node, Node) || 1619 !IsLegalToFold(Src, Node, Node, TM.getOptLevel())) 1620 break; 1621 1622 SDValue VL; 1623 if (Node->getOpcode() == ISD::SPLAT_VECTOR) 1624 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); 1625 else if (IsScalarMove) { 1626 // We could deal with more VL if we update the VSETVLI insert pass to 1627 // avoid introducing more VSETVLI. 1628 if (!isOneConstant(Node->getOperand(2))) 1629 break; 1630 selectVLOp(Node->getOperand(2), VL); 1631 } else 1632 selectVLOp(Node->getOperand(1), VL); 1633 1634 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1635 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 1636 1637 SDValue Operands[] = {Ld->getBasePtr(), 1638 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW, 1639 Ld->getChain()}; 1640 1641 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1642 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo( 1643 /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false, 1644 Log2SEW, static_cast<unsigned>(LMUL)); 1645 MachineSDNode *Load = 1646 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1647 1648 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1649 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1650 1651 ReplaceNode(Node, Load); 1652 return; 1653 } 1654 } 1655 1656 // Select the default instruction. 1657 SelectCode(Node); 1658 } 1659 1660 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand( 1661 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 1662 switch (ConstraintID) { 1663 case InlineAsm::Constraint_m: 1664 // We just support simple memory operands that have a single address 1665 // operand and need no special handling. 1666 OutOps.push_back(Op); 1667 return false; 1668 case InlineAsm::Constraint_A: 1669 OutOps.push_back(Op); 1670 return false; 1671 default: 1672 break; 1673 } 1674 1675 return true; 1676 } 1677 1678 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) { 1679 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1680 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1681 return true; 1682 } 1683 return false; 1684 } 1685 1686 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) { 1687 // If this is FrameIndex, select it directly. Otherwise just let it get 1688 // selected to a register independently. 1689 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) 1690 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1691 else 1692 Base = Addr; 1693 return true; 1694 } 1695 1696 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, 1697 SDValue &ShAmt) { 1698 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift 1699 // amount. If there is an AND on the shift amount, we can bypass it if it 1700 // doesn't affect any of those bits. 1701 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 1702 const APInt &AndMask = N->getConstantOperandAPInt(1); 1703 1704 // Since the max shift amount is a power of 2 we can subtract 1 to make a 1705 // mask that covers the bits needed to represent all shift amounts. 1706 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!"); 1707 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); 1708 1709 if (ShMask.isSubsetOf(AndMask)) { 1710 ShAmt = N.getOperand(0); 1711 return true; 1712 } 1713 1714 // SimplifyDemandedBits may have optimized the mask so try restoring any 1715 // bits that are known zero. 1716 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0)); 1717 if (ShMask.isSubsetOf(AndMask | Known.Zero)) { 1718 ShAmt = N.getOperand(0); 1719 return true; 1720 } 1721 } 1722 1723 ShAmt = N; 1724 return true; 1725 } 1726 1727 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) { 1728 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG && 1729 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) { 1730 Val = N.getOperand(0); 1731 return true; 1732 } 1733 MVT VT = N.getSimpleValueType(); 1734 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) { 1735 Val = N; 1736 return true; 1737 } 1738 1739 return false; 1740 } 1741 1742 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) { 1743 if (N.getOpcode() == ISD::AND) { 1744 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1745 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) { 1746 Val = N.getOperand(0); 1747 return true; 1748 } 1749 } 1750 MVT VT = N.getSimpleValueType(); 1751 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32); 1752 if (CurDAG->MaskedValueIsZero(N, Mask)) { 1753 Val = N; 1754 return true; 1755 } 1756 1757 return false; 1758 } 1759 1760 // Return true if all users of this SDNode* only consume the lower \p Bits. 1761 // This can be used to form W instructions for add/sub/mul/shl even when the 1762 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if 1763 // SimplifyDemandedBits has made it so some users see a sext_inreg and some 1764 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave 1765 // the add/sub/mul/shl to become non-W instructions. By checking the users we 1766 // may be able to use a W instruction and CSE with the other instruction if 1767 // this has happened. We could try to detect that the CSE opportunity exists 1768 // before doing this, but that would be more complicated. 1769 // TODO: Does this need to look through AND/OR/XOR to their users to find more 1770 // opportunities. 1771 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const { 1772 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB || 1773 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL || 1774 Node->getOpcode() == ISD::SRL || 1775 Node->getOpcode() == ISD::SIGN_EXTEND_INREG || 1776 isa<ConstantSDNode>(Node)) && 1777 "Unexpected opcode"); 1778 1779 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 1780 SDNode *User = *UI; 1781 // Users of this node should have already been instruction selected 1782 if (!User->isMachineOpcode()) 1783 return false; 1784 1785 // TODO: Add more opcodes? 1786 switch (User->getMachineOpcode()) { 1787 default: 1788 return false; 1789 case RISCV::ADDW: 1790 case RISCV::ADDIW: 1791 case RISCV::SUBW: 1792 case RISCV::MULW: 1793 case RISCV::SLLW: 1794 case RISCV::SLLIW: 1795 case RISCV::SRAW: 1796 case RISCV::SRAIW: 1797 case RISCV::SRLW: 1798 case RISCV::SRLIW: 1799 case RISCV::DIVW: 1800 case RISCV::DIVUW: 1801 case RISCV::REMW: 1802 case RISCV::REMUW: 1803 case RISCV::ROLW: 1804 case RISCV::RORW: 1805 case RISCV::RORIW: 1806 case RISCV::CLZW: 1807 case RISCV::CTZW: 1808 case RISCV::CPOPW: 1809 case RISCV::SLLI_UW: 1810 case RISCV::FCVT_H_W: 1811 case RISCV::FCVT_H_WU: 1812 case RISCV::FCVT_S_W: 1813 case RISCV::FCVT_S_WU: 1814 case RISCV::FCVT_D_W: 1815 case RISCV::FCVT_D_WU: 1816 if (Bits < 32) 1817 return false; 1818 break; 1819 case RISCV::SLLI: 1820 // SLLI only uses the lower (XLen - ShAmt) bits. 1821 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1)) 1822 return false; 1823 break; 1824 case RISCV::ANDI: 1825 if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1)))) 1826 return false; 1827 break; 1828 case RISCV::SEXT_B: 1829 if (Bits < 8) 1830 return false; 1831 break; 1832 case RISCV::SEXT_H: 1833 case RISCV::ZEXT_H_RV32: 1834 case RISCV::ZEXT_H_RV64: 1835 if (Bits < 16) 1836 return false; 1837 break; 1838 case RISCV::ADD_UW: 1839 case RISCV::SH1ADD_UW: 1840 case RISCV::SH2ADD_UW: 1841 case RISCV::SH3ADD_UW: 1842 // The first operand to add.uw/shXadd.uw is implicitly zero extended from 1843 // 32 bits. 1844 if (UI.getOperandNo() != 0 || Bits < 32) 1845 return false; 1846 break; 1847 case RISCV::SB: 1848 if (UI.getOperandNo() != 0 || Bits < 8) 1849 return false; 1850 break; 1851 case RISCV::SH: 1852 if (UI.getOperandNo() != 0 || Bits < 16) 1853 return false; 1854 break; 1855 case RISCV::SW: 1856 if (UI.getOperandNo() != 0 || Bits < 32) 1857 return false; 1858 break; 1859 } 1860 } 1861 1862 return true; 1863 } 1864 1865 // Select VL as a 5 bit immediate or a value that will become a register. This 1866 // allows us to choose betwen VSETIVLI or VSETVLI later. 1867 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { 1868 auto *C = dyn_cast<ConstantSDNode>(N); 1869 if (C && isUInt<5>(C->getZExtValue())) 1870 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N), 1871 N->getValueType(0)); 1872 else if (C && C->isAllOnesValue() && C->getOpcode() != ISD::TargetConstant) 1873 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 1874 N->getValueType(0)); 1875 else 1876 VL = N; 1877 1878 return true; 1879 } 1880 1881 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { 1882 if (N.getOpcode() != ISD::SPLAT_VECTOR && 1883 N.getOpcode() != RISCVISD::VMV_V_X_VL) 1884 return false; 1885 SplatVal = N.getOperand(0); 1886 return true; 1887 } 1888 1889 using ValidateFn = bool (*)(int64_t); 1890 1891 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, 1892 SelectionDAG &DAG, 1893 const RISCVSubtarget &Subtarget, 1894 ValidateFn ValidateImm) { 1895 if ((N.getOpcode() != ISD::SPLAT_VECTOR && 1896 N.getOpcode() != RISCVISD::VMV_V_X_VL) || 1897 !isa<ConstantSDNode>(N.getOperand(0))) 1898 return false; 1899 1900 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue(); 1901 1902 // ISD::SPLAT_VECTOR, RISCVISD::VMV_V_X_VL share semantics when the operand 1903 // type is wider than the resulting vector element type: an implicit 1904 // truncation first takes place. Therefore, perform a manual 1905 // truncation/sign-extension in order to ignore any truncated bits and catch 1906 // any zero-extended immediate. 1907 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first 1908 // sign-extending to (XLenVT -1). 1909 MVT XLenVT = Subtarget.getXLenVT(); 1910 assert(XLenVT == N.getOperand(0).getSimpleValueType() && 1911 "Unexpected splat operand type"); 1912 MVT EltVT = N.getSimpleValueType().getVectorElementType(); 1913 if (EltVT.bitsLT(XLenVT)) 1914 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits()); 1915 1916 if (!ValidateImm(SplatImm)) 1917 return false; 1918 1919 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT); 1920 return true; 1921 } 1922 1923 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) { 1924 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget, 1925 [](int64_t Imm) { return isInt<5>(Imm); }); 1926 } 1927 1928 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) { 1929 return selectVSplatSimmHelper( 1930 N, SplatVal, *CurDAG, *Subtarget, 1931 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; }); 1932 } 1933 1934 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N, 1935 SDValue &SplatVal) { 1936 return selectVSplatSimmHelper( 1937 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) { 1938 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16); 1939 }); 1940 } 1941 1942 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) { 1943 if ((N.getOpcode() != ISD::SPLAT_VECTOR && 1944 N.getOpcode() != RISCVISD::VMV_V_X_VL) || 1945 !isa<ConstantSDNode>(N.getOperand(0))) 1946 return false; 1947 1948 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue(); 1949 1950 if (!isUInt<5>(SplatImm)) 1951 return false; 1952 1953 SplatVal = 1954 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT()); 1955 1956 return true; 1957 } 1958 1959 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width, 1960 SDValue &Imm) { 1961 if (auto *C = dyn_cast<ConstantSDNode>(N)) { 1962 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width); 1963 1964 if (!isInt<5>(ImmVal)) 1965 return false; 1966 1967 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT()); 1968 return true; 1969 } 1970 1971 return false; 1972 } 1973 1974 // Merge an ADDI into the offset of a load/store instruction where possible. 1975 // (load (addi base, off1), off2) -> (load base, off1+off2) 1976 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2) 1977 // This is possible when off1+off2 fits a 12-bit immediate. 1978 bool RISCVDAGToDAGISel::doPeepholeLoadStoreADDI(SDNode *N) { 1979 int OffsetOpIdx; 1980 int BaseOpIdx; 1981 1982 // Only attempt this optimisation for I-type loads and S-type stores. 1983 switch (N->getMachineOpcode()) { 1984 default: 1985 return false; 1986 case RISCV::LB: 1987 case RISCV::LH: 1988 case RISCV::LW: 1989 case RISCV::LBU: 1990 case RISCV::LHU: 1991 case RISCV::LWU: 1992 case RISCV::LD: 1993 case RISCV::FLH: 1994 case RISCV::FLW: 1995 case RISCV::FLD: 1996 BaseOpIdx = 0; 1997 OffsetOpIdx = 1; 1998 break; 1999 case RISCV::SB: 2000 case RISCV::SH: 2001 case RISCV::SW: 2002 case RISCV::SD: 2003 case RISCV::FSH: 2004 case RISCV::FSW: 2005 case RISCV::FSD: 2006 BaseOpIdx = 1; 2007 OffsetOpIdx = 2; 2008 break; 2009 } 2010 2011 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx))) 2012 return false; 2013 2014 SDValue Base = N->getOperand(BaseOpIdx); 2015 2016 // If the base is an ADDI, we can merge it in to the load/store. 2017 if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI) 2018 return false; 2019 2020 SDValue ImmOperand = Base.getOperand(1); 2021 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx); 2022 2023 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) { 2024 int64_t Offset1 = Const->getSExtValue(); 2025 int64_t CombinedOffset = Offset1 + Offset2; 2026 if (!isInt<12>(CombinedOffset)) 2027 return false; 2028 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand), 2029 ImmOperand.getValueType()); 2030 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) { 2031 // If the off1 in (addi base, off1) is a global variable's address (its 2032 // low part, really), then we can rely on the alignment of that variable 2033 // to provide a margin of safety before off1 can overflow the 12 bits. 2034 // Check if off2 falls within that margin; if so off1+off2 can't overflow. 2035 const DataLayout &DL = CurDAG->getDataLayout(); 2036 Align Alignment = GA->getGlobal()->getPointerAlignment(DL); 2037 if (Offset2 != 0 && Alignment <= Offset2) 2038 return false; 2039 int64_t Offset1 = GA->getOffset(); 2040 int64_t CombinedOffset = Offset1 + Offset2; 2041 ImmOperand = CurDAG->getTargetGlobalAddress( 2042 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(), 2043 CombinedOffset, GA->getTargetFlags()); 2044 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) { 2045 // Ditto. 2046 Align Alignment = CP->getAlign(); 2047 if (Offset2 != 0 && Alignment <= Offset2) 2048 return false; 2049 int64_t Offset1 = CP->getOffset(); 2050 int64_t CombinedOffset = Offset1 + Offset2; 2051 ImmOperand = CurDAG->getTargetConstantPool( 2052 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(), 2053 CombinedOffset, CP->getTargetFlags()); 2054 } else { 2055 return false; 2056 } 2057 2058 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 2059 LLVM_DEBUG(Base->dump(CurDAG)); 2060 LLVM_DEBUG(dbgs() << "\nN: "); 2061 LLVM_DEBUG(N->dump(CurDAG)); 2062 LLVM_DEBUG(dbgs() << "\n"); 2063 2064 // Modify the offset operand of the load/store. 2065 if (BaseOpIdx == 0) // Load 2066 CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand, 2067 N->getOperand(2)); 2068 else // Store 2069 CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0), 2070 ImmOperand, N->getOperand(3)); 2071 2072 return true; 2073 } 2074 2075 // Try to remove sext.w if the input is a W instruction or can be made into 2076 // a W instruction cheaply. 2077 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { 2078 // Look for the sext.w pattern, addiw rd, rs1, 0. 2079 if (N->getMachineOpcode() != RISCV::ADDIW || 2080 !isNullConstant(N->getOperand(1))) 2081 return false; 2082 2083 SDValue N0 = N->getOperand(0); 2084 if (!N0.isMachineOpcode()) 2085 return false; 2086 2087 switch (N0.getMachineOpcode()) { 2088 default: 2089 break; 2090 case RISCV::ADD: 2091 case RISCV::ADDI: 2092 case RISCV::SUB: 2093 case RISCV::MUL: 2094 case RISCV::SLLI: { 2095 // Convert sext.w+add/sub/mul to their W instructions. This will create 2096 // a new independent instruction. This improves latency. 2097 unsigned Opc; 2098 switch (N0.getMachineOpcode()) { 2099 default: 2100 llvm_unreachable("Unexpected opcode!"); 2101 case RISCV::ADD: Opc = RISCV::ADDW; break; 2102 case RISCV::ADDI: Opc = RISCV::ADDIW; break; 2103 case RISCV::SUB: Opc = RISCV::SUBW; break; 2104 case RISCV::MUL: Opc = RISCV::MULW; break; 2105 case RISCV::SLLI: Opc = RISCV::SLLIW; break; 2106 } 2107 2108 SDValue N00 = N0.getOperand(0); 2109 SDValue N01 = N0.getOperand(1); 2110 2111 // Shift amount needs to be uimm5. 2112 if (N0.getMachineOpcode() == RISCV::SLLI && 2113 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue())) 2114 break; 2115 2116 SDNode *Result = 2117 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), 2118 N00, N01); 2119 ReplaceUses(N, Result); 2120 return true; 2121 } 2122 case RISCV::ADDW: 2123 case RISCV::ADDIW: 2124 case RISCV::SUBW: 2125 case RISCV::MULW: 2126 case RISCV::SLLIW: 2127 // Result is already sign extended just remove the sext.w. 2128 // NOTE: We only handle the nodes that are selected with hasAllWUsers. 2129 ReplaceUses(N, N0.getNode()); 2130 return true; 2131 } 2132 2133 return false; 2134 } 2135 2136 // This pass converts a legalized DAG into a RISCV-specific DAG, ready 2137 // for instruction scheduling. 2138 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) { 2139 return new RISCVDAGToDAGISel(TM); 2140 } 2141