1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the RISCV target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVISelDAGToDAG.h" 14 #include "MCTargetDesc/RISCVMCTargetDesc.h" 15 #include "MCTargetDesc/RISCVMatInt.h" 16 #include "RISCVISelLowering.h" 17 #include "RISCVMachineFunctionInfo.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/IR/IntrinsicsRISCV.h" 20 #include "llvm/Support/Alignment.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/KnownBits.h" 23 #include "llvm/Support/MathExtras.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 #define DEBUG_TYPE "riscv-isel" 29 30 namespace llvm { 31 namespace RISCV { 32 #define GET_RISCVVSSEGTable_IMPL 33 #define GET_RISCVVLSEGTable_IMPL 34 #define GET_RISCVVLXSEGTable_IMPL 35 #define GET_RISCVVSXSEGTable_IMPL 36 #define GET_RISCVVLETable_IMPL 37 #define GET_RISCVVSETable_IMPL 38 #define GET_RISCVVLXTable_IMPL 39 #define GET_RISCVVSXTable_IMPL 40 #define GET_RISCVMaskedPseudosTable_IMPL 41 #include "RISCVGenSearchableTables.inc" 42 } // namespace RISCV 43 } // namespace llvm 44 45 void RISCVDAGToDAGISel::PreprocessISelDAG() { 46 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 47 48 bool MadeChange = false; 49 while (Position != CurDAG->allnodes_begin()) { 50 SDNode *N = &*--Position; 51 if (N->use_empty()) 52 continue; 53 54 SDValue Result; 55 switch (N->getOpcode()) { 56 case ISD::SPLAT_VECTOR: { 57 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point 58 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden. 59 MVT VT = N->getSimpleValueType(0); 60 unsigned Opc = 61 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL; 62 SDLoc DL(N); 63 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()); 64 Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), 65 N->getOperand(0), VL); 66 break; 67 } 68 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: { 69 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector 70 // load. Done after lowering and combining so that we have a chance to 71 // optimize this to VMV_V_X_VL when the upper bits aren't needed. 72 assert(N->getNumOperands() == 4 && "Unexpected number of operands"); 73 MVT VT = N->getSimpleValueType(0); 74 SDValue Passthru = N->getOperand(0); 75 SDValue Lo = N->getOperand(1); 76 SDValue Hi = N->getOperand(2); 77 SDValue VL = N->getOperand(3); 78 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() && 79 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 && 80 "Unexpected VTs!"); 81 MachineFunction &MF = CurDAG->getMachineFunction(); 82 RISCVMachineFunctionInfo *FuncInfo = 83 MF.getInfo<RISCVMachineFunctionInfo>(); 84 SDLoc DL(N); 85 86 // We use the same frame index we use for moving two i32s into 64-bit FPR. 87 // This is an analogous operation. 88 int FI = FuncInfo->getMoveF64FrameIndex(MF); 89 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); 90 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 91 SDValue StackSlot = 92 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout())); 93 94 SDValue Chain = CurDAG->getEntryNode(); 95 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8)); 96 97 SDValue OffsetSlot = 98 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL); 99 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4), 100 Align(8)); 101 102 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 103 104 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other}); 105 SDValue IntID = 106 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64); 107 SDValue Ops[] = {Chain, 108 IntID, 109 Passthru, 110 StackSlot, 111 CurDAG->getRegister(RISCV::X0, MVT::i64), 112 VL}; 113 114 Result = CurDAG->getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, 115 MVT::i64, MPI, Align(8), 116 MachineMemOperand::MOLoad); 117 break; 118 } 119 } 120 121 if (Result) { 122 LLVM_DEBUG(dbgs() << "RISCV DAG preprocessing replacing:\nOld: "); 123 LLVM_DEBUG(N->dump(CurDAG)); 124 LLVM_DEBUG(dbgs() << "\nNew: "); 125 LLVM_DEBUG(Result->dump(CurDAG)); 126 LLVM_DEBUG(dbgs() << "\n"); 127 128 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 129 MadeChange = true; 130 } 131 } 132 133 if (MadeChange) 134 CurDAG->RemoveDeadNodes(); 135 } 136 137 void RISCVDAGToDAGISel::PostprocessISelDAG() { 138 HandleSDNode Dummy(CurDAG->getRoot()); 139 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 140 141 bool MadeChange = false; 142 while (Position != CurDAG->allnodes_begin()) { 143 SDNode *N = &*--Position; 144 // Skip dead nodes and any non-machine opcodes. 145 if (N->use_empty() || !N->isMachineOpcode()) 146 continue; 147 148 MadeChange |= doPeepholeSExtW(N); 149 MadeChange |= doPeepholeLoadStoreADDI(N); 150 MadeChange |= doPeepholeMaskedRVV(N); 151 } 152 153 CurDAG->setRoot(Dummy.getValue()); 154 155 if (MadeChange) 156 CurDAG->RemoveDeadNodes(); 157 } 158 159 // Returns true if N is a MachineSDNode that has a reg and simm12 memory 160 // operand. The indices of the base pointer and offset are returned in BaseOpIdx 161 // and OffsetOpIdx. 162 static bool hasMemOffset(SDNode *N, unsigned &BaseOpIdx, 163 unsigned &OffsetOpIdx) { 164 switch (N->getMachineOpcode()) { 165 case RISCV::LB: 166 case RISCV::LH: 167 case RISCV::LW: 168 case RISCV::LBU: 169 case RISCV::LHU: 170 case RISCV::LWU: 171 case RISCV::LD: 172 case RISCV::FLH: 173 case RISCV::FLW: 174 case RISCV::FLD: 175 BaseOpIdx = 0; 176 OffsetOpIdx = 1; 177 return true; 178 case RISCV::SB: 179 case RISCV::SH: 180 case RISCV::SW: 181 case RISCV::SD: 182 case RISCV::FSH: 183 case RISCV::FSW: 184 case RISCV::FSD: 185 BaseOpIdx = 1; 186 OffsetOpIdx = 2; 187 return true; 188 } 189 190 return false; 191 } 192 193 static SDNode *selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 194 RISCVMatInt::InstSeq &Seq) { 195 SDNode *Result = nullptr; 196 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT); 197 for (RISCVMatInt::Inst &Inst : Seq) { 198 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, VT); 199 switch (Inst.getOpndKind()) { 200 case RISCVMatInt::Imm: 201 Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SDImm); 202 break; 203 case RISCVMatInt::RegX0: 204 Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, 205 CurDAG->getRegister(RISCV::X0, VT)); 206 break; 207 case RISCVMatInt::RegReg: 208 Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SrcReg); 209 break; 210 case RISCVMatInt::RegImm: 211 Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SDImm); 212 break; 213 } 214 215 // Only the first instruction has X0 as its source. 216 SrcReg = SDValue(Result, 0); 217 } 218 219 return Result; 220 } 221 222 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, 223 int64_t Imm, const RISCVSubtarget &Subtarget) { 224 RISCVMatInt::InstSeq Seq = 225 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits()); 226 227 return selectImmSeq(CurDAG, DL, VT, Seq); 228 } 229 230 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs, 231 unsigned NF, RISCVII::VLMUL LMUL) { 232 static const unsigned M1TupleRegClassIDs[] = { 233 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID, 234 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID, 235 RISCV::VRN8M1RegClassID}; 236 static const unsigned M2TupleRegClassIDs[] = {RISCV::VRN2M2RegClassID, 237 RISCV::VRN3M2RegClassID, 238 RISCV::VRN4M2RegClassID}; 239 240 assert(Regs.size() >= 2 && Regs.size() <= 8); 241 242 unsigned RegClassID; 243 unsigned SubReg0; 244 switch (LMUL) { 245 default: 246 llvm_unreachable("Invalid LMUL."); 247 case RISCVII::VLMUL::LMUL_F8: 248 case RISCVII::VLMUL::LMUL_F4: 249 case RISCVII::VLMUL::LMUL_F2: 250 case RISCVII::VLMUL::LMUL_1: 251 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7, 252 "Unexpected subreg numbering"); 253 SubReg0 = RISCV::sub_vrm1_0; 254 RegClassID = M1TupleRegClassIDs[NF - 2]; 255 break; 256 case RISCVII::VLMUL::LMUL_2: 257 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3, 258 "Unexpected subreg numbering"); 259 SubReg0 = RISCV::sub_vrm2_0; 260 RegClassID = M2TupleRegClassIDs[NF - 2]; 261 break; 262 case RISCVII::VLMUL::LMUL_4: 263 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1, 264 "Unexpected subreg numbering"); 265 SubReg0 = RISCV::sub_vrm4_0; 266 RegClassID = RISCV::VRN2M4RegClassID; 267 break; 268 } 269 270 SDLoc DL(Regs[0]); 271 SmallVector<SDValue, 8> Ops; 272 273 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32)); 274 275 for (unsigned I = 0; I < Regs.size(); ++I) { 276 Ops.push_back(Regs[I]); 277 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32)); 278 } 279 SDNode *N = 280 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 281 return SDValue(N, 0); 282 } 283 284 void RISCVDAGToDAGISel::addVectorLoadStoreOperands( 285 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp, 286 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands, 287 bool IsLoad, MVT *IndexVT) { 288 SDValue Chain = Node->getOperand(0); 289 SDValue Glue; 290 291 Operands.push_back(Node->getOperand(CurOp++)); // Base pointer. 292 293 if (IsStridedOrIndexed) { 294 Operands.push_back(Node->getOperand(CurOp++)); // Index. 295 if (IndexVT) 296 *IndexVT = Operands.back()->getSimpleValueType(0); 297 } 298 299 if (IsMasked) { 300 // Mask needs to be copied to V0. 301 SDValue Mask = Node->getOperand(CurOp++); 302 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); 303 Glue = Chain.getValue(1); 304 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); 305 } 306 SDValue VL; 307 selectVLOp(Node->getOperand(CurOp++), VL); 308 Operands.push_back(VL); 309 310 MVT XLenVT = Subtarget->getXLenVT(); 311 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 312 Operands.push_back(SEWOp); 313 314 // Masked load has the tail policy argument. 315 if (IsMasked && IsLoad) { 316 // Policy must be a constant. 317 uint64_t Policy = Node->getConstantOperandVal(CurOp++); 318 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT); 319 Operands.push_back(PolicyOp); 320 } 321 322 Operands.push_back(Chain); // Chain. 323 if (Glue) 324 Operands.push_back(Glue); 325 } 326 327 static bool isAllUndef(ArrayRef<SDValue> Values) { 328 return llvm::all_of(Values, [](SDValue V) { return V->isUndef(); }); 329 } 330 331 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked, 332 bool IsStrided) { 333 SDLoc DL(Node); 334 unsigned NF = Node->getNumValues() - 1; 335 MVT VT = Node->getSimpleValueType(0); 336 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 337 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 338 339 unsigned CurOp = 2; 340 SmallVector<SDValue, 8> Operands; 341 342 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 343 Node->op_begin() + CurOp + NF); 344 bool IsTU = IsMasked || !isAllUndef(Regs); 345 if (IsTU) { 346 SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL); 347 Operands.push_back(Merge); 348 } 349 CurOp += NF; 350 351 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 352 Operands, /*IsLoad=*/true); 353 354 const RISCV::VLSEGPseudo *P = 355 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 356 static_cast<unsigned>(LMUL)); 357 MachineSDNode *Load = 358 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 359 360 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 361 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 362 363 SDValue SuperReg = SDValue(Load, 0); 364 for (unsigned I = 0; I < NF; ++I) { 365 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 366 ReplaceUses(SDValue(Node, I), 367 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 368 } 369 370 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 371 CurDAG->RemoveDeadNode(Node); 372 } 373 374 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) { 375 SDLoc DL(Node); 376 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. 377 MVT VT = Node->getSimpleValueType(0); 378 MVT XLenVT = Subtarget->getXLenVT(); 379 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 380 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 381 382 unsigned CurOp = 2; 383 SmallVector<SDValue, 7> Operands; 384 385 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 386 Node->op_begin() + CurOp + NF); 387 bool IsTU = IsMasked || !isAllUndef(Regs); 388 if (IsTU) { 389 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 390 Operands.push_back(MaskedOff); 391 } 392 CurOp += NF; 393 394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 395 /*IsStridedOrIndexed*/ false, Operands, 396 /*IsLoad=*/true); 397 398 const RISCV::VLSEGPseudo *P = 399 RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 400 Log2SEW, static_cast<unsigned>(LMUL)); 401 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, 402 XLenVT, MVT::Other, Operands); 403 404 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 405 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 406 407 SDValue SuperReg = SDValue(Load, 0); 408 for (unsigned I = 0; I < NF; ++I) { 409 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 410 ReplaceUses(SDValue(Node, I), 411 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 412 } 413 414 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL 415 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain 416 CurDAG->RemoveDeadNode(Node); 417 } 418 419 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked, 420 bool IsOrdered) { 421 SDLoc DL(Node); 422 unsigned NF = Node->getNumValues() - 1; 423 MVT VT = Node->getSimpleValueType(0); 424 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 425 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 426 427 unsigned CurOp = 2; 428 SmallVector<SDValue, 8> Operands; 429 430 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp, 431 Node->op_begin() + CurOp + NF); 432 bool IsTU = IsMasked || !isAllUndef(Regs); 433 if (IsTU) { 434 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); 435 Operands.push_back(MaskedOff); 436 } 437 CurOp += NF; 438 439 MVT IndexVT; 440 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 441 /*IsStridedOrIndexed*/ true, Operands, 442 /*IsLoad=*/true, &IndexVT); 443 444 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 445 "Element count mismatch"); 446 447 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 448 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 449 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 450 report_fatal_error("The V extension does not support EEW=64 for index " 451 "values when XLEN=32"); 452 } 453 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo( 454 NF, IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 455 static_cast<unsigned>(IndexLMUL)); 456 MachineSDNode *Load = 457 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); 458 459 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 460 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 461 462 SDValue SuperReg = SDValue(Load, 0); 463 for (unsigned I = 0; I < NF; ++I) { 464 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I); 465 ReplaceUses(SDValue(Node, I), 466 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg)); 467 } 468 469 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); 470 CurDAG->RemoveDeadNode(Node); 471 } 472 473 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked, 474 bool IsStrided) { 475 SDLoc DL(Node); 476 unsigned NF = Node->getNumOperands() - 4; 477 if (IsStrided) 478 NF--; 479 if (IsMasked) 480 NF--; 481 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 482 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 483 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 484 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 485 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 486 487 SmallVector<SDValue, 8> Operands; 488 Operands.push_back(StoreVal); 489 unsigned CurOp = 2 + NF; 490 491 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 492 Operands); 493 494 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo( 495 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 496 MachineSDNode *Store = 497 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 498 499 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 500 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 501 502 ReplaceNode(Node, Store); 503 } 504 505 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked, 506 bool IsOrdered) { 507 SDLoc DL(Node); 508 unsigned NF = Node->getNumOperands() - 5; 509 if (IsMasked) 510 --NF; 511 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 512 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 513 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 514 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); 515 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL); 516 517 SmallVector<SDValue, 8> Operands; 518 Operands.push_back(StoreVal); 519 unsigned CurOp = 2 + NF; 520 521 MVT IndexVT; 522 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 523 /*IsStridedOrIndexed*/ true, Operands, 524 /*IsLoad=*/false, &IndexVT); 525 526 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 527 "Element count mismatch"); 528 529 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 530 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 531 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 532 report_fatal_error("The V extension does not support EEW=64 for index " 533 "values when XLEN=32"); 534 } 535 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo( 536 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 537 static_cast<unsigned>(IndexLMUL)); 538 MachineSDNode *Store = 539 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); 540 541 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 542 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 543 544 ReplaceNode(Node, Store); 545 } 546 547 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) { 548 if (!Subtarget->hasVInstructions()) 549 return; 550 551 assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN || 552 Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) && 553 "Unexpected opcode"); 554 555 SDLoc DL(Node); 556 MVT XLenVT = Subtarget->getXLenVT(); 557 558 bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN; 559 unsigned IntNoOffset = HasChain ? 1 : 0; 560 unsigned IntNo = Node->getConstantOperandVal(IntNoOffset); 561 562 assert((IntNo == Intrinsic::riscv_vsetvli || 563 IntNo == Intrinsic::riscv_vsetvlimax || 564 IntNo == Intrinsic::riscv_vsetvli_opt || 565 IntNo == Intrinsic::riscv_vsetvlimax_opt) && 566 "Unexpected vsetvli intrinsic"); 567 568 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax || 569 IntNo == Intrinsic::riscv_vsetvlimax_opt; 570 unsigned Offset = IntNoOffset + (VLMax ? 1 : 2); 571 572 assert(Node->getNumOperands() == Offset + 2 && 573 "Unexpected number of operands"); 574 575 unsigned SEW = 576 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7); 577 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>( 578 Node->getConstantOperandVal(Offset + 1) & 0x7); 579 580 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true, 581 /*MaskAgnostic*/ false); 582 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT); 583 584 SmallVector<EVT, 2> VTs = {XLenVT}; 585 if (HasChain) 586 VTs.push_back(MVT::Other); 587 588 SDValue VLOperand; 589 unsigned Opcode = RISCV::PseudoVSETVLI; 590 if (VLMax) { 591 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT); 592 Opcode = RISCV::PseudoVSETVLIX0; 593 } else { 594 VLOperand = Node->getOperand(IntNoOffset + 1); 595 596 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { 597 uint64_t AVL = C->getZExtValue(); 598 if (isUInt<5>(AVL)) { 599 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); 600 SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp}; 601 if (HasChain) 602 Ops.push_back(Node->getOperand(0)); 603 ReplaceNode( 604 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops)); 605 return; 606 } 607 } 608 } 609 610 SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp}; 611 if (HasChain) 612 Ops.push_back(Node->getOperand(0)); 613 614 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops)); 615 } 616 617 void RISCVDAGToDAGISel::Select(SDNode *Node) { 618 // If we have a custom node, we have already selected. 619 if (Node->isMachineOpcode()) { 620 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 621 Node->setNodeId(-1); 622 return; 623 } 624 625 // Instruction Selection not handled by the auto-generated tablegen selection 626 // should be handled here. 627 unsigned Opcode = Node->getOpcode(); 628 MVT XLenVT = Subtarget->getXLenVT(); 629 SDLoc DL(Node); 630 MVT VT = Node->getSimpleValueType(0); 631 632 switch (Opcode) { 633 case ISD::Constant: { 634 auto *ConstNode = cast<ConstantSDNode>(Node); 635 if (VT == XLenVT && ConstNode->isZero()) { 636 SDValue New = 637 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT); 638 ReplaceNode(Node, New.getNode()); 639 return; 640 } 641 int64_t Imm = ConstNode->getSExtValue(); 642 // If the upper XLen-16 bits are not used, try to convert this to a simm12 643 // by sign extending bit 15. 644 if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) && 645 hasAllHUsers(Node)) 646 Imm = SignExtend64<16>(Imm); 647 // If the upper 32-bits are not used try to convert this into a simm32 by 648 // sign extending bit 32. 649 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node)) 650 Imm = SignExtend64<32>(Imm); 651 652 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget)); 653 return; 654 } 655 case ISD::ADD: { 656 // Try to select ADD + immediate used as memory addresses to 657 // (ADDI (ADD X, Imm-Lo12), Lo12) if it will allow the ADDI to be removed by 658 // doPeepholeLoadStoreADDI. 659 660 // LHS should be an immediate. 661 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 662 if (!N1C) 663 break; 664 665 int64_t Offset = N1C->getSExtValue(); 666 int64_t Lo12 = SignExtend64<12>(Offset); 667 668 // Don't do this if the lower 12 bits are 0 or we could use ADDI directly. 669 if (Lo12 == 0 || isInt<12>(Offset)) 670 break; 671 672 // Don't do this if we can use a pair of ADDIs. 673 if (isInt<12>(Offset / 2) && isInt<12>(Offset - Offset / 2)) 674 break; 675 676 RISCVMatInt::InstSeq Seq = 677 RISCVMatInt::generateInstSeq(Offset, Subtarget->getFeatureBits()); 678 679 Offset -= Lo12; 680 // Restore sign bits for RV32. 681 if (!Subtarget->is64Bit()) 682 Offset = SignExtend64<32>(Offset); 683 684 // We can fold if the last operation is an ADDI or its an ADDIW that could 685 // be treated as an ADDI. 686 if (Seq.back().Opc != RISCV::ADDI && 687 !(Seq.back().Opc == RISCV::ADDIW && isInt<32>(Offset))) 688 break; 689 assert(Seq.back().Imm == Lo12 && "Expected immediate to match Lo12"); 690 // Drop the last operation. 691 Seq.pop_back(); 692 assert(!Seq.empty() && "Expected more instructions in sequence"); 693 694 bool AllPointerUses = true; 695 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 696 SDNode *User = *UI; 697 698 // Is this user a memory instruction that uses a register and immediate 699 // that has this ADD as its pointer. 700 unsigned BaseOpIdx, OffsetOpIdx; 701 if (!User->isMachineOpcode() || 702 !hasMemOffset(User, BaseOpIdx, OffsetOpIdx) || 703 UI.getOperandNo() != BaseOpIdx) { 704 AllPointerUses = false; 705 break; 706 } 707 708 // If the memory instruction already has an offset, make sure the combined 709 // offset is foldable. 710 int64_t MemOffs = 711 cast<ConstantSDNode>(User->getOperand(OffsetOpIdx))->getSExtValue(); 712 MemOffs += Lo12; 713 if (!isInt<12>(MemOffs)) { 714 AllPointerUses = false; 715 break; 716 } 717 } 718 719 if (!AllPointerUses) 720 break; 721 722 // Emit (ADDI (ADD X, Hi), Lo) 723 SDNode *Imm = selectImmSeq(CurDAG, DL, VT, Seq); 724 SDNode *ADD = CurDAG->getMachineNode(RISCV::ADD, DL, VT, 725 Node->getOperand(0), SDValue(Imm, 0)); 726 SDNode *ADDI = 727 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, SDValue(ADD, 0), 728 CurDAG->getTargetConstant(Lo12, DL, VT)); 729 ReplaceNode(Node, ADDI); 730 return; 731 } 732 case ISD::SHL: { 733 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 734 if (!N1C) 735 break; 736 SDValue N0 = Node->getOperand(0); 737 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 738 !isa<ConstantSDNode>(N0.getOperand(1))) 739 break; 740 unsigned ShAmt = N1C->getZExtValue(); 741 uint64_t Mask = N0.getConstantOperandVal(1); 742 743 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C) where C2 has 744 // 32 leading zeros and C3 trailing zeros. 745 if (ShAmt <= 32 && isShiftedMask_64(Mask)) { 746 unsigned XLen = Subtarget->getXLen(); 747 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask)); 748 unsigned TrailingZeros = countTrailingZeros(Mask); 749 if (TrailingZeros > 0 && LeadingZeros == 32) { 750 SDNode *SRLIW = CurDAG->getMachineNode( 751 RISCV::SRLIW, DL, VT, N0->getOperand(0), 752 CurDAG->getTargetConstant(TrailingZeros, DL, VT)); 753 SDNode *SLLI = CurDAG->getMachineNode( 754 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0), 755 CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT)); 756 ReplaceNode(Node, SLLI); 757 return; 758 } 759 } 760 break; 761 } 762 case ISD::SRL: { 763 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 764 if (!N1C) 765 break; 766 SDValue N0 = Node->getOperand(0); 767 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || 768 !isa<ConstantSDNode>(N0.getOperand(1))) 769 break; 770 unsigned ShAmt = N1C->getZExtValue(); 771 uint64_t Mask = N0.getConstantOperandVal(1); 772 773 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has 774 // 32 leading zeros and C3 trailing zeros. 775 if (isShiftedMask_64(Mask)) { 776 unsigned XLen = Subtarget->getXLen(); 777 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask)); 778 unsigned TrailingZeros = countTrailingZeros(Mask); 779 if (LeadingZeros == 32 && TrailingZeros > ShAmt) { 780 SDNode *SRLIW = CurDAG->getMachineNode( 781 RISCV::SRLIW, DL, VT, N0->getOperand(0), 782 CurDAG->getTargetConstant(TrailingZeros, DL, VT)); 783 SDNode *SLLI = CurDAG->getMachineNode( 784 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0), 785 CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT)); 786 ReplaceNode(Node, SLLI); 787 return; 788 } 789 } 790 791 // Optimize (srl (and X, C2), C) -> 792 // (srli (slli X, (XLen-C3), (XLen-C3) + C) 793 // Where C2 is a mask with C3 trailing ones. 794 // Taking into account that the C2 may have had lower bits unset by 795 // SimplifyDemandedBits. This avoids materializing the C2 immediate. 796 // This pattern occurs when type legalizing right shifts for types with 797 // less than XLen bits. 798 Mask |= maskTrailingOnes<uint64_t>(ShAmt); 799 if (!isMask_64(Mask)) 800 break; 801 unsigned TrailingOnes = countTrailingOnes(Mask); 802 // 32 trailing ones should use srliw via tablegen pattern. 803 if (TrailingOnes == 32 || ShAmt >= TrailingOnes) 804 break; 805 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes; 806 SDNode *SLLI = 807 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 808 CurDAG->getTargetConstant(LShAmt, DL, VT)); 809 SDNode *SRLI = CurDAG->getMachineNode( 810 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 811 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 812 ReplaceNode(Node, SRLI); 813 return; 814 } 815 case ISD::SRA: { 816 // Optimize (sra (sext_inreg X, i16), C) -> 817 // (srai (slli X, (XLen-16), (XLen-16) + C) 818 // And (sra (sext_inreg X, i8), C) -> 819 // (srai (slli X, (XLen-8), (XLen-8) + C) 820 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal. 821 // This transform matches the code we get without Zbb. The shifts are more 822 // compressible, and this can help expose CSE opportunities in the sdiv by 823 // constant optimization. 824 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 825 if (!N1C) 826 break; 827 SDValue N0 = Node->getOperand(0); 828 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse()) 829 break; 830 unsigned ShAmt = N1C->getZExtValue(); 831 unsigned ExtSize = 832 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits(); 833 // ExtSize of 32 should use sraiw via tablegen pattern. 834 if (ExtSize >= 32 || ShAmt >= ExtSize) 835 break; 836 unsigned LShAmt = Subtarget->getXLen() - ExtSize; 837 SDNode *SLLI = 838 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0), 839 CurDAG->getTargetConstant(LShAmt, DL, VT)); 840 SDNode *SRAI = CurDAG->getMachineNode( 841 RISCV::SRAI, DL, VT, SDValue(SLLI, 0), 842 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT)); 843 ReplaceNode(Node, SRAI); 844 return; 845 } 846 case ISD::AND: { 847 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 848 if (!N1C) 849 break; 850 851 SDValue N0 = Node->getOperand(0); 852 853 bool LeftShift = N0.getOpcode() == ISD::SHL; 854 if (!LeftShift && N0.getOpcode() != ISD::SRL) 855 break; 856 857 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 858 if (!C) 859 break; 860 unsigned C2 = C->getZExtValue(); 861 unsigned XLen = Subtarget->getXLen(); 862 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!"); 863 864 uint64_t C1 = N1C->getZExtValue(); 865 866 // Keep track of whether this is a c.andi. If we can't use c.andi, the 867 // shift pair might offer more compression opportunities. 868 // TODO: We could check for C extension here, but we don't have many lit 869 // tests with the C extension enabled so not checking gets better coverage. 870 // TODO: What if ANDI faster than shift? 871 bool IsCANDI = isInt<6>(N1C->getSExtValue()); 872 873 // Clear irrelevant bits in the mask. 874 if (LeftShift) 875 C1 &= maskTrailingZeros<uint64_t>(C2); 876 else 877 C1 &= maskTrailingOnes<uint64_t>(XLen - C2); 878 879 // Some transforms should only be done if the shift has a single use or 880 // the AND would become (srli (slli X, 32), 32) 881 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF); 882 883 SDValue X = N0.getOperand(0); 884 885 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask 886 // with c3 leading zeros. 887 if (!LeftShift && isMask_64(C1)) { 888 unsigned Leading = XLen - (64 - countLeadingZeros(C1)); 889 if (C2 < Leading) { 890 // If the number of leading zeros is C2+32 this can be SRLIW. 891 if (C2 + 32 == Leading) { 892 SDNode *SRLIW = CurDAG->getMachineNode( 893 RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT)); 894 ReplaceNode(Node, SRLIW); 895 return; 896 } 897 898 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if 899 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1. 900 // 901 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type 902 // legalized and goes through DAG combine. 903 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() && 904 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 905 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) { 906 SDNode *SRAIW = 907 CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0), 908 CurDAG->getTargetConstant(31, DL, VT)); 909 SDNode *SRLIW = CurDAG->getMachineNode( 910 RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0), 911 CurDAG->getTargetConstant(Leading - 32, DL, VT)); 912 ReplaceNode(Node, SRLIW); 913 return; 914 } 915 916 // (srli (slli x, c3-c2), c3). 917 // Skip if we could use (zext.w (sraiw X, C2)). 918 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 && 919 X.getOpcode() == ISD::SIGN_EXTEND_INREG && 920 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32; 921 // Also Skip if we can use bexti. 922 Skip |= Subtarget->hasStdExtZbs() && Leading == XLen - 1; 923 if (OneUseOrZExtW && !Skip) { 924 SDNode *SLLI = CurDAG->getMachineNode( 925 RISCV::SLLI, DL, VT, X, 926 CurDAG->getTargetConstant(Leading - C2, DL, VT)); 927 SDNode *SRLI = CurDAG->getMachineNode( 928 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 929 CurDAG->getTargetConstant(Leading, DL, VT)); 930 ReplaceNode(Node, SRLI); 931 return; 932 } 933 } 934 } 935 936 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask 937 // shifted by c2 bits with c3 leading zeros. 938 if (LeftShift && isShiftedMask_64(C1)) { 939 unsigned Leading = XLen - (64 - countLeadingZeros(C1)); 940 941 if (C2 + Leading < XLen && 942 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) { 943 // Use slli.uw when possible. 944 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) { 945 SDNode *SLLI_UW = CurDAG->getMachineNode( 946 RISCV::SLLI_UW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT)); 947 ReplaceNode(Node, SLLI_UW); 948 return; 949 } 950 951 // (srli (slli c2+c3), c3) 952 if (OneUseOrZExtW && !IsCANDI) { 953 SDNode *SLLI = CurDAG->getMachineNode( 954 RISCV::SLLI, DL, VT, X, 955 CurDAG->getTargetConstant(C2 + Leading, DL, VT)); 956 SDNode *SRLI = CurDAG->getMachineNode( 957 RISCV::SRLI, DL, VT, SDValue(SLLI, 0), 958 CurDAG->getTargetConstant(Leading, DL, VT)); 959 ReplaceNode(Node, SRLI); 960 return; 961 } 962 } 963 } 964 965 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a 966 // shifted mask with c2 leading zeros and c3 trailing zeros. 967 if (!LeftShift && isShiftedMask_64(C1)) { 968 unsigned Leading = XLen - (64 - countLeadingZeros(C1)); 969 unsigned Trailing = countTrailingZeros(C1); 970 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW && !IsCANDI) { 971 unsigned SrliOpc = RISCV::SRLI; 972 // If the input is zexti32 we should use SRLIW. 973 if (X.getOpcode() == ISD::AND && isa<ConstantSDNode>(X.getOperand(1)) && 974 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) { 975 SrliOpc = RISCV::SRLIW; 976 X = X.getOperand(0); 977 } 978 SDNode *SRLI = CurDAG->getMachineNode( 979 SrliOpc, DL, VT, X, 980 CurDAG->getTargetConstant(C2 + Trailing, DL, VT)); 981 SDNode *SLLI = 982 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, SDValue(SRLI, 0), 983 CurDAG->getTargetConstant(Trailing, DL, VT)); 984 ReplaceNode(Node, SLLI); 985 return; 986 } 987 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI. 988 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 && 989 OneUseOrZExtW && !IsCANDI) { 990 SDNode *SRLIW = CurDAG->getMachineNode( 991 RISCV::SRLIW, DL, VT, X, 992 CurDAG->getTargetConstant(C2 + Trailing, DL, VT)); 993 SDNode *SLLI = 994 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, SDValue(SRLIW, 0), 995 CurDAG->getTargetConstant(Trailing, DL, VT)); 996 ReplaceNode(Node, SLLI); 997 return; 998 } 999 } 1000 1001 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a 1002 // shifted mask with no leading zeros and c3 trailing zeros. 1003 if (LeftShift && isShiftedMask_64(C1)) { 1004 unsigned Leading = XLen - (64 - countLeadingZeros(C1)); 1005 unsigned Trailing = countTrailingZeros(C1); 1006 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) { 1007 SDNode *SRLI = CurDAG->getMachineNode( 1008 RISCV::SRLI, DL, VT, X, 1009 CurDAG->getTargetConstant(Trailing - C2, DL, VT)); 1010 SDNode *SLLI = 1011 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, SDValue(SRLI, 0), 1012 CurDAG->getTargetConstant(Trailing, DL, VT)); 1013 ReplaceNode(Node, SLLI); 1014 return; 1015 } 1016 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI. 1017 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) { 1018 SDNode *SRLIW = CurDAG->getMachineNode( 1019 RISCV::SRLIW, DL, VT, X, 1020 CurDAG->getTargetConstant(Trailing - C2, DL, VT)); 1021 SDNode *SLLI = 1022 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, SDValue(SRLIW, 0), 1023 CurDAG->getTargetConstant(Trailing, DL, VT)); 1024 ReplaceNode(Node, SLLI); 1025 return; 1026 } 1027 } 1028 1029 break; 1030 } 1031 case ISD::MUL: { 1032 // Special case for calculating (mul (and X, C2), C1) where the full product 1033 // fits in XLen bits. We can shift X left by the number of leading zeros in 1034 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final 1035 // product has XLen trailing zeros, putting it in the output of MULHU. This 1036 // can avoid materializing a constant in a register for C2. 1037 1038 // RHS should be a constant. 1039 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1040 if (!N1C || !N1C->hasOneUse()) 1041 break; 1042 1043 // LHS should be an AND with constant. 1044 SDValue N0 = Node->getOperand(0); 1045 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) 1046 break; 1047 1048 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 1049 1050 // Constant should be a mask. 1051 if (!isMask_64(C2)) 1052 break; 1053 1054 // This should be the only use of the AND unless we will use 1055 // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND 1056 // constants. 1057 if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) 1058 break; 1059 1060 // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this 1061 // optimization. 1062 if (isInt<12>(C2) || 1063 (C2 == UINT64_C(0xFFFF) && 1064 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || 1065 (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) 1066 break; 1067 1068 // We need to shift left the AND input and C1 by a total of XLen bits. 1069 1070 // How far left do we need to shift the AND input? 1071 unsigned XLen = Subtarget->getXLen(); 1072 unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); 1073 1074 // The constant gets shifted by the remaining amount unless that would 1075 // shift bits out. 1076 uint64_t C1 = N1C->getZExtValue(); 1077 unsigned ConstantShift = XLen - LeadingZeros; 1078 if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) 1079 break; 1080 1081 uint64_t ShiftedC1 = C1 << ConstantShift; 1082 // If this RV32, we need to sign extend the constant. 1083 if (XLen == 32) 1084 ShiftedC1 = SignExtend64<32>(ShiftedC1); 1085 1086 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). 1087 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget); 1088 SDNode *SLLI = 1089 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), 1090 CurDAG->getTargetConstant(LeadingZeros, DL, VT)); 1091 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, 1092 SDValue(SLLI, 0), SDValue(Imm, 0)); 1093 ReplaceNode(Node, MULHU); 1094 return; 1095 } 1096 case ISD::INTRINSIC_WO_CHAIN: { 1097 unsigned IntNo = Node->getConstantOperandVal(0); 1098 switch (IntNo) { 1099 // By default we do not custom select any intrinsic. 1100 default: 1101 break; 1102 case Intrinsic::riscv_vmsgeu: 1103 case Intrinsic::riscv_vmsge: { 1104 SDValue Src1 = Node->getOperand(1); 1105 SDValue Src2 = Node->getOperand(2); 1106 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu; 1107 bool IsCmpUnsignedZero = false; 1108 // Only custom select scalar second operand. 1109 if (Src2.getValueType() != XLenVT) 1110 break; 1111 // Small constants are handled with patterns. 1112 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1113 int64_t CVal = C->getSExtValue(); 1114 if (CVal >= -15 && CVal <= 16) { 1115 if (!IsUnsigned || CVal != 0) 1116 break; 1117 IsCmpUnsignedZero = true; 1118 } 1119 } 1120 MVT Src1VT = Src1.getSimpleValueType(); 1121 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode; 1122 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1123 default: 1124 llvm_unreachable("Unexpected LMUL!"); 1125 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \ 1126 case RISCVII::VLMUL::lmulenum: \ 1127 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1128 : RISCV::PseudoVMSLT_VX_##suffix; \ 1129 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \ 1130 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \ 1131 break; 1132 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1) 1133 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2) 1134 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4) 1135 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8) 1136 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16) 1137 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32) 1138 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64) 1139 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES 1140 } 1141 SDValue SEW = CurDAG->getTargetConstant( 1142 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1143 SDValue VL; 1144 selectVLOp(Node->getOperand(3), VL); 1145 1146 // If vmsgeu with 0 immediate, expand it to vmset. 1147 if (IsCmpUnsignedZero) { 1148 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW)); 1149 return; 1150 } 1151 1152 // Expand to 1153 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd 1154 SDValue Cmp = SDValue( 1155 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1156 0); 1157 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT, 1158 {Cmp, Cmp, VL, SEW})); 1159 return; 1160 } 1161 case Intrinsic::riscv_vmsgeu_mask: 1162 case Intrinsic::riscv_vmsge_mask: { 1163 SDValue Src1 = Node->getOperand(2); 1164 SDValue Src2 = Node->getOperand(3); 1165 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask; 1166 bool IsCmpUnsignedZero = false; 1167 // Only custom select scalar second operand. 1168 if (Src2.getValueType() != XLenVT) 1169 break; 1170 // Small constants are handled with patterns. 1171 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) { 1172 int64_t CVal = C->getSExtValue(); 1173 if (CVal >= -15 && CVal <= 16) { 1174 if (!IsUnsigned || CVal != 0) 1175 break; 1176 IsCmpUnsignedZero = true; 1177 } 1178 } 1179 MVT Src1VT = Src1.getSimpleValueType(); 1180 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode, 1181 VMOROpcode; 1182 switch (RISCVTargetLowering::getLMUL(Src1VT)) { 1183 default: 1184 llvm_unreachable("Unexpected LMUL!"); 1185 #define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \ 1186 case RISCVII::VLMUL::lmulenum: \ 1187 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \ 1188 : RISCV::PseudoVMSLT_VX_##suffix; \ 1189 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \ 1190 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \ 1191 break; 1192 CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1) 1193 CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2) 1194 CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4) 1195 CASE_VMSLT_OPCODES(LMUL_1, M1, B8) 1196 CASE_VMSLT_OPCODES(LMUL_2, M2, B16) 1197 CASE_VMSLT_OPCODES(LMUL_4, M4, B32) 1198 CASE_VMSLT_OPCODES(LMUL_8, M8, B64) 1199 #undef CASE_VMSLT_OPCODES 1200 } 1201 // Mask operations use the LMUL from the mask type. 1202 switch (RISCVTargetLowering::getLMUL(VT)) { 1203 default: 1204 llvm_unreachable("Unexpected LMUL!"); 1205 #define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \ 1206 case RISCVII::VLMUL::lmulenum: \ 1207 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \ 1208 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \ 1209 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \ 1210 break; 1211 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8) 1212 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4) 1213 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2) 1214 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, M1) 1215 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, M2) 1216 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, M4) 1217 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, M8) 1218 #undef CASE_VMXOR_VMANDN_VMOR_OPCODES 1219 } 1220 SDValue SEW = CurDAG->getTargetConstant( 1221 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT); 1222 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT); 1223 SDValue VL; 1224 selectVLOp(Node->getOperand(5), VL); 1225 SDValue MaskedOff = Node->getOperand(1); 1226 SDValue Mask = Node->getOperand(4); 1227 1228 // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff. 1229 if (IsCmpUnsignedZero) { 1230 // We don't need vmor if the MaskedOff and the Mask are the same 1231 // value. 1232 if (Mask == MaskedOff) { 1233 ReplaceUses(Node, Mask.getNode()); 1234 return; 1235 } 1236 ReplaceNode(Node, 1237 CurDAG->getMachineNode(VMOROpcode, DL, VT, 1238 {Mask, MaskedOff, VL, MaskSEW})); 1239 return; 1240 } 1241 1242 // If the MaskedOff value and the Mask are the same value use 1243 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt 1244 // This avoids needing to copy v0 to vd before starting the next sequence. 1245 if (Mask == MaskedOff) { 1246 SDValue Cmp = SDValue( 1247 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}), 1248 0); 1249 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT, 1250 {Mask, Cmp, VL, MaskSEW})); 1251 return; 1252 } 1253 1254 // Mask needs to be copied to V0. 1255 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, 1256 RISCV::V0, Mask, SDValue()); 1257 SDValue Glue = Chain.getValue(1); 1258 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); 1259 1260 // Otherwise use 1261 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0 1262 // The result is mask undisturbed. 1263 // We use the same instructions to emulate mask agnostic behavior, because 1264 // the agnostic result can be either undisturbed or all 1. 1265 SDValue Cmp = SDValue( 1266 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, 1267 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), 1268 0); 1269 // vmxor.mm vd, vd, v0 is used to update active value. 1270 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, 1271 {Cmp, Mask, VL, MaskSEW})); 1272 return; 1273 } 1274 case Intrinsic::riscv_vsetvli_opt: 1275 case Intrinsic::riscv_vsetvlimax_opt: 1276 return selectVSETVLI(Node); 1277 } 1278 break; 1279 } 1280 case ISD::INTRINSIC_W_CHAIN: { 1281 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1282 switch (IntNo) { 1283 // By default we do not custom select any intrinsic. 1284 default: 1285 break; 1286 case Intrinsic::riscv_vsetvli: 1287 case Intrinsic::riscv_vsetvlimax: 1288 return selectVSETVLI(Node); 1289 case Intrinsic::riscv_vlseg2: 1290 case Intrinsic::riscv_vlseg3: 1291 case Intrinsic::riscv_vlseg4: 1292 case Intrinsic::riscv_vlseg5: 1293 case Intrinsic::riscv_vlseg6: 1294 case Intrinsic::riscv_vlseg7: 1295 case Intrinsic::riscv_vlseg8: { 1296 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1297 return; 1298 } 1299 case Intrinsic::riscv_vlseg2_mask: 1300 case Intrinsic::riscv_vlseg3_mask: 1301 case Intrinsic::riscv_vlseg4_mask: 1302 case Intrinsic::riscv_vlseg5_mask: 1303 case Intrinsic::riscv_vlseg6_mask: 1304 case Intrinsic::riscv_vlseg7_mask: 1305 case Intrinsic::riscv_vlseg8_mask: { 1306 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1307 return; 1308 } 1309 case Intrinsic::riscv_vlsseg2: 1310 case Intrinsic::riscv_vlsseg3: 1311 case Intrinsic::riscv_vlsseg4: 1312 case Intrinsic::riscv_vlsseg5: 1313 case Intrinsic::riscv_vlsseg6: 1314 case Intrinsic::riscv_vlsseg7: 1315 case Intrinsic::riscv_vlsseg8: { 1316 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1317 return; 1318 } 1319 case Intrinsic::riscv_vlsseg2_mask: 1320 case Intrinsic::riscv_vlsseg3_mask: 1321 case Intrinsic::riscv_vlsseg4_mask: 1322 case Intrinsic::riscv_vlsseg5_mask: 1323 case Intrinsic::riscv_vlsseg6_mask: 1324 case Intrinsic::riscv_vlsseg7_mask: 1325 case Intrinsic::riscv_vlsseg8_mask: { 1326 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1327 return; 1328 } 1329 case Intrinsic::riscv_vloxseg2: 1330 case Intrinsic::riscv_vloxseg3: 1331 case Intrinsic::riscv_vloxseg4: 1332 case Intrinsic::riscv_vloxseg5: 1333 case Intrinsic::riscv_vloxseg6: 1334 case Intrinsic::riscv_vloxseg7: 1335 case Intrinsic::riscv_vloxseg8: 1336 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1337 return; 1338 case Intrinsic::riscv_vluxseg2: 1339 case Intrinsic::riscv_vluxseg3: 1340 case Intrinsic::riscv_vluxseg4: 1341 case Intrinsic::riscv_vluxseg5: 1342 case Intrinsic::riscv_vluxseg6: 1343 case Intrinsic::riscv_vluxseg7: 1344 case Intrinsic::riscv_vluxseg8: 1345 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1346 return; 1347 case Intrinsic::riscv_vloxseg2_mask: 1348 case Intrinsic::riscv_vloxseg3_mask: 1349 case Intrinsic::riscv_vloxseg4_mask: 1350 case Intrinsic::riscv_vloxseg5_mask: 1351 case Intrinsic::riscv_vloxseg6_mask: 1352 case Intrinsic::riscv_vloxseg7_mask: 1353 case Intrinsic::riscv_vloxseg8_mask: 1354 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1355 return; 1356 case Intrinsic::riscv_vluxseg2_mask: 1357 case Intrinsic::riscv_vluxseg3_mask: 1358 case Intrinsic::riscv_vluxseg4_mask: 1359 case Intrinsic::riscv_vluxseg5_mask: 1360 case Intrinsic::riscv_vluxseg6_mask: 1361 case Intrinsic::riscv_vluxseg7_mask: 1362 case Intrinsic::riscv_vluxseg8_mask: 1363 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1364 return; 1365 case Intrinsic::riscv_vlseg8ff: 1366 case Intrinsic::riscv_vlseg7ff: 1367 case Intrinsic::riscv_vlseg6ff: 1368 case Intrinsic::riscv_vlseg5ff: 1369 case Intrinsic::riscv_vlseg4ff: 1370 case Intrinsic::riscv_vlseg3ff: 1371 case Intrinsic::riscv_vlseg2ff: { 1372 selectVLSEGFF(Node, /*IsMasked*/ false); 1373 return; 1374 } 1375 case Intrinsic::riscv_vlseg8ff_mask: 1376 case Intrinsic::riscv_vlseg7ff_mask: 1377 case Intrinsic::riscv_vlseg6ff_mask: 1378 case Intrinsic::riscv_vlseg5ff_mask: 1379 case Intrinsic::riscv_vlseg4ff_mask: 1380 case Intrinsic::riscv_vlseg3ff_mask: 1381 case Intrinsic::riscv_vlseg2ff_mask: { 1382 selectVLSEGFF(Node, /*IsMasked*/ true); 1383 return; 1384 } 1385 case Intrinsic::riscv_vloxei: 1386 case Intrinsic::riscv_vloxei_mask: 1387 case Intrinsic::riscv_vluxei: 1388 case Intrinsic::riscv_vluxei_mask: { 1389 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask || 1390 IntNo == Intrinsic::riscv_vluxei_mask; 1391 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei || 1392 IntNo == Intrinsic::riscv_vloxei_mask; 1393 1394 MVT VT = Node->getSimpleValueType(0); 1395 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1396 1397 unsigned CurOp = 2; 1398 // Masked intrinsic only have TU version pseduo instructions. 1399 bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef(); 1400 SmallVector<SDValue, 8> Operands; 1401 if (IsTU) 1402 Operands.push_back(Node->getOperand(CurOp++)); 1403 else 1404 // Skip the undef passthru operand for nomask TA version pseudo 1405 CurOp++; 1406 1407 MVT IndexVT; 1408 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1409 /*IsStridedOrIndexed*/ true, Operands, 1410 /*IsLoad=*/true, &IndexVT); 1411 1412 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1413 "Element count mismatch"); 1414 1415 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1416 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1417 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1418 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1419 report_fatal_error("The V extension does not support EEW=64 for index " 1420 "values when XLEN=32"); 1421 } 1422 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( 1423 IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL), 1424 static_cast<unsigned>(IndexLMUL)); 1425 MachineSDNode *Load = 1426 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1427 1428 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1429 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1430 1431 ReplaceNode(Node, Load); 1432 return; 1433 } 1434 case Intrinsic::riscv_vlm: 1435 case Intrinsic::riscv_vle: 1436 case Intrinsic::riscv_vle_mask: 1437 case Intrinsic::riscv_vlse: 1438 case Intrinsic::riscv_vlse_mask: { 1439 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask || 1440 IntNo == Intrinsic::riscv_vlse_mask; 1441 bool IsStrided = 1442 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask; 1443 1444 MVT VT = Node->getSimpleValueType(0); 1445 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1446 1447 unsigned CurOp = 2; 1448 // The riscv_vlm intrinsic are always tail agnostic and no passthru operand. 1449 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm; 1450 // Masked intrinsic only have TU version pseduo instructions. 1451 bool IsTU = HasPassthruOperand && 1452 (IsMasked || !Node->getOperand(CurOp).isUndef()); 1453 SmallVector<SDValue, 8> Operands; 1454 if (IsTU) 1455 Operands.push_back(Node->getOperand(CurOp++)); 1456 else if (HasPassthruOperand) 1457 // Skip the undef passthru operand for nomask TA version pseudo 1458 CurOp++; 1459 1460 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1461 Operands, /*IsLoad=*/true); 1462 1463 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1464 const RISCV::VLEPseudo *P = 1465 RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW, 1466 static_cast<unsigned>(LMUL)); 1467 MachineSDNode *Load = 1468 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1469 1470 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1471 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1472 1473 ReplaceNode(Node, Load); 1474 return; 1475 } 1476 case Intrinsic::riscv_vleff: 1477 case Intrinsic::riscv_vleff_mask: { 1478 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask; 1479 1480 MVT VT = Node->getSimpleValueType(0); 1481 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1482 1483 unsigned CurOp = 2; 1484 // Masked intrinsic only have TU version pseduo instructions. 1485 bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef(); 1486 SmallVector<SDValue, 7> Operands; 1487 if (IsTU) 1488 Operands.push_back(Node->getOperand(CurOp++)); 1489 else 1490 // Skip the undef passthru operand for nomask TA version pseudo 1491 CurOp++; 1492 1493 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1494 /*IsStridedOrIndexed*/ false, Operands, 1495 /*IsLoad=*/true); 1496 1497 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1498 const RISCV::VLEPseudo *P = 1499 RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true, 1500 Log2SEW, static_cast<unsigned>(LMUL)); 1501 MachineSDNode *Load = CurDAG->getMachineNode( 1502 P->Pseudo, DL, Node->getVTList(), Operands); 1503 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1504 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()}); 1505 1506 ReplaceNode(Node, Load); 1507 return; 1508 } 1509 } 1510 break; 1511 } 1512 case ISD::INTRINSIC_VOID: { 1513 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1514 switch (IntNo) { 1515 case Intrinsic::riscv_vsseg2: 1516 case Intrinsic::riscv_vsseg3: 1517 case Intrinsic::riscv_vsseg4: 1518 case Intrinsic::riscv_vsseg5: 1519 case Intrinsic::riscv_vsseg6: 1520 case Intrinsic::riscv_vsseg7: 1521 case Intrinsic::riscv_vsseg8: { 1522 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false); 1523 return; 1524 } 1525 case Intrinsic::riscv_vsseg2_mask: 1526 case Intrinsic::riscv_vsseg3_mask: 1527 case Intrinsic::riscv_vsseg4_mask: 1528 case Intrinsic::riscv_vsseg5_mask: 1529 case Intrinsic::riscv_vsseg6_mask: 1530 case Intrinsic::riscv_vsseg7_mask: 1531 case Intrinsic::riscv_vsseg8_mask: { 1532 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false); 1533 return; 1534 } 1535 case Intrinsic::riscv_vssseg2: 1536 case Intrinsic::riscv_vssseg3: 1537 case Intrinsic::riscv_vssseg4: 1538 case Intrinsic::riscv_vssseg5: 1539 case Intrinsic::riscv_vssseg6: 1540 case Intrinsic::riscv_vssseg7: 1541 case Intrinsic::riscv_vssseg8: { 1542 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true); 1543 return; 1544 } 1545 case Intrinsic::riscv_vssseg2_mask: 1546 case Intrinsic::riscv_vssseg3_mask: 1547 case Intrinsic::riscv_vssseg4_mask: 1548 case Intrinsic::riscv_vssseg5_mask: 1549 case Intrinsic::riscv_vssseg6_mask: 1550 case Intrinsic::riscv_vssseg7_mask: 1551 case Intrinsic::riscv_vssseg8_mask: { 1552 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true); 1553 return; 1554 } 1555 case Intrinsic::riscv_vsoxseg2: 1556 case Intrinsic::riscv_vsoxseg3: 1557 case Intrinsic::riscv_vsoxseg4: 1558 case Intrinsic::riscv_vsoxseg5: 1559 case Intrinsic::riscv_vsoxseg6: 1560 case Intrinsic::riscv_vsoxseg7: 1561 case Intrinsic::riscv_vsoxseg8: 1562 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true); 1563 return; 1564 case Intrinsic::riscv_vsuxseg2: 1565 case Intrinsic::riscv_vsuxseg3: 1566 case Intrinsic::riscv_vsuxseg4: 1567 case Intrinsic::riscv_vsuxseg5: 1568 case Intrinsic::riscv_vsuxseg6: 1569 case Intrinsic::riscv_vsuxseg7: 1570 case Intrinsic::riscv_vsuxseg8: 1571 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false); 1572 return; 1573 case Intrinsic::riscv_vsoxseg2_mask: 1574 case Intrinsic::riscv_vsoxseg3_mask: 1575 case Intrinsic::riscv_vsoxseg4_mask: 1576 case Intrinsic::riscv_vsoxseg5_mask: 1577 case Intrinsic::riscv_vsoxseg6_mask: 1578 case Intrinsic::riscv_vsoxseg7_mask: 1579 case Intrinsic::riscv_vsoxseg8_mask: 1580 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true); 1581 return; 1582 case Intrinsic::riscv_vsuxseg2_mask: 1583 case Intrinsic::riscv_vsuxseg3_mask: 1584 case Intrinsic::riscv_vsuxseg4_mask: 1585 case Intrinsic::riscv_vsuxseg5_mask: 1586 case Intrinsic::riscv_vsuxseg6_mask: 1587 case Intrinsic::riscv_vsuxseg7_mask: 1588 case Intrinsic::riscv_vsuxseg8_mask: 1589 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false); 1590 return; 1591 case Intrinsic::riscv_vsoxei: 1592 case Intrinsic::riscv_vsoxei_mask: 1593 case Intrinsic::riscv_vsuxei: 1594 case Intrinsic::riscv_vsuxei_mask: { 1595 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask || 1596 IntNo == Intrinsic::riscv_vsuxei_mask; 1597 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei || 1598 IntNo == Intrinsic::riscv_vsoxei_mask; 1599 1600 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1601 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1602 1603 unsigned CurOp = 2; 1604 SmallVector<SDValue, 8> Operands; 1605 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1606 1607 MVT IndexVT; 1608 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, 1609 /*IsStridedOrIndexed*/ true, Operands, 1610 /*IsLoad=*/false, &IndexVT); 1611 1612 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() && 1613 "Element count mismatch"); 1614 1615 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1616 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT); 1617 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits()); 1618 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { 1619 report_fatal_error("The V extension does not support EEW=64 for index " 1620 "values when XLEN=32"); 1621 } 1622 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( 1623 IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW, 1624 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL)); 1625 MachineSDNode *Store = 1626 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1627 1628 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1629 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1630 1631 ReplaceNode(Node, Store); 1632 return; 1633 } 1634 case Intrinsic::riscv_vsm: 1635 case Intrinsic::riscv_vse: 1636 case Intrinsic::riscv_vse_mask: 1637 case Intrinsic::riscv_vsse: 1638 case Intrinsic::riscv_vsse_mask: { 1639 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask || 1640 IntNo == Intrinsic::riscv_vsse_mask; 1641 bool IsStrided = 1642 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask; 1643 1644 MVT VT = Node->getOperand(2)->getSimpleValueType(0); 1645 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1646 1647 unsigned CurOp = 2; 1648 SmallVector<SDValue, 8> Operands; 1649 Operands.push_back(Node->getOperand(CurOp++)); // Store value. 1650 1651 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided, 1652 Operands); 1653 1654 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1655 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( 1656 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL)); 1657 MachineSDNode *Store = 1658 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1659 if (auto *MemOp = dyn_cast<MemSDNode>(Node)) 1660 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()}); 1661 1662 ReplaceNode(Node, Store); 1663 return; 1664 } 1665 } 1666 break; 1667 } 1668 case ISD::BITCAST: { 1669 MVT SrcVT = Node->getOperand(0).getSimpleValueType(); 1670 // Just drop bitcasts between vectors if both are fixed or both are 1671 // scalable. 1672 if ((VT.isScalableVector() && SrcVT.isScalableVector()) || 1673 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) { 1674 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); 1675 CurDAG->RemoveDeadNode(Node); 1676 return; 1677 } 1678 break; 1679 } 1680 case ISD::INSERT_SUBVECTOR: { 1681 SDValue V = Node->getOperand(0); 1682 SDValue SubV = Node->getOperand(1); 1683 SDLoc DL(SubV); 1684 auto Idx = Node->getConstantOperandVal(2); 1685 MVT SubVecVT = SubV.getSimpleValueType(); 1686 1687 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1688 MVT SubVecContainerVT = SubVecVT; 1689 // Establish the correct scalable-vector types for any fixed-length type. 1690 if (SubVecVT.isFixedLengthVector()) 1691 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT); 1692 if (VT.isFixedLengthVector()) 1693 VT = TLI.getContainerForFixedLengthVector(VT); 1694 1695 const auto *TRI = Subtarget->getRegisterInfo(); 1696 unsigned SubRegIdx; 1697 std::tie(SubRegIdx, Idx) = 1698 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1699 VT, SubVecContainerVT, Idx, TRI); 1700 1701 // If the Idx hasn't been completely eliminated then this is a subvector 1702 // insert which doesn't naturally align to a vector register. These must 1703 // be handled using instructions to manipulate the vector registers. 1704 if (Idx != 0) 1705 break; 1706 1707 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT); 1708 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 || 1709 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 || 1710 SubVecLMUL == RISCVII::VLMUL::LMUL_F8; 1711 (void)IsSubVecPartReg; // Silence unused variable warning without asserts. 1712 assert((!IsSubVecPartReg || V.isUndef()) && 1713 "Expecting lowering to have created legal INSERT_SUBVECTORs when " 1714 "the subvector is smaller than a full-sized register"); 1715 1716 // If we haven't set a SubRegIdx, then we must be going between 1717 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. 1718 if (SubRegIdx == RISCV::NoSubRegister) { 1719 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT); 1720 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1721 InRegClassID && 1722 "Unexpected subvector extraction"); 1723 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1724 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1725 DL, VT, SubV, RC); 1726 ReplaceNode(Node, NewNode); 1727 return; 1728 } 1729 1730 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV); 1731 ReplaceNode(Node, Insert.getNode()); 1732 return; 1733 } 1734 case ISD::EXTRACT_SUBVECTOR: { 1735 SDValue V = Node->getOperand(0); 1736 auto Idx = Node->getConstantOperandVal(1); 1737 MVT InVT = V.getSimpleValueType(); 1738 SDLoc DL(V); 1739 1740 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering(); 1741 MVT SubVecContainerVT = VT; 1742 // Establish the correct scalable-vector types for any fixed-length type. 1743 if (VT.isFixedLengthVector()) 1744 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT); 1745 if (InVT.isFixedLengthVector()) 1746 InVT = TLI.getContainerForFixedLengthVector(InVT); 1747 1748 const auto *TRI = Subtarget->getRegisterInfo(); 1749 unsigned SubRegIdx; 1750 std::tie(SubRegIdx, Idx) = 1751 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( 1752 InVT, SubVecContainerVT, Idx, TRI); 1753 1754 // If the Idx hasn't been completely eliminated then this is a subvector 1755 // extract which doesn't naturally align to a vector register. These must 1756 // be handled using instructions to manipulate the vector registers. 1757 if (Idx != 0) 1758 break; 1759 1760 // If we haven't set a SubRegIdx, then we must be going between 1761 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy. 1762 if (SubRegIdx == RISCV::NoSubRegister) { 1763 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT); 1764 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) == 1765 InRegClassID && 1766 "Unexpected subvector extraction"); 1767 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT); 1768 SDNode *NewNode = 1769 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC); 1770 ReplaceNode(Node, NewNode); 1771 return; 1772 } 1773 1774 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V); 1775 ReplaceNode(Node, Extract.getNode()); 1776 return; 1777 } 1778 case ISD::SPLAT_VECTOR: 1779 case RISCVISD::VMV_S_X_VL: 1780 case RISCVISD::VFMV_S_F_VL: 1781 case RISCVISD::VMV_V_X_VL: 1782 case RISCVISD::VFMV_V_F_VL: { 1783 // Try to match splat of a scalar load to a strided load with stride of x0. 1784 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL || 1785 Node->getOpcode() == RISCVISD::VFMV_S_F_VL; 1786 bool HasPassthruOperand = Node->getOpcode() != ISD::SPLAT_VECTOR; 1787 if (HasPassthruOperand && !Node->getOperand(0).isUndef()) 1788 break; 1789 SDValue Src = HasPassthruOperand ? Node->getOperand(1) : Node->getOperand(0); 1790 auto *Ld = dyn_cast<LoadSDNode>(Src); 1791 if (!Ld) 1792 break; 1793 EVT MemVT = Ld->getMemoryVT(); 1794 // The memory VT should be the same size as the element type. 1795 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize()) 1796 break; 1797 if (!IsProfitableToFold(Src, Node, Node) || 1798 !IsLegalToFold(Src, Node, Node, TM.getOptLevel())) 1799 break; 1800 1801 SDValue VL; 1802 if (Node->getOpcode() == ISD::SPLAT_VECTOR) 1803 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); 1804 else if (IsScalarMove) { 1805 // We could deal with more VL if we update the VSETVLI insert pass to 1806 // avoid introducing more VSETVLI. 1807 if (!isOneConstant(Node->getOperand(2))) 1808 break; 1809 selectVLOp(Node->getOperand(2), VL); 1810 } else 1811 selectVLOp(Node->getOperand(2), VL); 1812 1813 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); 1814 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); 1815 1816 SDValue Operands[] = {Ld->getBasePtr(), 1817 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW, 1818 Ld->getChain()}; 1819 1820 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT); 1821 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo( 1822 /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false, 1823 Log2SEW, static_cast<unsigned>(LMUL)); 1824 MachineSDNode *Load = 1825 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands); 1826 1827 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()}); 1828 1829 ReplaceNode(Node, Load); 1830 return; 1831 } 1832 } 1833 1834 // Select the default instruction. 1835 SelectCode(Node); 1836 } 1837 1838 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand( 1839 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 1840 switch (ConstraintID) { 1841 case InlineAsm::Constraint_m: 1842 // We just support simple memory operands that have a single address 1843 // operand and need no special handling. 1844 OutOps.push_back(Op); 1845 return false; 1846 case InlineAsm::Constraint_A: 1847 OutOps.push_back(Op); 1848 return false; 1849 default: 1850 break; 1851 } 1852 1853 return true; 1854 } 1855 1856 bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base, 1857 SDValue &Offset) { 1858 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1859 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT()); 1860 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT()); 1861 return true; 1862 } 1863 1864 return false; 1865 } 1866 1867 // Select a frame index and an optional immediate offset from an ADD or OR. 1868 bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, 1869 SDValue &Offset) { 1870 if (SelectAddrFrameIndex(Addr, Base, Offset)) 1871 return true; 1872 1873 if (!CurDAG->isBaseWithConstantOffset(Addr)) 1874 return false; 1875 1876 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) { 1877 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue(); 1878 if (isInt<12>(CVal)) { 1879 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), 1880 Subtarget->getXLenVT()); 1881 Offset = CurDAG->getTargetConstant(CVal, SDLoc(Addr), 1882 Subtarget->getXLenVT()); 1883 return true; 1884 } 1885 } 1886 1887 return false; 1888 } 1889 1890 bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, 1891 SDValue &Offset) { 1892 if (SelectAddrFrameIndex(Addr, Base, Offset)) 1893 return true; 1894 1895 SDLoc DL(Addr); 1896 MVT VT = Addr.getSimpleValueType(); 1897 1898 if (Addr.getOpcode() == RISCVISD::ADD_LO) { 1899 Base = Addr.getOperand(0); 1900 Offset = Addr.getOperand(1); 1901 return true; 1902 } 1903 1904 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1905 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue(); 1906 if (isInt<12>(CVal)) { 1907 Base = Addr.getOperand(0); 1908 if (Base.getOpcode() == RISCVISD::ADD_LO) { 1909 SDValue LoOperand = Base.getOperand(1); 1910 if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) { 1911 // If the Lo in (ADD_LO hi, lo) is a global variable's address 1912 // (its low part, really), then we can rely on the alignment of that 1913 // variable to provide a margin of safety before low part can overflow 1914 // the 12 bits of the load/store offset. Check if CVal falls within 1915 // that margin; if so (low part + CVal) can't overflow. 1916 const DataLayout &DL = CurDAG->getDataLayout(); 1917 Align Alignment = commonAlignment( 1918 GA->getGlobal()->getPointerAlignment(DL), GA->getOffset()); 1919 if (CVal == 0 || Alignment > CVal) { 1920 int64_t CombinedOffset = CVal + GA->getOffset(); 1921 Base = Base.getOperand(0); 1922 Offset = CurDAG->getTargetGlobalAddress( 1923 GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(), 1924 CombinedOffset, GA->getTargetFlags()); 1925 return true; 1926 } 1927 } 1928 } 1929 1930 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base)) 1931 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT); 1932 Offset = CurDAG->getTargetConstant(CVal, DL, VT); 1933 return true; 1934 } 1935 } 1936 1937 // Handle ADD with large immediates. 1938 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) { 1939 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue(); 1940 assert(!isInt<12>(CVal) && "simm12 not already handled?"); 1941 1942 if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) { 1943 // We can use an ADDI for part of the offset and fold the rest into the 1944 // load/store. This mirrors the AddiPair PatFrag in RISCVInstrInfo.td. 1945 int64_t Adj = CVal < 0 ? -2048 : 2047; 1946 Base = SDValue( 1947 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0), 1948 CurDAG->getTargetConstant(Adj, DL, VT)), 1949 0); 1950 Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT); 1951 return true; 1952 } 1953 } 1954 1955 Base = Addr; 1956 Offset = CurDAG->getTargetConstant(0, DL, VT); 1957 return true; 1958 } 1959 1960 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, 1961 SDValue &ShAmt) { 1962 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift 1963 // amount. If there is an AND on the shift amount, we can bypass it if it 1964 // doesn't affect any of those bits. 1965 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 1966 const APInt &AndMask = N->getConstantOperandAPInt(1); 1967 1968 // Since the max shift amount is a power of 2 we can subtract 1 to make a 1969 // mask that covers the bits needed to represent all shift amounts. 1970 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!"); 1971 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); 1972 1973 if (ShMask.isSubsetOf(AndMask)) { 1974 ShAmt = N.getOperand(0); 1975 return true; 1976 } 1977 1978 // SimplifyDemandedBits may have optimized the mask so try restoring any 1979 // bits that are known zero. 1980 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0)); 1981 if (ShMask.isSubsetOf(AndMask | Known.Zero)) { 1982 ShAmt = N.getOperand(0); 1983 return true; 1984 } 1985 } else if (N.getOpcode() == ISD::SUB && 1986 isa<ConstantSDNode>(N.getOperand(0))) { 1987 uint64_t Imm = N.getConstantOperandVal(0); 1988 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to 1989 // generate a NEG instead of a SUB of a constant. 1990 if (Imm != 0 && Imm % ShiftWidth == 0) { 1991 SDLoc DL(N); 1992 EVT VT = N.getValueType(); 1993 SDValue Zero = CurDAG->getRegister(RISCV::X0, VT); 1994 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB; 1995 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero, 1996 N.getOperand(1)); 1997 ShAmt = SDValue(Neg, 0); 1998 return true; 1999 } 2000 } 2001 2002 ShAmt = N; 2003 return true; 2004 } 2005 2006 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) { 2007 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG && 2008 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) { 2009 Val = N.getOperand(0); 2010 return true; 2011 } 2012 MVT VT = N.getSimpleValueType(); 2013 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) { 2014 Val = N; 2015 return true; 2016 } 2017 2018 return false; 2019 } 2020 2021 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) { 2022 if (N.getOpcode() == ISD::AND) { 2023 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 2024 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) { 2025 Val = N.getOperand(0); 2026 return true; 2027 } 2028 } 2029 MVT VT = N.getSimpleValueType(); 2030 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32); 2031 if (CurDAG->MaskedValueIsZero(N, Mask)) { 2032 Val = N; 2033 return true; 2034 } 2035 2036 return false; 2037 } 2038 2039 /// Look for various patterns that can be done with a SHL that can be folded 2040 /// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which 2041 /// SHXADD we are trying to match. 2042 bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt, 2043 SDValue &Val) { 2044 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) { 2045 SDValue N0 = N.getOperand(0); 2046 2047 bool LeftShift = N0.getOpcode() == ISD::SHL; 2048 if ((LeftShift || N0.getOpcode() == ISD::SRL) && 2049 isa<ConstantSDNode>(N0.getOperand(1))) { 2050 uint64_t Mask = N.getConstantOperandVal(1); 2051 unsigned C2 = N0.getConstantOperandVal(1); 2052 2053 unsigned XLen = Subtarget->getXLen(); 2054 if (LeftShift) 2055 Mask &= maskTrailingZeros<uint64_t>(C2); 2056 else 2057 Mask &= maskTrailingOnes<uint64_t>(XLen - C2); 2058 2059 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no 2060 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3 2061 // followed by a SHXADD with c3 for the X amount. 2062 if (isShiftedMask_64(Mask)) { 2063 unsigned Leading = XLen - (64 - countLeadingZeros(Mask)); 2064 unsigned Trailing = countTrailingZeros(Mask); 2065 if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) { 2066 SDLoc DL(N); 2067 EVT VT = N.getValueType(); 2068 Val = SDValue(CurDAG->getMachineNode( 2069 RISCV::SRLI, DL, VT, N0.getOperand(0), 2070 CurDAG->getTargetConstant(Trailing - C2, DL, VT)), 2071 0); 2072 return true; 2073 } 2074 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2 2075 // leading zeros and c3 trailing zeros. We can use an SRLI by C3 2076 // followed by a SHXADD using c3 for the X amount. 2077 if (!LeftShift && Leading == C2 && Trailing == ShAmt) { 2078 SDLoc DL(N); 2079 EVT VT = N.getValueType(); 2080 Val = SDValue( 2081 CurDAG->getMachineNode( 2082 RISCV::SRLI, DL, VT, N0.getOperand(0), 2083 CurDAG->getTargetConstant(Leading + Trailing, DL, VT)), 2084 0); 2085 return true; 2086 } 2087 } 2088 } 2089 } 2090 2091 bool LeftShift = N.getOpcode() == ISD::SHL; 2092 if ((LeftShift || N.getOpcode() == ISD::SRL) && 2093 isa<ConstantSDNode>(N.getOperand(1))) { 2094 SDValue N0 = N.getOperand(0); 2095 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() && 2096 isa<ConstantSDNode>(N0.getOperand(1))) { 2097 uint64_t Mask = N0.getConstantOperandVal(1); 2098 if (isShiftedMask_64(Mask)) { 2099 unsigned C1 = N.getConstantOperandVal(1); 2100 unsigned XLen = Subtarget->getXLen(); 2101 unsigned Leading = XLen - (64 - countLeadingZeros(Mask)); 2102 unsigned Trailing = countTrailingZeros(Mask); 2103 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and 2104 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD. 2105 if (LeftShift && Leading == 32 && Trailing > 0 && 2106 (Trailing + C1) == ShAmt) { 2107 SDLoc DL(N); 2108 EVT VT = N.getValueType(); 2109 Val = SDValue(CurDAG->getMachineNode( 2110 RISCV::SRLIW, DL, VT, N0.getOperand(0), 2111 CurDAG->getTargetConstant(Trailing, DL, VT)), 2112 0); 2113 return true; 2114 } 2115 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and 2116 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD. 2117 if (!LeftShift && Leading == 32 && Trailing > C1 && 2118 (Trailing - C1) == ShAmt) { 2119 SDLoc DL(N); 2120 EVT VT = N.getValueType(); 2121 Val = SDValue(CurDAG->getMachineNode( 2122 RISCV::SRLIW, DL, VT, N0.getOperand(0), 2123 CurDAG->getTargetConstant(Trailing, DL, VT)), 2124 0); 2125 return true; 2126 } 2127 } 2128 } 2129 } 2130 2131 return false; 2132 } 2133 2134 // Return true if all users of this SDNode* only consume the lower \p Bits. 2135 // This can be used to form W instructions for add/sub/mul/shl even when the 2136 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if 2137 // SimplifyDemandedBits has made it so some users see a sext_inreg and some 2138 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave 2139 // the add/sub/mul/shl to become non-W instructions. By checking the users we 2140 // may be able to use a W instruction and CSE with the other instruction if 2141 // this has happened. We could try to detect that the CSE opportunity exists 2142 // before doing this, but that would be more complicated. 2143 // TODO: Does this need to look through AND/OR/XOR to their users to find more 2144 // opportunities. 2145 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const { 2146 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB || 2147 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL || 2148 Node->getOpcode() == ISD::SRL || 2149 Node->getOpcode() == ISD::SIGN_EXTEND_INREG || 2150 Node->getOpcode() == RISCVISD::GREV || 2151 Node->getOpcode() == RISCVISD::GORC || 2152 isa<ConstantSDNode>(Node)) && 2153 "Unexpected opcode"); 2154 2155 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) { 2156 SDNode *User = *UI; 2157 // Users of this node should have already been instruction selected 2158 if (!User->isMachineOpcode()) 2159 return false; 2160 2161 // TODO: Add more opcodes? 2162 switch (User->getMachineOpcode()) { 2163 default: 2164 return false; 2165 case RISCV::ADDW: 2166 case RISCV::ADDIW: 2167 case RISCV::SUBW: 2168 case RISCV::MULW: 2169 case RISCV::SLLW: 2170 case RISCV::SLLIW: 2171 case RISCV::SRAW: 2172 case RISCV::SRAIW: 2173 case RISCV::SRLW: 2174 case RISCV::SRLIW: 2175 case RISCV::DIVW: 2176 case RISCV::DIVUW: 2177 case RISCV::REMW: 2178 case RISCV::REMUW: 2179 case RISCV::ROLW: 2180 case RISCV::RORW: 2181 case RISCV::RORIW: 2182 case RISCV::CLZW: 2183 case RISCV::CTZW: 2184 case RISCV::CPOPW: 2185 case RISCV::SLLI_UW: 2186 case RISCV::FMV_W_X: 2187 case RISCV::FCVT_H_W: 2188 case RISCV::FCVT_H_WU: 2189 case RISCV::FCVT_S_W: 2190 case RISCV::FCVT_S_WU: 2191 case RISCV::FCVT_D_W: 2192 case RISCV::FCVT_D_WU: 2193 if (Bits < 32) 2194 return false; 2195 break; 2196 case RISCV::SLLI: 2197 // SLLI only uses the lower (XLen - ShAmt) bits. 2198 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1)) 2199 return false; 2200 break; 2201 case RISCV::ANDI: 2202 if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1)))) 2203 return false; 2204 break; 2205 case RISCV::SEXT_B: 2206 if (Bits < 8) 2207 return false; 2208 break; 2209 case RISCV::SEXT_H: 2210 case RISCV::FMV_H_X: 2211 case RISCV::ZEXT_H_RV32: 2212 case RISCV::ZEXT_H_RV64: 2213 if (Bits < 16) 2214 return false; 2215 break; 2216 case RISCV::ADD_UW: 2217 case RISCV::SH1ADD_UW: 2218 case RISCV::SH2ADD_UW: 2219 case RISCV::SH3ADD_UW: 2220 // The first operand to add.uw/shXadd.uw is implicitly zero extended from 2221 // 32 bits. 2222 if (UI.getOperandNo() != 0 || Bits < 32) 2223 return false; 2224 break; 2225 case RISCV::SB: 2226 if (UI.getOperandNo() != 0 || Bits < 8) 2227 return false; 2228 break; 2229 case RISCV::SH: 2230 if (UI.getOperandNo() != 0 || Bits < 16) 2231 return false; 2232 break; 2233 case RISCV::SW: 2234 if (UI.getOperandNo() != 0 || Bits < 32) 2235 return false; 2236 break; 2237 } 2238 } 2239 2240 return true; 2241 } 2242 2243 // Select VL as a 5 bit immediate or a value that will become a register. This 2244 // allows us to choose betwen VSETIVLI or VSETVLI later. 2245 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { 2246 auto *C = dyn_cast<ConstantSDNode>(N); 2247 if (C && isUInt<5>(C->getZExtValue())) { 2248 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N), 2249 N->getValueType(0)); 2250 } else if (C && C->isAllOnesValue()) { 2251 // Treat all ones as VLMax. 2252 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2253 N->getValueType(0)); 2254 } else if (isa<RegisterSDNode>(N) && 2255 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) { 2256 // All our VL operands use an operand that allows GPRNoX0 or an immediate 2257 // as the register class. Convert X0 to a special immediate to pass the 2258 // MachineVerifier. This is recognized specially by the vsetvli insertion 2259 // pass. 2260 VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N), 2261 N->getValueType(0)); 2262 } else { 2263 VL = N; 2264 } 2265 2266 return true; 2267 } 2268 2269 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { 2270 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef()) 2271 return false; 2272 SplatVal = N.getOperand(1); 2273 return true; 2274 } 2275 2276 using ValidateFn = bool (*)(int64_t); 2277 2278 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, 2279 SelectionDAG &DAG, 2280 const RISCVSubtarget &Subtarget, 2281 ValidateFn ValidateImm) { 2282 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2283 !isa<ConstantSDNode>(N.getOperand(1))) 2284 return false; 2285 2286 int64_t SplatImm = 2287 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2288 2289 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand 2290 // type is wider than the resulting vector element type: an implicit 2291 // truncation first takes place. Therefore, perform a manual 2292 // truncation/sign-extension in order to ignore any truncated bits and catch 2293 // any zero-extended immediate. 2294 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first 2295 // sign-extending to (XLenVT -1). 2296 MVT XLenVT = Subtarget.getXLenVT(); 2297 assert(XLenVT == N.getOperand(1).getSimpleValueType() && 2298 "Unexpected splat operand type"); 2299 MVT EltVT = N.getSimpleValueType().getVectorElementType(); 2300 if (EltVT.bitsLT(XLenVT)) 2301 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits()); 2302 2303 if (!ValidateImm(SplatImm)) 2304 return false; 2305 2306 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT); 2307 return true; 2308 } 2309 2310 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) { 2311 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget, 2312 [](int64_t Imm) { return isInt<5>(Imm); }); 2313 } 2314 2315 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) { 2316 return selectVSplatSimmHelper( 2317 N, SplatVal, *CurDAG, *Subtarget, 2318 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; }); 2319 } 2320 2321 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N, 2322 SDValue &SplatVal) { 2323 return selectVSplatSimmHelper( 2324 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) { 2325 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16); 2326 }); 2327 } 2328 2329 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) { 2330 if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() || 2331 !isa<ConstantSDNode>(N.getOperand(1))) 2332 return false; 2333 2334 int64_t SplatImm = 2335 cast<ConstantSDNode>(N.getOperand(1))->getSExtValue(); 2336 2337 if (!isUInt<5>(SplatImm)) 2338 return false; 2339 2340 SplatVal = 2341 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT()); 2342 2343 return true; 2344 } 2345 2346 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width, 2347 SDValue &Imm) { 2348 if (auto *C = dyn_cast<ConstantSDNode>(N)) { 2349 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width); 2350 2351 if (!isInt<5>(ImmVal)) 2352 return false; 2353 2354 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT()); 2355 return true; 2356 } 2357 2358 return false; 2359 } 2360 2361 // Merge an ADDI into the offset of a load/store instruction where possible. 2362 // (load (addi base, off1), off2) -> (load base, off1+off2) 2363 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2) 2364 // (load (add base, (addi src, off1)), off2) 2365 // -> (load (add base, src), off1+off2) 2366 // (store val, (add base, (addi src, off1)), off2) 2367 // -> (store val, (add base, src), off1+off2) 2368 // This is possible when off1+off2 fits a 12-bit immediate. 2369 bool RISCVDAGToDAGISel::doPeepholeLoadStoreADDI(SDNode *N) { 2370 unsigned OffsetOpIdx, BaseOpIdx; 2371 if (!hasMemOffset(N, BaseOpIdx, OffsetOpIdx)) 2372 return false; 2373 2374 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx))) 2375 return false; 2376 2377 SDValue Base = N->getOperand(BaseOpIdx); 2378 2379 if (!Base.isMachineOpcode()) 2380 return false; 2381 2382 if (Base.getMachineOpcode() == RISCV::ADDI) { 2383 // If the base is an ADDI, we can merge it in to the load/store. 2384 } else if (Base.getMachineOpcode() == RISCV::ADDIW && 2385 isa<ConstantSDNode>(Base.getOperand(1)) && 2386 Base.getOperand(0).isMachineOpcode() && 2387 Base.getOperand(0).getMachineOpcode() == RISCV::LUI && 2388 isa<ConstantSDNode>(Base.getOperand(0).getOperand(0))) { 2389 // ADDIW can be merged if it's part of LUI+ADDIW constant materialization 2390 // and LUI+ADDI would have produced the same result. This is true for all 2391 // simm32 values except 0x7ffff800-0x7fffffff. 2392 int64_t Offset = 2393 SignExtend64<32>(Base.getOperand(0).getConstantOperandVal(0) << 12); 2394 Offset += cast<ConstantSDNode>(Base.getOperand(1))->getSExtValue(); 2395 if (!isInt<32>(Offset)) 2396 return false; 2397 } else 2398 return false; 2399 2400 SDValue ImmOperand = Base.getOperand(1); 2401 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx); 2402 2403 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) { 2404 int64_t Offset1 = Const->getSExtValue(); 2405 int64_t CombinedOffset = Offset1 + Offset2; 2406 if (!isInt<12>(CombinedOffset)) 2407 return false; 2408 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand), 2409 ImmOperand.getValueType()); 2410 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) { 2411 // If the off1 in (addi base, off1) is a global variable's address (its 2412 // low part, really), then we can rely on the alignment of that variable 2413 // to provide a margin of safety before off1 can overflow the 12 bits. 2414 // Check if off2 falls within that margin; if so off1+off2 can't overflow. 2415 const DataLayout &DL = CurDAG->getDataLayout(); 2416 Align Alignment = commonAlignment(GA->getGlobal()->getPointerAlignment(DL), 2417 GA->getOffset()); 2418 if (Offset2 != 0 && Alignment <= Offset2) 2419 return false; 2420 int64_t Offset1 = GA->getOffset(); 2421 int64_t CombinedOffset = Offset1 + Offset2; 2422 ImmOperand = CurDAG->getTargetGlobalAddress( 2423 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(), 2424 CombinedOffset, GA->getTargetFlags()); 2425 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) { 2426 // Ditto. 2427 Align Alignment = commonAlignment(CP->getAlign(), CP->getOffset()); 2428 if (Offset2 != 0 && Alignment <= Offset2) 2429 return false; 2430 int64_t Offset1 = CP->getOffset(); 2431 int64_t CombinedOffset = Offset1 + Offset2; 2432 ImmOperand = CurDAG->getTargetConstantPool( 2433 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(), 2434 CombinedOffset, CP->getTargetFlags()); 2435 } else { 2436 return false; 2437 } 2438 2439 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 2440 LLVM_DEBUG(Base->dump(CurDAG)); 2441 LLVM_DEBUG(dbgs() << "\nN: "); 2442 LLVM_DEBUG(N->dump(CurDAG)); 2443 LLVM_DEBUG(dbgs() << "\n"); 2444 2445 // Modify the offset operand of the load/store. 2446 if (BaseOpIdx == 0) { // Load 2447 N = CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand, 2448 N->getOperand(2)); 2449 } else { // Store 2450 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0), 2451 ImmOperand, N->getOperand(3)); 2452 } 2453 2454 return true; 2455 } 2456 2457 // Try to remove sext.w if the input is a W instruction or can be made into 2458 // a W instruction cheaply. 2459 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { 2460 // Look for the sext.w pattern, addiw rd, rs1, 0. 2461 if (N->getMachineOpcode() != RISCV::ADDIW || 2462 !isNullConstant(N->getOperand(1))) 2463 return false; 2464 2465 SDValue N0 = N->getOperand(0); 2466 if (!N0.isMachineOpcode()) 2467 return false; 2468 2469 switch (N0.getMachineOpcode()) { 2470 default: 2471 break; 2472 case RISCV::ADD: 2473 case RISCV::ADDI: 2474 case RISCV::SUB: 2475 case RISCV::MUL: 2476 case RISCV::SLLI: { 2477 // Convert sext.w+add/sub/mul to their W instructions. This will create 2478 // a new independent instruction. This improves latency. 2479 unsigned Opc; 2480 switch (N0.getMachineOpcode()) { 2481 default: 2482 llvm_unreachable("Unexpected opcode!"); 2483 case RISCV::ADD: Opc = RISCV::ADDW; break; 2484 case RISCV::ADDI: Opc = RISCV::ADDIW; break; 2485 case RISCV::SUB: Opc = RISCV::SUBW; break; 2486 case RISCV::MUL: Opc = RISCV::MULW; break; 2487 case RISCV::SLLI: Opc = RISCV::SLLIW; break; 2488 } 2489 2490 SDValue N00 = N0.getOperand(0); 2491 SDValue N01 = N0.getOperand(1); 2492 2493 // Shift amount needs to be uimm5. 2494 if (N0.getMachineOpcode() == RISCV::SLLI && 2495 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue())) 2496 break; 2497 2498 SDNode *Result = 2499 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), 2500 N00, N01); 2501 ReplaceUses(N, Result); 2502 return true; 2503 } 2504 case RISCV::ADDW: 2505 case RISCV::ADDIW: 2506 case RISCV::SUBW: 2507 case RISCV::MULW: 2508 case RISCV::SLLIW: 2509 case RISCV::GREVIW: 2510 case RISCV::GORCIW: 2511 // Result is already sign extended just remove the sext.w. 2512 // NOTE: We only handle the nodes that are selected with hasAllWUsers. 2513 ReplaceUses(N, N0.getNode()); 2514 return true; 2515 } 2516 2517 return false; 2518 } 2519 2520 // Optimize masked RVV pseudo instructions with a known all-ones mask to their 2521 // corresponding "unmasked" pseudo versions. The mask we're interested in will 2522 // take the form of a V0 physical register operand, with a glued 2523 // register-setting instruction. 2524 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) { 2525 const RISCV::RISCVMaskedPseudoInfo *I = 2526 RISCV::getMaskedPseudoInfo(N->getMachineOpcode()); 2527 if (!I) 2528 return false; 2529 2530 unsigned MaskOpIdx = I->MaskOpIdx; 2531 2532 // Check that we're using V0 as a mask register. 2533 if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) || 2534 cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0) 2535 return false; 2536 2537 // The glued user defines V0. 2538 const auto *Glued = N->getGluedNode(); 2539 2540 if (!Glued || Glued->getOpcode() != ISD::CopyToReg) 2541 return false; 2542 2543 // Check that we're defining V0 as a mask register. 2544 if (!isa<RegisterSDNode>(Glued->getOperand(1)) || 2545 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0) 2546 return false; 2547 2548 // Check the instruction defining V0; it needs to be a VMSET pseudo. 2549 SDValue MaskSetter = Glued->getOperand(2); 2550 2551 const auto IsVMSet = [](unsigned Opc) { 2552 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 || 2553 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 || 2554 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 || 2555 Opc == RISCV::PseudoVMSET_M_B8; 2556 }; 2557 2558 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has 2559 // undefined behaviour if it's the wrong bitwidth, so we could choose to 2560 // assume that it's all-ones? Same applies to its VL. 2561 if (!MaskSetter->isMachineOpcode() || !IsVMSet(MaskSetter.getMachineOpcode())) 2562 return false; 2563 2564 // Retrieve the tail policy operand index, if any. 2565 Optional<unsigned> TailPolicyOpIdx; 2566 const RISCVInstrInfo &TII = *Subtarget->getInstrInfo(); 2567 const MCInstrDesc &MaskedMCID = TII.get(N->getMachineOpcode()); 2568 2569 bool IsTA = true; 2570 if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) { 2571 // The last operand of the pseudo is the policy op, but we might have a 2572 // Glue operand last. We might also have a chain. 2573 TailPolicyOpIdx = N->getNumOperands() - 1; 2574 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Glue) 2575 (*TailPolicyOpIdx)--; 2576 if (N->getOperand(*TailPolicyOpIdx).getValueType() == MVT::Other) 2577 (*TailPolicyOpIdx)--; 2578 2579 if (!(N->getConstantOperandVal(*TailPolicyOpIdx) & 2580 RISCVII::TAIL_AGNOSTIC)) { 2581 // Keep the true-masked instruction when there is no unmasked TU 2582 // instruction 2583 if (I->UnmaskedTUPseudo == I->MaskedPseudo && !N->getOperand(0).isUndef()) 2584 return false; 2585 // We can't use TA if the tie-operand is not IMPLICIT_DEF 2586 if (!N->getOperand(0).isUndef()) 2587 IsTA = false; 2588 } 2589 } 2590 2591 unsigned Opc = IsTA ? I->UnmaskedPseudo : I->UnmaskedTUPseudo; 2592 2593 // Check that we're dropping the mask operand and any policy operand 2594 // when we transform to this unmasked pseudo. Additionally, if this insturtion 2595 // is tail agnostic, the unmasked instruction should not have a merge op. 2596 uint64_t TSFlags = TII.get(Opc).TSFlags; 2597 assert((IsTA != RISCVII::hasMergeOp(TSFlags)) && 2598 RISCVII::hasDummyMaskOp(TSFlags) && 2599 !RISCVII::hasVecPolicyOp(TSFlags) && 2600 "Unexpected pseudo to transform to"); 2601 (void)TSFlags; 2602 2603 SmallVector<SDValue, 8> Ops; 2604 // Skip the merge operand at index 0 if IsTA 2605 for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) { 2606 // Skip the mask, the policy, and the Glue. 2607 SDValue Op = N->getOperand(I); 2608 if (I == MaskOpIdx || I == TailPolicyOpIdx || 2609 Op.getValueType() == MVT::Glue) 2610 continue; 2611 Ops.push_back(Op); 2612 } 2613 2614 // Transitively apply any node glued to our new node. 2615 if (auto *TGlued = Glued->getGluedNode()) 2616 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1)); 2617 2618 SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 2619 ReplaceUses(N, Result); 2620 2621 return true; 2622 } 2623 2624 // This pass converts a legalized DAG into a RISCV-specific DAG, ready 2625 // for instruction scheduling. 2626 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM, 2627 CodeGenOpt::Level OptLevel) { 2628 return new RISCVDAGToDAGISel(TM, OptLevel); 2629 } 2630