1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVMacroFusion.h"
18 #include "RISCVTargetObjectFile.h"
19 #include "RISCVTargetTransformInfo.h"
20 #include "TargetInfo/RISCVTargetInfo.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
24 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
25 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
26 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
27 #include "llvm/CodeGen/MIRParser/MIParser.h"
28 #include "llvm/CodeGen/MIRYamlMapping.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/CodeGen/RegAllocRegistry.h"
31 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
32 #include "llvm/CodeGen/TargetPassConfig.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/MC/TargetRegistry.h"
35 #include "llvm/Support/FormattedStream.h"
36 #include "llvm/Target/TargetOptions.h"
37 #include "llvm/Transforms/IPO.h"
38 #include "llvm/Transforms/Scalar.h"
39 #include <optional>
40 using namespace llvm;
41 
42 static cl::opt<bool> EnableRedundantCopyElimination(
43     "riscv-enable-copyelim",
44     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
45     cl::Hidden);
46 
47 // FIXME: Unify control over GlobalMerge.
48 static cl::opt<cl::boolOrDefault>
49     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
50                       cl::desc("Enable the global merge pass"));
51 
52 static cl::opt<bool>
53     EnableMachineCombiner("riscv-enable-machine-combiner",
54                           cl::desc("Enable the machine combiner pass"),
55                           cl::init(true), cl::Hidden);
56 
57 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
58     "riscv-v-vector-bits-max",
59     cl::desc("Assume V extension vector registers are at most this big, "
60              "with zero meaning no maximum size is assumed."),
61     cl::init(0), cl::Hidden);
62 
63 static cl::opt<int> RVVVectorBitsMinOpt(
64     "riscv-v-vector-bits-min",
65     cl::desc("Assume V extension vector registers are at least this big, "
66              "with zero meaning no minimum size is assumed. A value of -1 "
67              "means use Zvl*b extension. This is primarily used to enable "
68              "autovectorization with fixed width vectors."),
69     cl::init(-1), cl::Hidden);
70 
71 static cl::opt<bool> EnableRISCVCopyPropagation(
72     "riscv-enable-copy-propagation",
73     cl::desc("Enable the copy propagation with RISC-V copy instr"),
74     cl::init(true), cl::Hidden);
75 
76 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
77     "riscv-enable-dead-defs", cl::Hidden,
78     cl::desc("Enable the pass that removes dead"
79              " definitons and replaces stores to"
80              " them with stores to x0"),
81     cl::init(true));
82 
83 static cl::opt<bool>
84     EnableSinkFold("riscv-enable-sink-fold",
85                    cl::desc("Enable sinking and folding of instruction copies"),
86                    cl::init(false), cl::Hidden);
87 
88 static cl::opt<bool>
89     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
90                            cl::desc("Enable the loop data prefetch pass"),
91                            cl::init(true));
92 
93 static cl::opt<bool>
94     EnableSplitRegAlloc("riscv-split-regalloc", cl::Hidden,
95                         cl::desc("Enable Split RegisterAlloc for RVV"),
96                         cl::init(true));
97 
98 static cl::opt<bool> EnableMISchedLoadClustering(
99     "riscv-misched-load-clustering", cl::Hidden,
100     cl::desc("Enable load clustering in the machine scheduler"),
101     cl::init(false));
102 
103 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
104   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
105   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
106   auto *PR = PassRegistry::getPassRegistry();
107   initializeGlobalISel(*PR);
108   initializeRISCVO0PreLegalizerCombinerPass(*PR);
109   initializeRISCVPreLegalizerCombinerPass(*PR);
110   initializeRISCVPostLegalizerCombinerPass(*PR);
111   initializeKCFIPass(*PR);
112   initializeRISCVDeadRegisterDefinitionsPass(*PR);
113   initializeRISCVMakeCompressibleOptPass(*PR);
114   initializeRISCVGatherScatterLoweringPass(*PR);
115   initializeRISCVCodeGenPreparePass(*PR);
116   initializeRISCVPostRAExpandPseudoPass(*PR);
117   initializeRISCVMergeBaseOffsetOptPass(*PR);
118   initializeRISCVOptWInstrsPass(*PR);
119   initializeRISCVPreRAExpandPseudoPass(*PR);
120   initializeRISCVExpandPseudoPass(*PR);
121   initializeRISCVFoldMasksPass(*PR);
122   initializeRISCVInsertVSETVLIPass(*PR);
123   initializeRISCVInsertReadWriteCSRPass(*PR);
124   initializeRISCVInsertWriteVXRMPass(*PR);
125   initializeRISCVDAGToDAGISelPass(*PR);
126   initializeRISCVInitUndefPass(*PR);
127   initializeRISCVMoveMergePass(*PR);
128   initializeRISCVPushPopOptPass(*PR);
129 }
130 
131 static StringRef computeDataLayout(const Triple &TT,
132                                    const TargetOptions &Options) {
133   StringRef ABIName = Options.MCOptions.getABIName();
134   if (TT.isArch64Bit()) {
135     if (ABIName == "lp64e")
136       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
137 
138     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
139   }
140   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
141 
142   if (ABIName == "ilp32e")
143     return "e-m:e-p:32:32-i64:64-n32-S32";
144 
145   return "e-m:e-p:32:32-i64:64-n32-S128";
146 }
147 
148 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
149                                            std::optional<Reloc::Model> RM) {
150   return RM.value_or(Reloc::Static);
151 }
152 
153 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
154                                        StringRef CPU, StringRef FS,
155                                        const TargetOptions &Options,
156                                        std::optional<Reloc::Model> RM,
157                                        std::optional<CodeModel::Model> CM,
158                                        CodeGenOptLevel OL, bool JIT)
159     : LLVMTargetMachine(T, computeDataLayout(TT, Options), TT, CPU, FS, Options,
160                         getEffectiveRelocModel(TT, RM),
161                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
162       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
163   initAsmInfo();
164 
165   // RISC-V supports the MachineOutliner.
166   setMachineOutliner(true);
167   setSupportsDefaultOutlining(true);
168 
169   if (TT.isOSFuchsia() && !TT.isArch64Bit())
170     report_fatal_error("Fuchsia is only supported for 64-bit");
171 }
172 
173 const RISCVSubtarget *
174 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
175   Attribute CPUAttr = F.getFnAttribute("target-cpu");
176   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
177   Attribute FSAttr = F.getFnAttribute("target-features");
178 
179   std::string CPU =
180       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
181   std::string TuneCPU =
182       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
183   std::string FS =
184       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
185 
186   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
187   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
188 
189   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
190   if (VScaleRangeAttr.isValid()) {
191     if (!RVVVectorBitsMinOpt.getNumOccurrences())
192       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
193     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
194     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
195       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
196   }
197 
198   if (RVVBitsMin != -1U) {
199     // FIXME: Change to >= 32 when VLEN = 32 is supported.
200     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
201                                 isPowerOf2_32(RVVBitsMin))) &&
202            "V or Zve* extension requires vector length to be in the range of "
203            "64 to 65536 and a power 2!");
204     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
205            "Minimum V extension vector length should not be larger than its "
206            "maximum!");
207   }
208   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
209                               isPowerOf2_32(RVVBitsMax))) &&
210          "V or Zve* extension requires vector length to be in the range of "
211          "64 to 65536 and a power 2!");
212 
213   if (RVVBitsMin != -1U) {
214     if (RVVBitsMax != 0) {
215       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
216       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
217     }
218 
219     RVVBitsMin = llvm::bit_floor(
220         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
221   }
222   RVVBitsMax =
223       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
224 
225   SmallString<512> Key;
226   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
227                            << CPU << TuneCPU << FS;
228   auto &I = SubtargetMap[Key];
229   if (!I) {
230     // This needs to be done before we create a new subtarget since any
231     // creation will depend on the TM and the code generation flags on the
232     // function that reside in TargetOptions.
233     resetTargetOptions(F);
234     auto ABIName = Options.MCOptions.getABIName();
235     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
236             F.getParent()->getModuleFlag("target-abi"))) {
237       auto TargetABI = RISCVABI::getTargetABI(ABIName);
238       if (TargetABI != RISCVABI::ABI_Unknown &&
239           ModuleTargetABI->getString() != ABIName) {
240         report_fatal_error("-target-abi option != target-abi module flag");
241       }
242       ABIName = ModuleTargetABI->getString();
243     }
244     I = std::make_unique<RISCVSubtarget>(
245         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
246   }
247   return I.get();
248 }
249 
250 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
251     BumpPtrAllocator &Allocator, const Function &F,
252     const TargetSubtargetInfo *STI) const {
253   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
254                                                                     F, STI);
255 }
256 
257 TargetTransformInfo
258 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
259   return TargetTransformInfo(RISCVTTIImpl(this, F));
260 }
261 
262 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
263 // for all memory accesses, so it is reasonable to assume that an
264 // implementation has no-op address space casts. If an implementation makes a
265 // change to this, they can override it here.
266 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
267                                              unsigned DstAS) const {
268   return true;
269 }
270 
271 namespace {
272 
273 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
274 public:
275   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
276       : RegisterRegAllocBase(N, D, C) {}
277 };
278 
279 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
280                                const TargetRegisterClass &RC) {
281   return RISCV::VRRegClass.hasSubClassEq(&RC) ||
282          RISCV::VRM2RegClass.hasSubClassEq(&RC) ||
283          RISCV::VRM4RegClass.hasSubClassEq(&RC) ||
284          RISCV::VRM8RegClass.hasSubClassEq(&RC) ||
285          RISCV::VRN2M1RegClass.hasSubClassEq(&RC) ||
286          RISCV::VRN2M2RegClass.hasSubClassEq(&RC) ||
287          RISCV::VRN2M4RegClass.hasSubClassEq(&RC) ||
288          RISCV::VRN3M1RegClass.hasSubClassEq(&RC) ||
289          RISCV::VRN3M2RegClass.hasSubClassEq(&RC) ||
290          RISCV::VRN4M1RegClass.hasSubClassEq(&RC) ||
291          RISCV::VRN4M2RegClass.hasSubClassEq(&RC) ||
292          RISCV::VRN5M1RegClass.hasSubClassEq(&RC) ||
293          RISCV::VRN6M1RegClass.hasSubClassEq(&RC) ||
294          RISCV::VRN7M1RegClass.hasSubClassEq(&RC) ||
295          RISCV::VRN8M1RegClass.hasSubClassEq(&RC);
296 }
297 
298 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
299 
300 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
301 
302 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
303 /// This option could designate the rvv register allocator only.
304 /// For example: -riscv-rvv-regalloc=basic
305 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
306                RegisterPassParser<RVVRegisterRegAlloc>>
307     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
308                 cl::init(&useDefaultRegisterAllocator),
309                 cl::desc("Register allocator to use for RVV register."));
310 
311 static void initializeDefaultRVVRegisterAllocatorOnce() {
312   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
313 
314   if (!Ctor) {
315     Ctor = RVVRegAlloc;
316     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
317   }
318 }
319 
320 static FunctionPass *createBasicRVVRegisterAllocator() {
321   return createBasicRegisterAllocator(onlyAllocateRVVReg);
322 }
323 
324 static FunctionPass *createGreedyRVVRegisterAllocator() {
325   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
326 }
327 
328 static FunctionPass *createFastRVVRegisterAllocator() {
329   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
330 }
331 
332 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
333                                                "basic register allocator",
334                                                createBasicRVVRegisterAllocator);
335 static RVVRegisterRegAlloc
336     greedyRegAllocRVVReg("greedy", "greedy register allocator",
337                          createGreedyRVVRegisterAllocator);
338 
339 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
340                                               createFastRVVRegisterAllocator);
341 
342 class RISCVPassConfig : public TargetPassConfig {
343 public:
344   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
345       : TargetPassConfig(TM, PM) {
346     if (TM.getOptLevel() != CodeGenOptLevel::None)
347       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
348     setEnableSinkAndFold(EnableSinkFold);
349   }
350 
351   RISCVTargetMachine &getRISCVTargetMachine() const {
352     return getTM<RISCVTargetMachine>();
353   }
354 
355   ScheduleDAGInstrs *
356   createMachineScheduler(MachineSchedContext *C) const override {
357     const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
358     ScheduleDAGMILive *DAG = nullptr;
359     if (EnableMISchedLoadClustering) {
360       DAG = createGenericSchedLive(C);
361       DAG->addMutation(createLoadClusterDAGMutation(
362           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
363     }
364     if (ST.hasMacroFusion()) {
365       DAG = DAG ? DAG : createGenericSchedLive(C);
366       DAG->addMutation(createRISCVMacroFusionDAGMutation());
367     }
368     return DAG;
369   }
370 
371   ScheduleDAGInstrs *
372   createPostMachineScheduler(MachineSchedContext *C) const override {
373     const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
374     if (ST.hasMacroFusion()) {
375       ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
376       DAG->addMutation(createRISCVMacroFusionDAGMutation());
377       return DAG;
378     }
379     return nullptr;
380   }
381 
382   void addIRPasses() override;
383   bool addPreISel() override;
384   bool addInstSelector() override;
385   bool addIRTranslator() override;
386   void addPreLegalizeMachineIR() override;
387   bool addLegalizeMachineIR() override;
388   void addPreRegBankSelect() override;
389   bool addRegBankSelect() override;
390   bool addGlobalInstructionSelect() override;
391   void addPreEmitPass() override;
392   void addPreEmitPass2() override;
393   void addPreSched2() override;
394   void addMachineSSAOptimization() override;
395   FunctionPass *createRVVRegAllocPass(bool Optimized);
396   bool addRegAssignAndRewriteFast() override;
397   bool addRegAssignAndRewriteOptimized() override;
398   void addPreRegAlloc() override;
399   void addPostRegAlloc() override;
400   void addOptimizedRegAlloc() override;
401   void addFastRegAlloc() override;
402 };
403 } // namespace
404 
405 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
406   return new RISCVPassConfig(*this, PM);
407 }
408 
409 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
410   // Initialize the global default.
411   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
412                   initializeDefaultRVVRegisterAllocatorOnce);
413 
414   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
415   if (Ctor != useDefaultRegisterAllocator)
416     return Ctor();
417 
418   if (Optimized)
419     return createGreedyRVVRegisterAllocator();
420 
421   return createFastRVVRegisterAllocator();
422 }
423 
424 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
425   if (EnableSplitRegAlloc)
426     addPass(createRVVRegAllocPass(false));
427   return TargetPassConfig::addRegAssignAndRewriteFast();
428 }
429 
430 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
431   if (EnableSplitRegAlloc) {
432     addPass(createRVVRegAllocPass(true));
433     addPass(createVirtRegRewriter(false));
434   }
435   return TargetPassConfig::addRegAssignAndRewriteOptimized();
436 }
437 
438 void RISCVPassConfig::addIRPasses() {
439   addPass(createAtomicExpandPass());
440 
441   if (getOptLevel() != CodeGenOptLevel::None) {
442     if (EnableLoopDataPrefetch)
443       addPass(createLoopDataPrefetchPass());
444 
445     addPass(createRISCVGatherScatterLoweringPass());
446     addPass(createInterleavedAccessPass());
447     addPass(createRISCVCodeGenPreparePass());
448   }
449 
450   TargetPassConfig::addIRPasses();
451 }
452 
453 bool RISCVPassConfig::addPreISel() {
454   if (TM->getOptLevel() != CodeGenOptLevel::None) {
455     // Add a barrier before instruction selection so that we will not get
456     // deleted block address after enabling default outlining. See D99707 for
457     // more details.
458     addPass(createBarrierNoopPass());
459   }
460 
461   if (EnableGlobalMerge == cl::BOU_TRUE) {
462     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
463                                   /* OnlyOptimizeForSize */ false,
464                                   /* MergeExternalByDefault */ true));
465   }
466 
467   return false;
468 }
469 
470 bool RISCVPassConfig::addInstSelector() {
471   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
472 
473   return false;
474 }
475 
476 bool RISCVPassConfig::addIRTranslator() {
477   addPass(new IRTranslator(getOptLevel()));
478   return false;
479 }
480 
481 void RISCVPassConfig::addPreLegalizeMachineIR() {
482   if (getOptLevel() == CodeGenOptLevel::None) {
483     addPass(createRISCVO0PreLegalizerCombiner());
484   } else {
485     addPass(createRISCVPreLegalizerCombiner());
486   }
487 }
488 
489 bool RISCVPassConfig::addLegalizeMachineIR() {
490   addPass(new Legalizer());
491   return false;
492 }
493 
494 void RISCVPassConfig::addPreRegBankSelect() {
495   if (getOptLevel() != CodeGenOptLevel::None)
496     addPass(createRISCVPostLegalizerCombiner());
497 }
498 
499 bool RISCVPassConfig::addRegBankSelect() {
500   addPass(new RegBankSelect());
501   return false;
502 }
503 
504 bool RISCVPassConfig::addGlobalInstructionSelect() {
505   addPass(new InstructionSelect(getOptLevel()));
506   return false;
507 }
508 
509 void RISCVPassConfig::addPreSched2() {
510   addPass(createRISCVPostRAExpandPseudoPass());
511 
512   // Emit KCFI checks for indirect calls.
513   addPass(createKCFIPass());
514 }
515 
516 void RISCVPassConfig::addPreEmitPass() {
517   addPass(&BranchRelaxationPassID);
518   addPass(createRISCVMakeCompressibleOptPass());
519 
520   // TODO: It would potentially be better to schedule copy propagation after
521   // expanding pseudos (in addPreEmitPass2). However, performing copy
522   // propagation after the machine outliner (which runs after addPreEmitPass)
523   // currently leads to incorrect code-gen, where copies to registers within
524   // outlined functions are removed erroneously.
525   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
526       EnableRISCVCopyPropagation)
527     addPass(createMachineCopyPropagationPass(true));
528 }
529 
530 void RISCVPassConfig::addPreEmitPass2() {
531   if (TM->getOptLevel() != CodeGenOptLevel::None) {
532     addPass(createRISCVMoveMergePass());
533     // Schedule PushPop Optimization before expansion of Pseudo instruction,
534     // ensuring return instruction is detected correctly.
535     addPass(createRISCVPushPopOptimizationPass());
536   }
537   addPass(createRISCVExpandPseudoPass());
538 
539   // Schedule the expansion of AMOs at the last possible moment, avoiding the
540   // possibility for other passes to break the requirements for forward
541   // progress in the LR/SC block.
542   addPass(createRISCVExpandAtomicPseudoPass());
543 
544   // KCFI indirect call checks are lowered to a bundle.
545   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
546     return MF.getFunction().getParent()->getModuleFlag("kcfi");
547   }));
548 }
549 
550 void RISCVPassConfig::addMachineSSAOptimization() {
551   addPass(createRISCVFoldMasksPass());
552 
553   TargetPassConfig::addMachineSSAOptimization();
554 
555   if (EnableMachineCombiner)
556     addPass(&MachineCombinerID);
557 
558   if (TM->getTargetTriple().isRISCV64()) {
559     addPass(createRISCVOptWInstrsPass());
560   }
561 }
562 
563 void RISCVPassConfig::addPreRegAlloc() {
564   addPass(createRISCVPreRAExpandPseudoPass());
565   if (TM->getOptLevel() != CodeGenOptLevel::None)
566     addPass(createRISCVMergeBaseOffsetOptPass());
567   addPass(createRISCVInsertVSETVLIPass());
568   if (TM->getOptLevel() != CodeGenOptLevel::None &&
569       EnableRISCVDeadRegisterElimination)
570     addPass(createRISCVDeadRegisterDefinitionsPass());
571   addPass(createRISCVInsertReadWriteCSRPass());
572   addPass(createRISCVInsertWriteVXRMPass());
573 }
574 
575 void RISCVPassConfig::addOptimizedRegAlloc() {
576   insertPass(&DetectDeadLanesID, &RISCVInitUndefID);
577 
578   TargetPassConfig::addOptimizedRegAlloc();
579 }
580 
581 void RISCVPassConfig::addFastRegAlloc() {
582   addPass(createRISCVInitUndefPass());
583   TargetPassConfig::addFastRegAlloc();
584 }
585 
586 
587 void RISCVPassConfig::addPostRegAlloc() {
588   if (TM->getOptLevel() != CodeGenOptLevel::None &&
589       EnableRedundantCopyElimination)
590     addPass(createRISCVRedundantCopyEliminationPass());
591 }
592 
593 yaml::MachineFunctionInfo *
594 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
595   return new yaml::RISCVMachineFunctionInfo();
596 }
597 
598 yaml::MachineFunctionInfo *
599 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
600   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
601   return new yaml::RISCVMachineFunctionInfo(*MFI);
602 }
603 
604 bool RISCVTargetMachine::parseMachineFunctionInfo(
605     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
606     SMDiagnostic &Error, SMRange &SourceRange) const {
607   const auto &YamlMFI =
608       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
609   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
610   return false;
611 }
612