1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBankInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/CodeGenCoverage.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/LEB128.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <cstdint>
41 
42 namespace llvm {
43 
44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45           class CustomRendererFn>
executeMatchTable(TgtExecutor & Exec,MatcherState & State,const ExecInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ExecInfo,MachineIRBuilder & Builder,const uint8_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage * CoverageInfo)46 bool GIMatchTableExecutor::executeMatchTable(
47     TgtExecutor &Exec, MatcherState &State,
48     const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49         &ExecInfo,
50     MachineIRBuilder &Builder, const uint8_t *MatchTable,
51     const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52     const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53     const PredicateBitset &AvailableFeatures,
54     CodeGenCoverage *CoverageInfo) const {
55 
56   uint64_t CurrentIdx = 0;
57   SmallVector<uint64_t, 4> OnFailResumeAt;
58   NewMIVector OutMIs;
59 
60   GISelChangeObserver *Observer = Builder.getObserver();
61   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63 
64   const uint16_t Flags = State.MIs[0]->getFlags();
65 
66   enum RejectAction { RejectAndGiveUp, RejectAndResume };
67   auto handleReject = [&]() -> RejectAction {
68     DEBUG_WITH_TYPE(TgtExecutor::getName(),
69                     dbgs() << CurrentIdx << ": Rejected\n");
70     if (OnFailResumeAt.empty())
71       return RejectAndGiveUp;
72     CurrentIdx = OnFailResumeAt.pop_back_val();
73     DEBUG_WITH_TYPE(TgtExecutor::getName(),
74                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75                            << OnFailResumeAt.size() << " try-blocks remain)\n");
76     return RejectAndResume;
77   };
78 
79   const auto propagateFlags = [&]() {
80     for (auto MIB : OutMIs) {
81       // Set the NoFPExcept flag when no original matched instruction could
82       // raise an FP exception, but the new instruction potentially might.
83       uint16_t MIBFlags = Flags;
84       if (NoFPException && MIB->mayRaiseFPException())
85         MIBFlags |= MachineInstr::NoFPExcept;
86       if (Observer)
87         Observer->changingInstr(*MIB);
88       MIB.setMIFlags(MIBFlags);
89       if (Observer)
90         Observer->changedInstr(*MIB);
91     }
92   };
93 
94   // If the index is >= 0, it's an index in the type objects generated by
95   // TableGen. If the index is <0, it's an index in the recorded types object.
96   const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97     if (Idx >= 0)
98       return ExecInfo.TypeObjects[Idx];
99     return State.RecordedTypes[1 - Idx];
100   };
101 
102   const auto readULEB = [&]() {
103     unsigned N = 0;
104     uint64_t Val = decodeULEB128(MatchTable + CurrentIdx, &N);
105     CurrentIdx += N;
106     return Val;
107   };
108 
109   // Convenience function to return a signed value. This avoids
110   // us forgetting to first cast to int8_t before casting to a
111   // wider signed int type.
112   // if we casted uint8 directly to a wider type we'd lose
113   // negative values.
114   const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
115 
116   const auto readU16 = [&]() {
117     auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
118     CurrentIdx += 2;
119     return V;
120   };
121 
122   const auto readU32 = [&]() {
123     auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
124     CurrentIdx += 4;
125     return V;
126   };
127 
128   const auto readU64 = [&]() {
129     auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
130     CurrentIdx += 8;
131     return V;
132   };
133 
134   while (true) {
135     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
136     uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
137     switch (MatcherOpcode) {
138     case GIM_Try: {
139       DEBUG_WITH_TYPE(TgtExecutor::getName(),
140                       dbgs() << CurrentIdx << ": Begin try-block\n");
141       OnFailResumeAt.push_back(readU32());
142       break;
143     }
144 
145     case GIM_RecordInsn:
146     case GIM_RecordInsnIgnoreCopies: {
147       uint64_t NewInsnID = readULEB();
148       uint64_t InsnID = readULEB();
149       uint64_t OpIdx = readULEB();
150 
151       // As an optimisation we require that MIs[0] is always the root. Refuse
152       // any attempt to modify it.
153       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
154 
155       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
156       if (!MO.isReg()) {
157         DEBUG_WITH_TYPE(TgtExecutor::getName(),
158                         dbgs() << CurrentIdx << ": Not a register\n");
159         if (handleReject() == RejectAndGiveUp)
160           return false;
161         break;
162       }
163       if (MO.getReg().isPhysical()) {
164         DEBUG_WITH_TYPE(TgtExecutor::getName(),
165                         dbgs() << CurrentIdx << ": Is a physical register\n");
166         if (handleReject() == RejectAndGiveUp)
167           return false;
168         break;
169       }
170 
171       MachineInstr *NewMI;
172       if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
173         NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
174       else
175         NewMI = MRI.getVRegDef(MO.getReg());
176 
177       if ((size_t)NewInsnID < State.MIs.size())
178         State.MIs[NewInsnID] = NewMI;
179       else {
180         assert((size_t)NewInsnID == State.MIs.size() &&
181                "Expected to store MIs in order");
182         State.MIs.push_back(NewMI);
183       }
184       DEBUG_WITH_TYPE(TgtExecutor::getName(),
185                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
186                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
187                              << ")\n");
188       break;
189     }
190 
191     case GIM_CheckFeatures: {
192       uint16_t ExpectedBitsetID = readU16();
193       DEBUG_WITH_TYPE(TgtExecutor::getName(),
194                       dbgs() << CurrentIdx
195                              << ": GIM_CheckFeatures(ExpectedBitsetID="
196                              << ExpectedBitsetID << ")\n");
197       if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
198           ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
199         if (handleReject() == RejectAndGiveUp)
200           return false;
201       }
202       break;
203     }
204     case GIM_CheckOpcode:
205     case GIM_CheckOpcodeIsEither: {
206       uint64_t InsnID = readULEB();
207       uint16_t Expected0 = readU16();
208       uint16_t Expected1 = -1;
209       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
210         Expected1 = readU16();
211 
212       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
213       unsigned Opcode = State.MIs[InsnID]->getOpcode();
214 
215       DEBUG_WITH_TYPE(TgtExecutor::getName(),
216                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
217                              << "], ExpectedOpcode=" << Expected0;
218                       if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
219                       << " || " << Expected1;
220                       dbgs() << ") // Got=" << Opcode << "\n";);
221 
222       if (Opcode != Expected0 && Opcode != Expected1) {
223         if (handleReject() == RejectAndGiveUp)
224           return false;
225       }
226       break;
227     }
228     case GIM_SwitchOpcode: {
229       uint64_t InsnID = readULEB();
230       uint16_t LowerBound = readU16();
231       uint16_t UpperBound = readU16();
232       uint32_t Default = readU32();
233 
234       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
236 
237       DEBUG_WITH_TYPE(TgtExecutor::getName(), {
238         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
239                << LowerBound << ", " << UpperBound << "), Default=" << Default
240                << ", JumpTable...) // Got=" << Opcode << "\n";
241       });
242       if (Opcode < LowerBound || UpperBound <= Opcode) {
243         CurrentIdx = Default;
244         break;
245       }
246       const auto EntryIdx = (Opcode - LowerBound);
247       // Each entry is 4 bytes
248       CurrentIdx =
249           readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
250       if (!CurrentIdx) {
251         CurrentIdx = Default;
252         break;
253       }
254       OnFailResumeAt.push_back(Default);
255       break;
256     }
257 
258     case GIM_SwitchType: {
259       uint64_t InsnID = readULEB();
260       uint64_t OpIdx = readULEB();
261       uint16_t LowerBound = readU16();
262       uint16_t UpperBound = readU16();
263       int64_t Default = readU32();
264 
265       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
266       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
267 
268       DEBUG_WITH_TYPE(TgtExecutor::getName(), {
269         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
270                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
271                << UpperBound << "), Default=" << Default
272                << ", JumpTable...) // Got=";
273         if (!MO.isReg())
274           dbgs() << "Not a VReg\n";
275         else
276           dbgs() << MRI.getType(MO.getReg()) << "\n";
277       });
278       if (!MO.isReg()) {
279         CurrentIdx = Default;
280         break;
281       }
282       const LLT Ty = MRI.getType(MO.getReg());
283       const auto TyI = ExecInfo.TypeIDMap.find(Ty);
284       if (TyI == ExecInfo.TypeIDMap.end()) {
285         CurrentIdx = Default;
286         break;
287       }
288       const int64_t TypeID = TyI->second;
289       if (TypeID < LowerBound || UpperBound <= TypeID) {
290         CurrentIdx = Default;
291         break;
292       }
293       const auto NumEntry = (TypeID - LowerBound);
294       // Each entry is 4 bytes
295       CurrentIdx =
296           readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
297       if (!CurrentIdx) {
298         CurrentIdx = Default;
299         break;
300       }
301       OnFailResumeAt.push_back(Default);
302       break;
303     }
304 
305     case GIM_CheckNumOperands: {
306       uint64_t InsnID = readULEB();
307       uint64_t Expected = readULEB();
308       DEBUG_WITH_TYPE(TgtExecutor::getName(),
309                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
310                              << InsnID << "], Expected=" << Expected << ")\n");
311       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
312       if (State.MIs[InsnID]->getNumOperands() != Expected) {
313         if (handleReject() == RejectAndGiveUp)
314           return false;
315       }
316       break;
317     }
318     case GIM_CheckI64ImmPredicate:
319     case GIM_CheckImmOperandPredicate: {
320       uint64_t InsnID = readULEB();
321       unsigned OpIdx =
322           MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
323       uint16_t Predicate = readU16();
324       DEBUG_WITH_TYPE(TgtExecutor::getName(),
325                       dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
326                              << InsnID << "]->getOperand(" << OpIdx
327                              << "), Predicate=" << Predicate << ")\n");
328       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
329       assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
330               State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
331              "Expected immediate operand");
332       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
333       int64_t Value = 0;
334       if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
335         Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
336       else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
337         Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
338       else
339         llvm_unreachable("Expected Imm or CImm operand");
340 
341       if (!testImmPredicate_I64(Predicate, Value))
342         if (handleReject() == RejectAndGiveUp)
343           return false;
344       break;
345     }
346     case GIM_CheckAPIntImmPredicate: {
347       uint64_t InsnID = readULEB();
348       uint16_t Predicate = readU16();
349       DEBUG_WITH_TYPE(TgtExecutor::getName(),
350                       dbgs()
351                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
352                           << InsnID << "], Predicate=" << Predicate << ")\n");
353       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
354       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
355              "Expected G_CONSTANT");
356       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
357       if (!State.MIs[InsnID]->getOperand(1).isCImm())
358         llvm_unreachable("Expected Imm or CImm operand");
359 
360       const APInt &Value =
361           State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
362       if (!testImmPredicate_APInt(Predicate, Value))
363         if (handleReject() == RejectAndGiveUp)
364           return false;
365       break;
366     }
367     case GIM_CheckAPFloatImmPredicate: {
368       uint64_t InsnID = readULEB();
369       uint16_t Predicate = readU16();
370       DEBUG_WITH_TYPE(TgtExecutor::getName(),
371                       dbgs()
372                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
373                           << InsnID << "], Predicate=" << Predicate << ")\n");
374       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
375       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
376              "Expected G_FCONSTANT");
377       assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
378              "Expected FPImm operand");
379       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
380       const APFloat &Value =
381           State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
382 
383       if (!testImmPredicate_APFloat(Predicate, Value))
384         if (handleReject() == RejectAndGiveUp)
385           return false;
386       break;
387     }
388     case GIM_CheckIsBuildVectorAllOnes:
389     case GIM_CheckIsBuildVectorAllZeros: {
390       uint64_t InsnID = readULEB();
391 
392       DEBUG_WITH_TYPE(TgtExecutor::getName(),
393                       dbgs() << CurrentIdx
394                              << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
395                              << InsnID << "])\n");
396       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
397 
398       const MachineInstr *MI = State.MIs[InsnID];
399       assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
400               MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
401              "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
402 
403       if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
404         if (!isBuildVectorAllOnes(*MI, MRI)) {
405           if (handleReject() == RejectAndGiveUp)
406             return false;
407         }
408       } else {
409         if (!isBuildVectorAllZeros(*MI, MRI)) {
410           if (handleReject() == RejectAndGiveUp)
411             return false;
412         }
413       }
414 
415       break;
416     }
417     case GIM_CheckSimplePredicate: {
418       // Note: we don't check for invalid here because this is purely a hook to
419       // allow some executors (such as the combiner) to check arbitrary,
420       // contextless predicates, such as whether a rule is enabled or not.
421       uint16_t Predicate = readU16();
422       DEBUG_WITH_TYPE(TgtExecutor::getName(),
423                       dbgs() << CurrentIdx
424                              << ": GIM_CheckSimplePredicate(Predicate="
425                              << Predicate << ")\n");
426       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
427       if (!testSimplePredicate(Predicate)) {
428         if (handleReject() == RejectAndGiveUp)
429           return false;
430       }
431       break;
432     }
433     case GIM_CheckCxxInsnPredicate: {
434       uint64_t InsnID = readULEB();
435       uint16_t Predicate = readU16();
436       DEBUG_WITH_TYPE(TgtExecutor::getName(),
437                       dbgs()
438                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
439                           << InsnID << "], Predicate=" << Predicate << ")\n");
440       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
441       assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
442 
443       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
444         if (handleReject() == RejectAndGiveUp)
445           return false;
446       break;
447     }
448     case GIM_CheckHasNoUse: {
449       uint64_t InsnID = readULEB();
450 
451       DEBUG_WITH_TYPE(TgtExecutor::getName(),
452                       dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
453                              << InsnID << "]\n");
454 
455       const MachineInstr *MI = State.MIs[InsnID];
456       assert(MI && "Used insn before defined");
457       assert(MI->getNumDefs() > 0 && "No defs");
458       const Register Res = MI->getOperand(0).getReg();
459 
460       if (!MRI.use_nodbg_empty(Res)) {
461         if (handleReject() == RejectAndGiveUp)
462           return false;
463       }
464 
465       break;
466     }
467     case GIM_CheckAtomicOrdering: {
468       uint64_t InsnID = readULEB();
469       auto Ordering = (AtomicOrdering)readULEB();
470       DEBUG_WITH_TYPE(TgtExecutor::getName(),
471                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
472                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
473       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
474       if (!State.MIs[InsnID]->hasOneMemOperand())
475         if (handleReject() == RejectAndGiveUp)
476           return false;
477 
478       for (const auto &MMO : State.MIs[InsnID]->memoperands())
479         if (MMO->getMergedOrdering() != Ordering)
480           if (handleReject() == RejectAndGiveUp)
481             return false;
482       break;
483     }
484     case GIM_CheckAtomicOrderingOrStrongerThan: {
485       uint64_t InsnID = readULEB();
486       auto Ordering = (AtomicOrdering)readULEB();
487       DEBUG_WITH_TYPE(TgtExecutor::getName(),
488                       dbgs() << CurrentIdx
489                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
490                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
491       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
492       if (!State.MIs[InsnID]->hasOneMemOperand())
493         if (handleReject() == RejectAndGiveUp)
494           return false;
495 
496       for (const auto &MMO : State.MIs[InsnID]->memoperands())
497         if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
498           if (handleReject() == RejectAndGiveUp)
499             return false;
500       break;
501     }
502     case GIM_CheckAtomicOrderingWeakerThan: {
503       uint64_t InsnID = readULEB();
504       auto Ordering = (AtomicOrdering)readULEB();
505       DEBUG_WITH_TYPE(TgtExecutor::getName(),
506                       dbgs() << CurrentIdx
507                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
508                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
509       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510       if (!State.MIs[InsnID]->hasOneMemOperand())
511         if (handleReject() == RejectAndGiveUp)
512           return false;
513 
514       for (const auto &MMO : State.MIs[InsnID]->memoperands())
515         if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
516           if (handleReject() == RejectAndGiveUp)
517             return false;
518       break;
519     }
520     case GIM_CheckMemoryAddressSpace: {
521       uint64_t InsnID = readULEB();
522       uint64_t MMOIdx = readULEB();
523       // This accepts a list of possible address spaces.
524       const uint64_t NumAddrSpace = readULEB();
525 
526       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
527         if (handleReject() == RejectAndGiveUp)
528           return false;
529         break;
530       }
531 
532       // Need to still jump to the end of the list of address spaces if we find
533       // a match earlier.
534       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
535 
536       const MachineMemOperand *MMO =
537           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
538       const unsigned MMOAddrSpace = MMO->getAddrSpace();
539 
540       bool Success = false;
541       for (unsigned I = 0; I != NumAddrSpace; ++I) {
542         uint64_t AddrSpace = readULEB();
543         DEBUG_WITH_TYPE(TgtExecutor::getName(),
544                         dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
545                                << AddrSpace << '\n');
546 
547         if (AddrSpace == MMOAddrSpace) {
548           Success = true;
549           break;
550         }
551       }
552 
553       CurrentIdx = LastIdx;
554       if (!Success && handleReject() == RejectAndGiveUp)
555         return false;
556       break;
557     }
558     case GIM_CheckMemoryAlignment: {
559       uint64_t InsnID = readULEB();
560       uint64_t MMOIdx = readULEB();
561       uint64_t MinAlign = readULEB();
562 
563       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
564 
565       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
566         if (handleReject() == RejectAndGiveUp)
567           return false;
568         break;
569       }
570 
571       MachineMemOperand *MMO =
572           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
573       DEBUG_WITH_TYPE(TgtExecutor::getName(),
574                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
575                              << "(MIs[" << InsnID << "]->memoperands() + "
576                              << MMOIdx << ")->getAlignment() >= " << MinAlign
577                              << ")\n");
578       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
579         return false;
580 
581       break;
582     }
583     case GIM_CheckMemorySizeEqualTo: {
584       uint64_t InsnID = readULEB();
585       uint64_t MMOIdx = readULEB();
586       uint32_t Size = readU32();
587 
588       DEBUG_WITH_TYPE(TgtExecutor::getName(),
589                       dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
590                              << InsnID << "]->memoperands() + " << MMOIdx
591                              << ", Size=" << Size << ")\n");
592       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593 
594       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
595         if (handleReject() == RejectAndGiveUp)
596           return false;
597         break;
598       }
599 
600       MachineMemOperand *MMO =
601           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
602 
603       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
604                                                      << " bytes vs " << Size
605                                                      << " bytes\n");
606       if (MMO->getSize() != Size)
607         if (handleReject() == RejectAndGiveUp)
608           return false;
609 
610       break;
611     }
612     case GIM_CheckMemorySizeEqualToLLT:
613     case GIM_CheckMemorySizeLessThanLLT:
614     case GIM_CheckMemorySizeGreaterThanLLT: {
615       uint64_t InsnID = readULEB();
616       uint64_t MMOIdx = readULEB();
617       uint64_t OpIdx = readULEB();
618 
619       DEBUG_WITH_TYPE(
620           TgtExecutor::getName(),
621           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
622                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
623                      : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
624                          ? "GreaterThan"
625                          : "LessThan")
626                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
627                  << ", OpIdx=" << OpIdx << ")\n");
628       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
629 
630       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
631       if (!MO.isReg()) {
632         DEBUG_WITH_TYPE(TgtExecutor::getName(),
633                         dbgs() << CurrentIdx << ": Not a register\n");
634         if (handleReject() == RejectAndGiveUp)
635           return false;
636         break;
637       }
638 
639       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
640         if (handleReject() == RejectAndGiveUp)
641           return false;
642         break;
643       }
644 
645       MachineMemOperand *MMO =
646           *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
647 
648       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
649       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
650           MMO->getSizeInBits() != Size) {
651         if (handleReject() == RejectAndGiveUp)
652           return false;
653       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
654                  MMO->getSizeInBits() >= Size) {
655         if (handleReject() == RejectAndGiveUp)
656           return false;
657       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
658                  MMO->getSizeInBits() <= Size)
659         if (handleReject() == RejectAndGiveUp)
660           return false;
661 
662       break;
663     }
664     case GIM_CheckType: {
665       uint64_t InsnID = readULEB();
666       uint64_t OpIdx = readULEB();
667       int TypeID = readS8();
668       DEBUG_WITH_TYPE(TgtExecutor::getName(),
669                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
670                              << "]->getOperand(" << OpIdx
671                              << "), TypeID=" << TypeID << ")\n");
672       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
673       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
674       if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
675         if (handleReject() == RejectAndGiveUp)
676           return false;
677       }
678       break;
679     }
680     case GIM_CheckPointerToAny: {
681       uint64_t InsnID = readULEB();
682       uint64_t OpIdx = readULEB();
683       uint64_t SizeInBits = readULEB();
684 
685       DEBUG_WITH_TYPE(TgtExecutor::getName(),
686                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
687                              << InsnID << "]->getOperand(" << OpIdx
688                              << "), SizeInBits=" << SizeInBits << ")\n");
689       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
690       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
691       const LLT Ty = MRI.getType(MO.getReg());
692 
693       // iPTR must be looked up in the target.
694       if (SizeInBits == 0) {
695         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
696         const unsigned AddrSpace = Ty.getAddressSpace();
697         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
698       }
699 
700       assert(SizeInBits != 0 && "Pointer size must be known");
701 
702       if (MO.isReg()) {
703         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
704           if (handleReject() == RejectAndGiveUp)
705             return false;
706       } else if (handleReject() == RejectAndGiveUp)
707         return false;
708 
709       break;
710     }
711     case GIM_RecordNamedOperand: {
712       uint64_t InsnID = readULEB();
713       uint64_t OpIdx = readULEB();
714       uint64_t StoreIdx = readULEB();
715 
716       DEBUG_WITH_TYPE(TgtExecutor::getName(),
717                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
718                              << InsnID << "]->getOperand(" << OpIdx
719                              << "), StoreIdx=" << StoreIdx << ")\n");
720       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
721       assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
722       State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
723       break;
724     }
725     case GIM_RecordRegType: {
726       uint64_t InsnID = readULEB();
727       uint64_t OpIdx = readULEB();
728       int TypeIdx = readS8();
729 
730       DEBUG_WITH_TYPE(TgtExecutor::getName(),
731                       dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
732                              << InsnID << "]->getOperand(" << OpIdx
733                              << "), TypeIdx=" << TypeIdx << ")\n");
734       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
735       assert(TypeIdx < 0 && "Temp types always have negative indexes!");
736       // Indexes start at -1.
737       TypeIdx = 1 - TypeIdx;
738       const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
739       if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
740         State.RecordedTypes.resize(TypeIdx + 1, LLT());
741       State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
742       break;
743     }
744     case GIM_CheckRegBankForClass: {
745       uint64_t InsnID = readULEB();
746       uint64_t OpIdx = readULEB();
747       uint16_t RCEnum = readU16();
748       DEBUG_WITH_TYPE(TgtExecutor::getName(),
749                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
750                              << InsnID << "]->getOperand(" << OpIdx
751                              << "), RCEnum=" << RCEnum << ")\n");
752       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
753       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
754       if (!MO.isReg() ||
755           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
756                                       MRI.getType(MO.getReg())) !=
757               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
758         if (handleReject() == RejectAndGiveUp)
759           return false;
760       }
761       break;
762     }
763 
764     case GIM_CheckComplexPattern: {
765       uint64_t InsnID = readULEB();
766       uint64_t OpIdx = readULEB();
767       uint16_t RendererID = readU16();
768       uint16_t ComplexPredicateID = readU16();
769       DEBUG_WITH_TYPE(TgtExecutor::getName(),
770                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
771                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
772                              << "]->getOperand(" << OpIdx
773                              << "), ComplexPredicateID=" << ComplexPredicateID
774                              << ")\n");
775       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
776       // FIXME: Use std::invoke() when it's available.
777       ComplexRendererFns Renderer =
778           (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
779               State.MIs[InsnID]->getOperand(OpIdx));
780       if (Renderer)
781         State.Renderers[RendererID] = *Renderer;
782       else if (handleReject() == RejectAndGiveUp)
783         return false;
784       break;
785     }
786 
787     case GIM_CheckConstantInt:
788     case GIM_CheckConstantInt8: {
789       const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
790 
791       uint64_t InsnID = readULEB();
792       uint64_t OpIdx = readULEB();
793       uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
794       DEBUG_WITH_TYPE(TgtExecutor::getName(),
795                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
796                              << InsnID << "]->getOperand(" << OpIdx
797                              << "), Value=" << Value << ")\n");
798       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
799       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
800       if (MO.isReg()) {
801         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
802         LLT Ty = MRI.getType(MO.getReg());
803         // If the type is > 64 bits, it can't be a constant int, so we bail
804         // early because SignExtend64 will assert otherwise.
805         if (Ty.getScalarSizeInBits() > 64) {
806           if (handleReject() == RejectAndGiveUp)
807             return false;
808           break;
809         }
810 
811         Value = SignExtend64(Value, Ty.getScalarSizeInBits());
812         if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
813           if (handleReject() == RejectAndGiveUp)
814             return false;
815         }
816       } else if (handleReject() == RejectAndGiveUp)
817         return false;
818 
819       break;
820     }
821 
822     case GIM_CheckLiteralInt: {
823       uint64_t InsnID = readULEB();
824       uint64_t OpIdx = readULEB();
825       int64_t Value = readU64();
826       DEBUG_WITH_TYPE(TgtExecutor::getName(),
827                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
828                              << InsnID << "]->getOperand(" << OpIdx
829                              << "), Value=" << Value << ")\n");
830       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
831       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
832       if (MO.isImm() && MO.getImm() == Value)
833         break;
834 
835       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
836         break;
837 
838       if (handleReject() == RejectAndGiveUp)
839         return false;
840 
841       break;
842     }
843 
844     case GIM_CheckIntrinsicID: {
845       uint64_t InsnID = readULEB();
846       uint64_t OpIdx = readULEB();
847       uint16_t Value = readU16();
848       DEBUG_WITH_TYPE(TgtExecutor::getName(),
849                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
850                              << InsnID << "]->getOperand(" << OpIdx
851                              << "), Value=" << Value << ")\n");
852       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
853       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
854       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
855         if (handleReject() == RejectAndGiveUp)
856           return false;
857       break;
858     }
859     case GIM_CheckCmpPredicate: {
860       uint64_t InsnID = readULEB();
861       uint64_t OpIdx = readULEB();
862       uint16_t Value = readU16();
863       DEBUG_WITH_TYPE(TgtExecutor::getName(),
864                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
865                              << InsnID << "]->getOperand(" << OpIdx
866                              << "), Value=" << Value << ")\n");
867       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
868       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
869       if (!MO.isPredicate() || MO.getPredicate() != Value)
870         if (handleReject() == RejectAndGiveUp)
871           return false;
872       break;
873     }
874     case GIM_CheckIsMBB: {
875       uint64_t InsnID = readULEB();
876       uint64_t OpIdx = readULEB();
877       DEBUG_WITH_TYPE(TgtExecutor::getName(),
878                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
879                              << "]->getOperand(" << OpIdx << "))\n");
880       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
881       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
882         if (handleReject() == RejectAndGiveUp)
883           return false;
884       }
885       break;
886     }
887     case GIM_CheckIsImm: {
888       uint64_t InsnID = readULEB();
889       uint64_t OpIdx = readULEB();
890       DEBUG_WITH_TYPE(TgtExecutor::getName(),
891                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
892                              << "]->getOperand(" << OpIdx << "))\n");
893       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
894       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
895         if (handleReject() == RejectAndGiveUp)
896           return false;
897       }
898       break;
899     }
900     case GIM_CheckIsSafeToFold: {
901       uint64_t InsnID = readULEB();
902       DEBUG_WITH_TYPE(TgtExecutor::getName(),
903                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
904                              << InsnID << "])\n");
905       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
906       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
907         if (handleReject() == RejectAndGiveUp)
908           return false;
909       }
910       break;
911     }
912     case GIM_CheckIsSameOperand:
913     case GIM_CheckIsSameOperandIgnoreCopies: {
914       uint64_t InsnID = readULEB();
915       uint64_t OpIdx = readULEB();
916       uint64_t OtherInsnID = readULEB();
917       uint64_t OtherOpIdx = readULEB();
918       DEBUG_WITH_TYPE(TgtExecutor::getName(),
919                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
920                              << InsnID << "][" << OpIdx << "], MIs["
921                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
922       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
923       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
924 
925       MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
926       MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
927 
928       if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
929         if (Op.isReg() && OtherOp.isReg()) {
930           if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
931               getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
932             break;
933         }
934       }
935 
936       if (!Op.isIdenticalTo(OtherOp)) {
937         if (handleReject() == RejectAndGiveUp)
938           return false;
939       }
940       break;
941     }
942     case GIM_CheckCanReplaceReg: {
943       uint64_t OldInsnID = readULEB();
944       uint64_t OldOpIdx = readULEB();
945       uint64_t NewInsnID = readULEB();
946       uint64_t NewOpIdx = readULEB();
947 
948       DEBUG_WITH_TYPE(TgtExecutor::getName(),
949                       dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
950                              << OldInsnID << "][" << OldOpIdx << "] = MIs["
951                              << NewInsnID << "][" << NewOpIdx << "])\n");
952 
953       Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
954       Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
955       if (!canReplaceReg(Old, New, MRI)) {
956         if (handleReject() == RejectAndGiveUp)
957           return false;
958       }
959       break;
960     }
961     case GIM_MIFlags: {
962       uint64_t InsnID = readULEB();
963       uint32_t Flags = readU32();
964 
965       DEBUG_WITH_TYPE(TgtExecutor::getName(),
966                       dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
967                              << "], " << Flags << ")\n");
968       if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
969         if (handleReject() == RejectAndGiveUp)
970           return false;
971       }
972       break;
973     }
974     case GIM_MIFlagsNot: {
975       uint64_t InsnID = readULEB();
976       uint32_t Flags = readU32();
977 
978       DEBUG_WITH_TYPE(TgtExecutor::getName(),
979                       dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
980                              << "], " << Flags << ")\n");
981       if ((State.MIs[InsnID]->getFlags() & Flags)) {
982         if (handleReject() == RejectAndGiveUp)
983           return false;
984       }
985       break;
986     }
987     case GIM_Reject:
988       DEBUG_WITH_TYPE(TgtExecutor::getName(),
989                       dbgs() << CurrentIdx << ": GIM_Reject\n");
990       if (handleReject() == RejectAndGiveUp)
991         return false;
992       break;
993     case GIR_MutateOpcode: {
994       uint64_t OldInsnID = readULEB();
995       uint64_t NewInsnID = readULEB();
996       uint16_t NewOpcode = readU16();
997       if (NewInsnID >= OutMIs.size())
998         OutMIs.resize(NewInsnID + 1);
999 
1000       MachineInstr *OldMI = State.MIs[OldInsnID];
1001       if (Observer)
1002         Observer->changingInstr(*OldMI);
1003       OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1004       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1005       if (Observer)
1006         Observer->changedInstr(*OldMI);
1007       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1008                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1009                              << NewInsnID << "], MIs[" << OldInsnID << "], "
1010                              << NewOpcode << ")\n");
1011       break;
1012     }
1013 
1014     case GIR_BuildMI: {
1015       uint64_t NewInsnID = readULEB();
1016       uint16_t Opcode = readU16();
1017       if (NewInsnID >= OutMIs.size())
1018         OutMIs.resize(NewInsnID + 1);
1019 
1020       OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1021       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1022                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1023                              << NewInsnID << "], " << Opcode << ")\n");
1024       break;
1025     }
1026 
1027     case GIR_BuildConstant: {
1028       uint64_t TempRegID = readULEB();
1029       uint64_t Imm = readU64();
1030       Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1031       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1032                       dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1033                              << TempRegID << "], Imm=" << Imm << ")\n");
1034       break;
1035     }
1036 
1037     case GIR_Copy: {
1038       uint64_t NewInsnID = readULEB();
1039       uint64_t OldInsnID = readULEB();
1040       uint64_t OpIdx = readULEB();
1041       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1042       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1043       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1044                       dbgs()
1045                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1046                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1047       break;
1048     }
1049 
1050     case GIR_CopyOrAddZeroReg: {
1051       uint64_t NewInsnID = readULEB();
1052       uint64_t OldInsnID = readULEB();
1053       uint64_t OpIdx = readULEB();
1054       uint16_t ZeroReg = readU16();
1055       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1056       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1057       if (isOperandImmEqual(MO, 0, MRI))
1058         OutMIs[NewInsnID].addReg(ZeroReg);
1059       else
1060         OutMIs[NewInsnID].add(MO);
1061       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1062                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1063                              << NewInsnID << "], MIs[" << OldInsnID << "], "
1064                              << OpIdx << ", " << ZeroReg << ")\n");
1065       break;
1066     }
1067 
1068     case GIR_CopySubReg: {
1069       uint64_t NewInsnID = readULEB();
1070       uint64_t OldInsnID = readULEB();
1071       uint64_t OpIdx = readULEB();
1072       uint16_t SubRegIdx = readU16();
1073       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1074       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1075                                0, SubRegIdx);
1076       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1077                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1078                              << NewInsnID << "], MIs[" << OldInsnID << "], "
1079                              << OpIdx << ", " << SubRegIdx << ")\n");
1080       break;
1081     }
1082 
1083     case GIR_AddImplicitDef: {
1084       uint64_t InsnID = readULEB();
1085       uint16_t RegNum = readU16();
1086       uint16_t Flags = readU16();
1087       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1088       Flags |= RegState::Implicit;
1089       OutMIs[InsnID].addDef(RegNum, Flags);
1090       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1091                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1092                              << InsnID << "], " << RegNum << ")\n");
1093       break;
1094     }
1095 
1096     case GIR_AddImplicitUse: {
1097       uint64_t InsnID = readULEB();
1098       uint16_t RegNum = readU16();
1099       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1100       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1101       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1102                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1103                              << InsnID << "], " << RegNum << ")\n");
1104       break;
1105     }
1106 
1107     case GIR_AddRegister: {
1108       uint64_t InsnID = readULEB();
1109       uint16_t RegNum = readU16();
1110       uint16_t RegFlags = readU16();
1111       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1112       OutMIs[InsnID].addReg(RegNum, RegFlags);
1113       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1114                       dbgs()
1115                           << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1116                           << "], " << RegNum << ", " << RegFlags << ")\n");
1117       break;
1118     }
1119     case GIR_SetImplicitDefDead: {
1120       uint64_t InsnID = readULEB();
1121       uint64_t OpIdx = readULEB();
1122       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1123                       dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1124                              << InsnID << "], OpIdx=" << OpIdx << ")\n");
1125       MachineInstr *MI = OutMIs[InsnID];
1126       assert(MI && "Modifying undefined instruction");
1127       MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1128       break;
1129     }
1130     case GIR_SetMIFlags: {
1131       uint64_t InsnID = readULEB();
1132       uint32_t Flags = readU32();
1133 
1134       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1135                       dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1136                              << InsnID << "], " << Flags << ")\n");
1137       MachineInstr *MI = OutMIs[InsnID];
1138       MI->setFlags(MI->getFlags() | Flags);
1139       break;
1140     }
1141     case GIR_UnsetMIFlags: {
1142       uint64_t InsnID = readULEB();
1143       uint32_t Flags = readU32();
1144 
1145       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1146                       dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1147                              << InsnID << "], " << Flags << ")\n");
1148       MachineInstr *MI = OutMIs[InsnID];
1149       MI->setFlags(MI->getFlags() & ~Flags);
1150       break;
1151     }
1152     case GIR_CopyMIFlags: {
1153       uint64_t InsnID = readULEB();
1154       uint64_t OldInsnID = readULEB();
1155 
1156       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1157                       dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1158                              << InsnID << "], MIs[" << OldInsnID << "])\n");
1159       MachineInstr *MI = OutMIs[InsnID];
1160       MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1161       break;
1162     }
1163     case GIR_AddSimpleTempRegister:
1164     case GIR_AddTempRegister:
1165     case GIR_AddTempSubRegister: {
1166       uint64_t InsnID = readULEB();
1167       uint64_t TempRegID = readULEB();
1168       uint16_t TempRegFlags = 0;
1169       if (MatcherOpcode != GIR_AddSimpleTempRegister)
1170         TempRegFlags = readU16();
1171       uint16_t SubReg = 0;
1172       if (MatcherOpcode == GIR_AddTempSubRegister)
1173         SubReg = readU16();
1174 
1175       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1176 
1177       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1178                             SubReg);
1179       DEBUG_WITH_TYPE(
1180           TgtExecutor::getName(),
1181           dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1182                  << "], TempRegisters[" << TempRegID << "]";
1183           if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1184           dbgs() << ", " << TempRegFlags << ")\n");
1185       break;
1186     }
1187 
1188     case GIR_AddImm8:
1189     case GIR_AddImm: {
1190       const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1191       uint64_t InsnID = readULEB();
1192       uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1193       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1194       OutMIs[InsnID].addImm(Imm);
1195       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1196                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1197                              << "], " << Imm << ")\n");
1198       break;
1199     }
1200 
1201     case GIR_AddCImm: {
1202       uint64_t InsnID = readULEB();
1203       int TypeID = readS8();
1204       uint64_t Imm = readU64();
1205       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1206 
1207       unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1208       LLVMContext &Ctx = MF->getFunction().getContext();
1209       OutMIs[InsnID].addCImm(
1210           ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1211       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1212                       dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1213                              << "], TypeID=" << TypeID << ", Imm=" << Imm
1214                              << ")\n");
1215       break;
1216     }
1217 
1218     case GIR_ComplexRenderer: {
1219       uint64_t InsnID = readULEB();
1220       uint16_t RendererID = readU16();
1221       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1222       for (const auto &RenderOpFn : State.Renderers[RendererID])
1223         RenderOpFn(OutMIs[InsnID]);
1224       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1225                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1226                              << InsnID << "], " << RendererID << ")\n");
1227       break;
1228     }
1229     case GIR_ComplexSubOperandRenderer: {
1230       uint64_t InsnID = readULEB();
1231       uint16_t RendererID = readU16();
1232       uint64_t RenderOpID = readULEB();
1233       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1234       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1235       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1236                       dbgs() << CurrentIdx
1237                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
1238                              << InsnID << "], " << RendererID << ", "
1239                              << RenderOpID << ")\n");
1240       break;
1241     }
1242     case GIR_ComplexSubOperandSubRegRenderer: {
1243       uint64_t InsnID = readULEB();
1244       uint16_t RendererID = readU16();
1245       uint64_t RenderOpID = readULEB();
1246       uint16_t SubRegIdx = readU16();
1247       MachineInstrBuilder &MI = OutMIs[InsnID];
1248       assert(MI && "Attempted to add to undefined instruction");
1249       State.Renderers[RendererID][RenderOpID](MI);
1250       MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1251       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1252                       dbgs() << CurrentIdx
1253                              << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1254                              << InsnID << "], " << RendererID << ", "
1255                              << RenderOpID << ", " << SubRegIdx << ")\n");
1256       break;
1257     }
1258 
1259     case GIR_CopyConstantAsSImm: {
1260       uint64_t NewInsnID = readULEB();
1261       uint64_t OldInsnID = readULEB();
1262       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1263       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1264              "Expected G_CONSTANT");
1265       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1266         OutMIs[NewInsnID].addImm(
1267             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1268       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1269         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1270       else
1271         llvm_unreachable("Expected Imm or CImm operand");
1272       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1273                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1274                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1275       break;
1276     }
1277 
1278     // TODO: Needs a test case once we have a pattern that uses this.
1279     case GIR_CopyFConstantAsFPImm: {
1280       uint64_t NewInsnID = readULEB();
1281       uint64_t OldInsnID = readULEB();
1282       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1283       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1284              "Expected G_FCONSTANT");
1285       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1286         OutMIs[NewInsnID].addFPImm(
1287             State.MIs[OldInsnID]->getOperand(1).getFPImm());
1288       else
1289         llvm_unreachable("Expected FPImm operand");
1290       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1291                       dbgs()
1292                           << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1293                           << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1294       break;
1295     }
1296 
1297     case GIR_CustomRenderer: {
1298       uint64_t InsnID = readULEB();
1299       uint64_t OldInsnID = readULEB();
1300       uint16_t RendererFnID = readU16();
1301       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1302       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1303                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1304                              << InsnID << "], MIs[" << OldInsnID << "], "
1305                              << RendererFnID << ")\n");
1306       (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1307           OutMIs[InsnID], *State.MIs[OldInsnID],
1308           -1); // Not a source operand of the old instruction.
1309       break;
1310     }
1311     case GIR_CustomAction: {
1312       uint16_t FnID = readU16();
1313       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1314                       dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
1315                              << ")\n");
1316       assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1317       runCustomAction(FnID, State, OutMIs);
1318       break;
1319     }
1320     case GIR_CustomOperandRenderer: {
1321       uint64_t InsnID = readULEB();
1322       uint64_t OldInsnID = readULEB();
1323       uint64_t OpIdx = readULEB();
1324       uint16_t RendererFnID = readU16();
1325       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1326 
1327       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1328                       dbgs() << CurrentIdx
1329                              << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1330                              << "], MIs[" << OldInsnID << "]->getOperand("
1331                              << OpIdx << "), " << RendererFnID << ")\n");
1332       (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1333           OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1334       break;
1335     }
1336     case GIR_ConstrainOperandRC: {
1337       uint64_t InsnID = readULEB();
1338       uint64_t OpIdx = readULEB();
1339       uint16_t RCEnum = readU16();
1340       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1341       MachineInstr &I = *OutMIs[InsnID].getInstr();
1342       MachineFunction &MF = *I.getParent()->getParent();
1343       MachineRegisterInfo &MRI = MF.getRegInfo();
1344       const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1345       MachineOperand &MO = I.getOperand(OpIdx);
1346       constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1347       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1348                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1349                              << InsnID << "], " << OpIdx << ", " << RCEnum
1350                              << ")\n");
1351       break;
1352     }
1353 
1354     case GIR_ConstrainSelectedInstOperands: {
1355       uint64_t InsnID = readULEB();
1356       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1357       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1358                                        RBI);
1359       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1360                       dbgs() << CurrentIdx
1361                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1362                              << InsnID << "])\n");
1363       break;
1364     }
1365 
1366     case GIR_MergeMemOperands: {
1367       uint64_t InsnID = readULEB();
1368       uint64_t NumInsn = MatchTable[CurrentIdx++];
1369       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1370 
1371       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1372                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1373                              << InsnID << "]");
1374       for (unsigned K = 0; K < NumInsn; ++K) {
1375         uint64_t NextID = readULEB();
1376         DEBUG_WITH_TYPE(TgtExecutor::getName(),
1377                         dbgs() << ", MIs[" << NextID << "]");
1378         for (const auto &MMO : State.MIs[NextID]->memoperands())
1379           OutMIs[InsnID].addMemOperand(MMO);
1380       }
1381       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1382       break;
1383     }
1384 
1385     case GIR_EraseFromParent: {
1386       uint64_t InsnID = readULEB();
1387       MachineInstr *MI = State.MIs[InsnID];
1388       assert(MI && "Attempted to erase an undefined instruction");
1389       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1390                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1391                              << InsnID << "])\n");
1392       // If we're erasing the insertion point, ensure we don't leave a dangling
1393       // pointer in the builder.
1394       if (Builder.getInsertPt() == MI)
1395         Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
1396       if (Observer)
1397         Observer->erasingInstr(*MI);
1398       MI->eraseFromParent();
1399       break;
1400     }
1401 
1402     case GIR_MakeTempReg: {
1403       uint64_t TempRegID = readULEB();
1404       int TypeID = readS8();
1405 
1406       State.TempRegisters[TempRegID] =
1407           MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1408       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1409                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1410                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1411       break;
1412     }
1413     case GIR_ReplaceReg: {
1414       uint64_t OldInsnID = readULEB();
1415       uint64_t OldOpIdx = readULEB();
1416       uint64_t NewInsnID = readULEB();
1417       uint64_t NewOpIdx = readULEB();
1418 
1419       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1420                       dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1421                              << OldInsnID << "][" << OldOpIdx << "] = MIs["
1422                              << NewInsnID << "][" << NewOpIdx << "])\n");
1423 
1424       Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1425       Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1426       if (Observer)
1427         Observer->changingAllUsesOfReg(MRI, Old);
1428       MRI.replaceRegWith(Old, New);
1429       if (Observer)
1430         Observer->finishedChangingAllUsesOfReg();
1431       break;
1432     }
1433     case GIR_ReplaceRegWithTempReg: {
1434       uint64_t OldInsnID = readULEB();
1435       uint64_t OldOpIdx = readULEB();
1436       uint64_t TempRegID = readULEB();
1437 
1438       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1439                       dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1440                              << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1441                              << TempRegID << "])\n");
1442 
1443       Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1444       Register New = State.TempRegisters[TempRegID];
1445       if (Observer)
1446         Observer->changingAllUsesOfReg(MRI, Old);
1447       MRI.replaceRegWith(Old, New);
1448       if (Observer)
1449         Observer->finishedChangingAllUsesOfReg();
1450       break;
1451     }
1452     case GIR_Coverage: {
1453       uint32_t RuleID = readU32();
1454       assert(CoverageInfo);
1455       CoverageInfo->setCovered(RuleID);
1456 
1457       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1458                                                      << ": GIR_Coverage("
1459                                                      << RuleID << ")");
1460       break;
1461     }
1462 
1463     case GIR_Done:
1464       DEBUG_WITH_TYPE(TgtExecutor::getName(),
1465                       dbgs() << CurrentIdx << ": GIR_Done\n");
1466       propagateFlags();
1467       return true;
1468     default:
1469       llvm_unreachable("Unexpected command");
1470     }
1471   }
1472 }
1473 
1474 } // end namespace llvm
1475 
1476 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1477