Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

Rev

Blame | Last modification | View Log | Download | RSS feed

  1. //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. //
  9. /// \file This file declares the API for the instruction selector.
  10. /// This class is responsible for selecting machine instructions.
  11. /// It's implemented by the target. It's used by the InstructionSelect pass.
  12. //
  13. //===----------------------------------------------------------------------===//
  14.  
  15. #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  16. #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17.  
  18. #include "llvm/ADT/SmallVector.h"
  19. #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  20. #include "llvm/CodeGen/GlobalISel/Utils.h"
  21. #include "llvm/CodeGen/MachineInstrBuilder.h"
  22. #include "llvm/CodeGen/MachineOperand.h"
  23. #include "llvm/CodeGen/MachineRegisterInfo.h"
  24. #include "llvm/CodeGen/RegisterBankInfo.h"
  25. #include "llvm/CodeGen/TargetInstrInfo.h"
  26. #include "llvm/CodeGen/TargetOpcodes.h"
  27. #include "llvm/CodeGen/TargetRegisterInfo.h"
  28. #include "llvm/IR/Constants.h"
  29. #include "llvm/IR/DataLayout.h"
  30. #include "llvm/Support/CodeGenCoverage.h"
  31. #include "llvm/Support/Debug.h"
  32. #include "llvm/Support/ErrorHandling.h"
  33. #include "llvm/Support/raw_ostream.h"
  34. #include <cassert>
  35. #include <cstddef>
  36. #include <cstdint>
  37.  
  38. namespace llvm {
  39.  
  40. /// GlobalISel PatFrag Predicates
  41. enum {
  42.   GIPFP_I64_Invalid = 0,
  43.   GIPFP_APInt_Invalid = 0,
  44.   GIPFP_APFloat_Invalid = 0,
  45.   GIPFP_MI_Invalid = 0,
  46. };
  47.  
  48. template <class TgtInstructionSelector, class PredicateBitset,
  49.           class ComplexMatcherMemFn, class CustomRendererFn>
  50. bool InstructionSelector::executeMatchTable(
  51.     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  52.     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
  53.         &ISelInfo,
  54.     const int64_t *MatchTable, const TargetInstrInfo &TII,
  55.     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  56.     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  57.     CodeGenCoverage &CoverageInfo) const {
  58.  
  59.   uint64_t CurrentIdx = 0;
  60.   SmallVector<uint64_t, 4> OnFailResumeAt;
  61.  
  62.   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
  63.   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
  64.  
  65.   const uint16_t Flags = State.MIs[0]->getFlags();
  66.  
  67.   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  68.   auto handleReject = [&]() -> RejectAction {
  69.     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  70.                     dbgs() << CurrentIdx << ": Rejected\n");
  71.     if (OnFailResumeAt.empty())
  72.       return RejectAndGiveUp;
  73.     CurrentIdx = OnFailResumeAt.pop_back_val();
  74.     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  75.                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  76.                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  77.     return RejectAndResume;
  78.   };
  79.  
  80.   auto propagateFlags = [=](NewMIVector &OutMIs) {
  81.     for (auto MIB : OutMIs) {
  82.       // Set the NoFPExcept flag when no original matched instruction could
  83.       // raise an FP exception, but the new instruction potentially might.
  84.       uint16_t MIBFlags = Flags;
  85.       if (NoFPException && MIB->mayRaiseFPException())
  86.         MIBFlags |= MachineInstr::NoFPExcept;
  87.       MIB.setMIFlags(MIBFlags);
  88.     }
  89.  
  90.     return true;
  91.   };
  92.  
  93.   while (true) {
  94.     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  95.     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
  96.     switch (MatcherOpcode) {
  97.     case GIM_Try: {
  98.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  99.                       dbgs() << CurrentIdx << ": Begin try-block\n");
  100.       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
  101.       break;
  102.     }
  103.  
  104.     case GIM_RecordInsn: {
  105.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  106.       int64_t InsnID = MatchTable[CurrentIdx++];
  107.       int64_t OpIdx = MatchTable[CurrentIdx++];
  108.  
  109.       // As an optimisation we require that MIs[0] is always the root. Refuse
  110.       // any attempt to modify it.
  111.       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
  112.  
  113.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  114.       if (!MO.isReg()) {
  115.         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  116.                         dbgs() << CurrentIdx << ": Not a register\n");
  117.         if (handleReject() == RejectAndGiveUp)
  118.           return false;
  119.         break;
  120.       }
  121.       if (MO.getReg().isPhysical()) {
  122.         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  123.                         dbgs() << CurrentIdx << ": Is a physical register\n");
  124.         if (handleReject() == RejectAndGiveUp)
  125.           return false;
  126.         break;
  127.       }
  128.  
  129.       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
  130.       if ((size_t)NewInsnID < State.MIs.size())
  131.         State.MIs[NewInsnID] = NewMI;
  132.       else {
  133.         assert((size_t)NewInsnID == State.MIs.size() &&
  134.                "Expected to store MIs in order");
  135.         State.MIs.push_back(NewMI);
  136.       }
  137.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  138.                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
  139.                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
  140.                              << ")\n");
  141.       break;
  142.     }
  143.  
  144.     case GIM_CheckFeatures: {
  145.       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
  146.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  147.                       dbgs() << CurrentIdx
  148.                              << ": GIM_CheckFeatures(ExpectedBitsetID="
  149.                              << ExpectedBitsetID << ")\n");
  150.       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
  151.           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
  152.         if (handleReject() == RejectAndGiveUp)
  153.           return false;
  154.       }
  155.       break;
  156.     }
  157.  
  158.     case GIM_CheckOpcode:
  159.     case GIM_CheckOpcodeIsEither: {
  160.       int64_t InsnID = MatchTable[CurrentIdx++];
  161.       int64_t Expected0 = MatchTable[CurrentIdx++];
  162.       int64_t Expected1 = -1;
  163.       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
  164.         Expected1 = MatchTable[CurrentIdx++];
  165.  
  166.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  167.       unsigned Opcode = State.MIs[InsnID]->getOpcode();
  168.  
  169.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  170.         dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
  171.         << "], ExpectedOpcode=" << Expected0;
  172.         if (MatcherOpcode == GIM_CheckOpcodeIsEither)
  173.           dbgs() << " || " << Expected1;
  174.         dbgs() << ") // Got=" << Opcode << "\n";
  175.       );
  176.  
  177.       if (Opcode != Expected0 && Opcode != Expected1) {
  178.         if (handleReject() == RejectAndGiveUp)
  179.           return false;
  180.       }
  181.       break;
  182.     }
  183.     case GIM_SwitchOpcode: {
  184.       int64_t InsnID = MatchTable[CurrentIdx++];
  185.       int64_t LowerBound = MatchTable[CurrentIdx++];
  186.       int64_t UpperBound = MatchTable[CurrentIdx++];
  187.       int64_t Default = MatchTable[CurrentIdx++];
  188.  
  189.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  190.       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
  191.  
  192.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
  193.         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
  194.                << LowerBound << ", " << UpperBound << "), Default=" << Default
  195.                << ", JumpTable...) // Got=" << Opcode << "\n";
  196.       });
  197.       if (Opcode < LowerBound || UpperBound <= Opcode) {
  198.         CurrentIdx = Default;
  199.         break;
  200.       }
  201.       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
  202.       if (!CurrentIdx) {
  203.         CurrentIdx = Default;
  204.         break;
  205.       }
  206.       OnFailResumeAt.push_back(Default);
  207.       break;
  208.     }
  209.  
  210.     case GIM_SwitchType: {
  211.       int64_t InsnID = MatchTable[CurrentIdx++];
  212.       int64_t OpIdx = MatchTable[CurrentIdx++];
  213.       int64_t LowerBound = MatchTable[CurrentIdx++];
  214.       int64_t UpperBound = MatchTable[CurrentIdx++];
  215.       int64_t Default = MatchTable[CurrentIdx++];
  216.  
  217.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  218.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  219.  
  220.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
  221.         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
  222.                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
  223.                << UpperBound << "), Default=" << Default
  224.                << ", JumpTable...) // Got=";
  225.         if (!MO.isReg())
  226.           dbgs() << "Not a VReg\n";
  227.         else
  228.           dbgs() << MRI.getType(MO.getReg()) << "\n";
  229.       });
  230.       if (!MO.isReg()) {
  231.         CurrentIdx = Default;
  232.         break;
  233.       }
  234.       const LLT Ty = MRI.getType(MO.getReg());
  235.       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
  236.       if (TyI == ISelInfo.TypeIDMap.end()) {
  237.         CurrentIdx = Default;
  238.         break;
  239.       }
  240.       const int64_t TypeID = TyI->second;
  241.       if (TypeID < LowerBound || UpperBound <= TypeID) {
  242.         CurrentIdx = Default;
  243.         break;
  244.       }
  245.       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
  246.       if (!CurrentIdx) {
  247.         CurrentIdx = Default;
  248.         break;
  249.       }
  250.       OnFailResumeAt.push_back(Default);
  251.       break;
  252.     }
  253.  
  254.     case GIM_CheckNumOperands: {
  255.       int64_t InsnID = MatchTable[CurrentIdx++];
  256.       int64_t Expected = MatchTable[CurrentIdx++];
  257.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  258.                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
  259.                              << InsnID << "], Expected=" << Expected << ")\n");
  260.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  261.       if (State.MIs[InsnID]->getNumOperands() != Expected) {
  262.         if (handleReject() == RejectAndGiveUp)
  263.           return false;
  264.       }
  265.       break;
  266.     }
  267.     case GIM_CheckI64ImmPredicate:
  268.     case GIM_CheckImmOperandPredicate: {
  269.       int64_t InsnID = MatchTable[CurrentIdx++];
  270.       int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
  271.                           ? MatchTable[CurrentIdx++]
  272.                           : 1;
  273.       int64_t Predicate = MatchTable[CurrentIdx++];
  274.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  275.                       dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
  276.                              << InsnID << "]->getOperand(" << OpIdx
  277.                              << "), Predicate=" << Predicate << ")\n");
  278.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  279.       assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
  280.               State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
  281.              "Expected immediate operand");
  282.       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
  283.       int64_t Value = 0;
  284.       if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
  285.         Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
  286.       else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
  287.         Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
  288.       else
  289.         llvm_unreachable("Expected Imm or CImm operand");
  290.  
  291.       if (!testImmPredicate_I64(Predicate, Value))
  292.         if (handleReject() == RejectAndGiveUp)
  293.           return false;
  294.       break;
  295.     }
  296.     case GIM_CheckAPIntImmPredicate: {
  297.       int64_t InsnID = MatchTable[CurrentIdx++];
  298.       int64_t Predicate = MatchTable[CurrentIdx++];
  299.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  300.                       dbgs()
  301.                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
  302.                           << InsnID << "], Predicate=" << Predicate << ")\n");
  303.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  304.       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
  305.              "Expected G_CONSTANT");
  306.       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
  307.       APInt Value;
  308.       if (State.MIs[InsnID]->getOperand(1).isCImm())
  309.         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
  310.       else
  311.         llvm_unreachable("Expected Imm or CImm operand");
  312.  
  313.       if (!testImmPredicate_APInt(Predicate, Value))
  314.         if (handleReject() == RejectAndGiveUp)
  315.           return false;
  316.       break;
  317.     }
  318.     case GIM_CheckAPFloatImmPredicate: {
  319.       int64_t InsnID = MatchTable[CurrentIdx++];
  320.       int64_t Predicate = MatchTable[CurrentIdx++];
  321.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  322.                       dbgs()
  323.                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
  324.                           << InsnID << "], Predicate=" << Predicate << ")\n");
  325.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  326.       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
  327.              "Expected G_FCONSTANT");
  328.       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
  329.       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
  330.       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
  331.  
  332.       if (!testImmPredicate_APFloat(Predicate, Value))
  333.         if (handleReject() == RejectAndGiveUp)
  334.           return false;
  335.       break;
  336.     }
  337.     case GIM_CheckIsBuildVectorAllOnes:
  338.     case GIM_CheckIsBuildVectorAllZeros: {
  339.       int64_t InsnID = MatchTable[CurrentIdx++];
  340.  
  341.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  342.                       dbgs() << CurrentIdx
  343.                              << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
  344.                              << InsnID << "])\n");
  345.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  346.  
  347.       const MachineInstr *MI = State.MIs[InsnID];
  348.       assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
  349.               MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
  350.              "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
  351.  
  352.       if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
  353.         if (!isBuildVectorAllOnes(*MI, MRI)) {
  354.           if (handleReject() == RejectAndGiveUp)
  355.             return false;
  356.         }
  357.       } else {
  358.         if (!isBuildVectorAllZeros(*MI, MRI)) {
  359.           if (handleReject() == RejectAndGiveUp)
  360.             return false;
  361.         }
  362.       }
  363.  
  364.       break;
  365.     }
  366.     case GIM_CheckCxxInsnPredicate: {
  367.       int64_t InsnID = MatchTable[CurrentIdx++];
  368.       int64_t Predicate = MatchTable[CurrentIdx++];
  369.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  370.                       dbgs()
  371.                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
  372.                           << InsnID << "], Predicate=" << Predicate << ")\n");
  373.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  374.       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
  375.  
  376.       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
  377.                               State.RecordedOperands))
  378.         if (handleReject() == RejectAndGiveUp)
  379.           return false;
  380.       break;
  381.     }
  382.     case GIM_CheckHasNoUse: {
  383.       int64_t InsnID = MatchTable[CurrentIdx++];
  384.  
  385.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  386.                       dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
  387.                              << InsnID << "]\n");
  388.  
  389.       const MachineInstr *MI = State.MIs[InsnID];
  390.       assert(MI && "Used insn before defined");
  391.       assert(MI->getNumDefs() > 0 && "No defs");
  392.       const Register Res = MI->getOperand(0).getReg();
  393.  
  394.       if (!MRI.use_nodbg_empty(Res)) {
  395.         if (handleReject() == RejectAndGiveUp)
  396.           return false;
  397.       }
  398.  
  399.       break;
  400.     }
  401.     case GIM_CheckAtomicOrdering: {
  402.       int64_t InsnID = MatchTable[CurrentIdx++];
  403.       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
  404.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  405.                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
  406.                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
  407.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  408.       if (!State.MIs[InsnID]->hasOneMemOperand())
  409.         if (handleReject() == RejectAndGiveUp)
  410.           return false;
  411.  
  412.       for (const auto &MMO : State.MIs[InsnID]->memoperands())
  413.         if (MMO->getMergedOrdering() != Ordering)
  414.           if (handleReject() == RejectAndGiveUp)
  415.             return false;
  416.       break;
  417.     }
  418.     case GIM_CheckAtomicOrderingOrStrongerThan: {
  419.       int64_t InsnID = MatchTable[CurrentIdx++];
  420.       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
  421.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  422.                       dbgs() << CurrentIdx
  423.                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
  424.                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
  425.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  426.       if (!State.MIs[InsnID]->hasOneMemOperand())
  427.         if (handleReject() == RejectAndGiveUp)
  428.           return false;
  429.  
  430.       for (const auto &MMO : State.MIs[InsnID]->memoperands())
  431.         if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
  432.           if (handleReject() == RejectAndGiveUp)
  433.             return false;
  434.       break;
  435.     }
  436.     case GIM_CheckAtomicOrderingWeakerThan: {
  437.       int64_t InsnID = MatchTable[CurrentIdx++];
  438.       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
  439.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  440.                       dbgs() << CurrentIdx
  441.                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
  442.                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
  443.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  444.       if (!State.MIs[InsnID]->hasOneMemOperand())
  445.         if (handleReject() == RejectAndGiveUp)
  446.           return false;
  447.  
  448.       for (const auto &MMO : State.MIs[InsnID]->memoperands())
  449.         if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
  450.           if (handleReject() == RejectAndGiveUp)
  451.             return false;
  452.       break;
  453.     }
  454.     case GIM_CheckMemoryAddressSpace: {
  455.       int64_t InsnID = MatchTable[CurrentIdx++];
  456.       int64_t MMOIdx = MatchTable[CurrentIdx++];
  457.       // This accepts a list of possible address spaces.
  458.       const int NumAddrSpace = MatchTable[CurrentIdx++];
  459.  
  460.       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
  461.         if (handleReject() == RejectAndGiveUp)
  462.           return false;
  463.         break;
  464.       }
  465.  
  466.       // Need to still jump to the end of the list of address spaces if we find
  467.       // a match earlier.
  468.       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
  469.  
  470.       const MachineMemOperand *MMO
  471.         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
  472.       const unsigned MMOAddrSpace = MMO->getAddrSpace();
  473.  
  474.       bool Success = false;
  475.       for (int I = 0; I != NumAddrSpace; ++I) {
  476.         unsigned AddrSpace = MatchTable[CurrentIdx++];
  477.         DEBUG_WITH_TYPE(
  478.           TgtInstructionSelector::getName(),
  479.           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
  480.                  << AddrSpace << '\n');
  481.  
  482.         if (AddrSpace == MMOAddrSpace) {
  483.           Success = true;
  484.           break;
  485.         }
  486.       }
  487.  
  488.       CurrentIdx = LastIdx;
  489.       if (!Success && handleReject() == RejectAndGiveUp)
  490.         return false;
  491.       break;
  492.     }
  493.     case GIM_CheckMemoryAlignment: {
  494.       int64_t InsnID = MatchTable[CurrentIdx++];
  495.       int64_t MMOIdx = MatchTable[CurrentIdx++];
  496.       unsigned MinAlign = MatchTable[CurrentIdx++];
  497.  
  498.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  499.  
  500.       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
  501.         if (handleReject() == RejectAndGiveUp)
  502.           return false;
  503.         break;
  504.       }
  505.  
  506.       MachineMemOperand *MMO
  507.         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
  508.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  509.                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
  510.                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
  511.                       << ")->getAlignment() >= " << MinAlign << ")\n");
  512.       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
  513.         return false;
  514.  
  515.       break;
  516.     }
  517.     case GIM_CheckMemorySizeEqualTo: {
  518.       int64_t InsnID = MatchTable[CurrentIdx++];
  519.       int64_t MMOIdx = MatchTable[CurrentIdx++];
  520.       uint64_t Size = MatchTable[CurrentIdx++];
  521.  
  522.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  523.                       dbgs() << CurrentIdx
  524.                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
  525.                              << "]->memoperands() + " << MMOIdx
  526.                              << ", Size=" << Size << ")\n");
  527.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  528.  
  529.       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
  530.         if (handleReject() == RejectAndGiveUp)
  531.           return false;
  532.         break;
  533.       }
  534.  
  535.       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
  536.  
  537.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  538.                       dbgs() << MMO->getSize() << " bytes vs " << Size
  539.                              << " bytes\n");
  540.       if (MMO->getSize() != Size)
  541.         if (handleReject() == RejectAndGiveUp)
  542.           return false;
  543.  
  544.       break;
  545.     }
  546.     case GIM_CheckMemorySizeEqualToLLT:
  547.     case GIM_CheckMemorySizeLessThanLLT:
  548.     case GIM_CheckMemorySizeGreaterThanLLT: {
  549.       int64_t InsnID = MatchTable[CurrentIdx++];
  550.       int64_t MMOIdx = MatchTable[CurrentIdx++];
  551.       int64_t OpIdx = MatchTable[CurrentIdx++];
  552.  
  553.       DEBUG_WITH_TYPE(
  554.           TgtInstructionSelector::getName(),
  555.           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
  556.                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
  557.                          ? "EqualTo"
  558.                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
  559.                                ? "GreaterThan"
  560.                                : "LessThan")
  561.                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
  562.                  << ", OpIdx=" << OpIdx << ")\n");
  563.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  564.  
  565.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  566.       if (!MO.isReg()) {
  567.         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  568.                         dbgs() << CurrentIdx << ": Not a register\n");
  569.         if (handleReject() == RejectAndGiveUp)
  570.           return false;
  571.         break;
  572.       }
  573.  
  574.       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
  575.         if (handleReject() == RejectAndGiveUp)
  576.           return false;
  577.         break;
  578.       }
  579.  
  580.       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
  581.  
  582.       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
  583.       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
  584.           MMO->getSizeInBits() != Size) {
  585.         if (handleReject() == RejectAndGiveUp)
  586.           return false;
  587.       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
  588.                  MMO->getSizeInBits() >= Size) {
  589.         if (handleReject() == RejectAndGiveUp)
  590.           return false;
  591.       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
  592.                  MMO->getSizeInBits() <= Size)
  593.         if (handleReject() == RejectAndGiveUp)
  594.           return false;
  595.  
  596.       break;
  597.     }
  598.     case GIM_CheckType: {
  599.       int64_t InsnID = MatchTable[CurrentIdx++];
  600.       int64_t OpIdx = MatchTable[CurrentIdx++];
  601.       int64_t TypeID = MatchTable[CurrentIdx++];
  602.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  603.                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
  604.                              << "]->getOperand(" << OpIdx
  605.                              << "), TypeID=" << TypeID << ")\n");
  606.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  607.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  608.       if (!MO.isReg() ||
  609.           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
  610.         if (handleReject() == RejectAndGiveUp)
  611.           return false;
  612.       }
  613.       break;
  614.     }
  615.     case GIM_CheckPointerToAny: {
  616.       int64_t InsnID = MatchTable[CurrentIdx++];
  617.       int64_t OpIdx = MatchTable[CurrentIdx++];
  618.       uint64_t SizeInBits = MatchTable[CurrentIdx++];
  619.  
  620.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  621.                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
  622.                              << InsnID << "]->getOperand(" << OpIdx
  623.                              << "), SizeInBits=" << SizeInBits << ")\n");
  624.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  625.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  626.       const LLT Ty = MRI.getType(MO.getReg());
  627.  
  628.       // iPTR must be looked up in the target.
  629.       if (SizeInBits == 0) {
  630.         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
  631.         const unsigned AddrSpace = Ty.getAddressSpace();
  632.         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
  633.       }
  634.  
  635.       assert(SizeInBits != 0 && "Pointer size must be known");
  636.  
  637.       if (MO.isReg()) {
  638.         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
  639.           if (handleReject() == RejectAndGiveUp)
  640.             return false;
  641.       } else if (handleReject() == RejectAndGiveUp)
  642.         return false;
  643.  
  644.       break;
  645.     }
  646.     case GIM_RecordNamedOperand: {
  647.       int64_t InsnID = MatchTable[CurrentIdx++];
  648.       int64_t OpIdx = MatchTable[CurrentIdx++];
  649.       uint64_t StoreIdx = MatchTable[CurrentIdx++];
  650.  
  651.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  652.                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
  653.                              << InsnID << "]->getOperand(" << OpIdx
  654.                              << "), StoreIdx=" << StoreIdx << ")\n");
  655.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  656.       assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
  657.       State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
  658.       break;
  659.     }
  660.     case GIM_CheckRegBankForClass: {
  661.       int64_t InsnID = MatchTable[CurrentIdx++];
  662.       int64_t OpIdx = MatchTable[CurrentIdx++];
  663.       int64_t RCEnum = MatchTable[CurrentIdx++];
  664.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  665.                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
  666.                              << InsnID << "]->getOperand(" << OpIdx
  667.                              << "), RCEnum=" << RCEnum << ")\n");
  668.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  669.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  670.       if (!MO.isReg() ||
  671.           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
  672.                                       MRI.getType(MO.getReg())) !=
  673.               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
  674.         if (handleReject() == RejectAndGiveUp)
  675.           return false;
  676.       }
  677.       break;
  678.     }
  679.  
  680.     case GIM_CheckComplexPattern: {
  681.       int64_t InsnID = MatchTable[CurrentIdx++];
  682.       int64_t OpIdx = MatchTable[CurrentIdx++];
  683.       int64_t RendererID = MatchTable[CurrentIdx++];
  684.       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
  685.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  686.                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
  687.                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
  688.                              << "]->getOperand(" << OpIdx
  689.                              << "), ComplexPredicateID=" << ComplexPredicateID
  690.                              << ")\n");
  691.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  692.       // FIXME: Use std::invoke() when it's available.
  693.       ComplexRendererFns Renderer =
  694.           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
  695.               State.MIs[InsnID]->getOperand(OpIdx));
  696.       if (Renderer)
  697.         State.Renderers[RendererID] = *Renderer;
  698.       else
  699.         if (handleReject() == RejectAndGiveUp)
  700.           return false;
  701.       break;
  702.     }
  703.  
  704.     case GIM_CheckConstantInt: {
  705.       int64_t InsnID = MatchTable[CurrentIdx++];
  706.       int64_t OpIdx = MatchTable[CurrentIdx++];
  707.       int64_t Value = MatchTable[CurrentIdx++];
  708.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  709.                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
  710.                              << InsnID << "]->getOperand(" << OpIdx
  711.                              << "), Value=" << Value << ")\n");
  712.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  713.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  714.       if (MO.isReg()) {
  715.         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
  716.         LLT Ty = MRI.getType(MO.getReg());
  717.         Value = SignExtend64(Value, Ty.getSizeInBits());
  718.  
  719.         if (!isOperandImmEqual(MO, Value, MRI)) {
  720.           if (handleReject() == RejectAndGiveUp)
  721.             return false;
  722.         }
  723.       } else if (handleReject() == RejectAndGiveUp)
  724.         return false;
  725.  
  726.       break;
  727.     }
  728.  
  729.     case GIM_CheckLiteralInt: {
  730.       int64_t InsnID = MatchTable[CurrentIdx++];
  731.       int64_t OpIdx = MatchTable[CurrentIdx++];
  732.       int64_t Value = MatchTable[CurrentIdx++];
  733.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  734.                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
  735.                              << InsnID << "]->getOperand(" << OpIdx
  736.                              << "), Value=" << Value << ")\n");
  737.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  738.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  739.       if (MO.isImm() && MO.getImm() == Value)
  740.         break;
  741.  
  742.       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
  743.         break;
  744.  
  745.       if (handleReject() == RejectAndGiveUp)
  746.         return false;
  747.  
  748.       break;
  749.     }
  750.  
  751.     case GIM_CheckIntrinsicID: {
  752.       int64_t InsnID = MatchTable[CurrentIdx++];
  753.       int64_t OpIdx = MatchTable[CurrentIdx++];
  754.       int64_t Value = MatchTable[CurrentIdx++];
  755.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  756.                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
  757.                              << InsnID << "]->getOperand(" << OpIdx
  758.                              << "), Value=" << Value << ")\n");
  759.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  760.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  761.       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
  762.         if (handleReject() == RejectAndGiveUp)
  763.           return false;
  764.       break;
  765.     }
  766.     case GIM_CheckCmpPredicate: {
  767.       int64_t InsnID = MatchTable[CurrentIdx++];
  768.       int64_t OpIdx = MatchTable[CurrentIdx++];
  769.       int64_t Value = MatchTable[CurrentIdx++];
  770.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  771.                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
  772.                              << InsnID << "]->getOperand(" << OpIdx
  773.                              << "), Value=" << Value << ")\n");
  774.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  775.       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  776.       if (!MO.isPredicate() || MO.getPredicate() != Value)
  777.         if (handleReject() == RejectAndGiveUp)
  778.           return false;
  779.       break;
  780.     }
  781.     case GIM_CheckIsMBB: {
  782.       int64_t InsnID = MatchTable[CurrentIdx++];
  783.       int64_t OpIdx = MatchTable[CurrentIdx++];
  784.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  785.                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
  786.                              << "]->getOperand(" << OpIdx << "))\n");
  787.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  788.       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
  789.         if (handleReject() == RejectAndGiveUp)
  790.           return false;
  791.       }
  792.       break;
  793.     }
  794.     case GIM_CheckIsImm: {
  795.       int64_t InsnID = MatchTable[CurrentIdx++];
  796.       int64_t OpIdx = MatchTable[CurrentIdx++];
  797.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  798.                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
  799.                              << "]->getOperand(" << OpIdx << "))\n");
  800.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  801.       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
  802.         if (handleReject() == RejectAndGiveUp)
  803.           return false;
  804.       }
  805.       break;
  806.     }
  807.     case GIM_CheckIsSafeToFold: {
  808.       int64_t InsnID = MatchTable[CurrentIdx++];
  809.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  810.                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
  811.                              << InsnID << "])\n");
  812.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  813.       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
  814.         if (handleReject() == RejectAndGiveUp)
  815.           return false;
  816.       }
  817.       break;
  818.     }
  819.     case GIM_CheckIsSameOperand: {
  820.       int64_t InsnID = MatchTable[CurrentIdx++];
  821.       int64_t OpIdx = MatchTable[CurrentIdx++];
  822.       int64_t OtherInsnID = MatchTable[CurrentIdx++];
  823.       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
  824.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  825.                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
  826.                              << InsnID << "][" << OpIdx << "], MIs["
  827.                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
  828.       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
  829.       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
  830.       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
  831.               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
  832.         if (handleReject() == RejectAndGiveUp)
  833.           return false;
  834.       }
  835.       break;
  836.     }
  837.     case GIM_Reject:
  838.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  839.                       dbgs() << CurrentIdx << ": GIM_Reject\n");
  840.       if (handleReject() == RejectAndGiveUp)
  841.         return false;
  842.       break;
  843.  
  844.     case GIR_MutateOpcode: {
  845.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  846.       uint64_t NewInsnID = MatchTable[CurrentIdx++];
  847.       int64_t NewOpcode = MatchTable[CurrentIdx++];
  848.       if (NewInsnID >= OutMIs.size())
  849.         OutMIs.resize(NewInsnID + 1);
  850.  
  851.       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
  852.                                               State.MIs[OldInsnID]);
  853.       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
  854.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  855.                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
  856.                              << NewInsnID << "], MIs[" << OldInsnID << "], "
  857.                              << NewOpcode << ")\n");
  858.       break;
  859.     }
  860.  
  861.     case GIR_BuildMI: {
  862.       uint64_t NewInsnID = MatchTable[CurrentIdx++];
  863.       int64_t Opcode = MatchTable[CurrentIdx++];
  864.       if (NewInsnID >= OutMIs.size())
  865.         OutMIs.resize(NewInsnID + 1);
  866.  
  867.       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
  868.                                   MIMetadata(*State.MIs[0]), TII.get(Opcode));
  869.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  870.                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
  871.                              << NewInsnID << "], " << Opcode << ")\n");
  872.       break;
  873.     }
  874.  
  875.     case GIR_Copy: {
  876.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  877.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  878.       int64_t OpIdx = MatchTable[CurrentIdx++];
  879.       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
  880.       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
  881.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  882.                       dbgs()
  883.                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
  884.                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
  885.       break;
  886.     }
  887.  
  888.     case GIR_CopyOrAddZeroReg: {
  889.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  890.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  891.       int64_t OpIdx = MatchTable[CurrentIdx++];
  892.       int64_t ZeroReg = MatchTable[CurrentIdx++];
  893.       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
  894.       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
  895.       if (isOperandImmEqual(MO, 0, MRI))
  896.         OutMIs[NewInsnID].addReg(ZeroReg);
  897.       else
  898.         OutMIs[NewInsnID].add(MO);
  899.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  900.                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
  901.                              << NewInsnID << "], MIs[" << OldInsnID << "], "
  902.                              << OpIdx << ", " << ZeroReg << ")\n");
  903.       break;
  904.     }
  905.  
  906.     case GIR_CopySubReg: {
  907.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  908.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  909.       int64_t OpIdx = MatchTable[CurrentIdx++];
  910.       int64_t SubRegIdx = MatchTable[CurrentIdx++];
  911.       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
  912.       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
  913.                                0, SubRegIdx);
  914.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  915.                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
  916.                              << NewInsnID << "], MIs[" << OldInsnID << "], "
  917.                              << OpIdx << ", " << SubRegIdx << ")\n");
  918.       break;
  919.     }
  920.  
  921.     case GIR_AddImplicitDef: {
  922.       int64_t InsnID = MatchTable[CurrentIdx++];
  923.       int64_t RegNum = MatchTable[CurrentIdx++];
  924.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  925.       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
  926.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  927.                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
  928.                              << InsnID << "], " << RegNum << ")\n");
  929.       break;
  930.     }
  931.  
  932.     case GIR_AddImplicitUse: {
  933.       int64_t InsnID = MatchTable[CurrentIdx++];
  934.       int64_t RegNum = MatchTable[CurrentIdx++];
  935.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  936.       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
  937.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  938.                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
  939.                              << InsnID << "], " << RegNum << ")\n");
  940.       break;
  941.     }
  942.  
  943.     case GIR_AddRegister: {
  944.       int64_t InsnID = MatchTable[CurrentIdx++];
  945.       int64_t RegNum = MatchTable[CurrentIdx++];
  946.       uint64_t RegFlags = MatchTable[CurrentIdx++];
  947.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  948.       OutMIs[InsnID].addReg(RegNum, RegFlags);
  949.       DEBUG_WITH_TYPE(
  950.         TgtInstructionSelector::getName(),
  951.         dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
  952.         << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
  953.       break;
  954.     }
  955.  
  956.     case GIR_AddTempRegister:
  957.     case GIR_AddTempSubRegister: {
  958.       int64_t InsnID = MatchTable[CurrentIdx++];
  959.       int64_t TempRegID = MatchTable[CurrentIdx++];
  960.       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
  961.       unsigned SubReg = 0;
  962.       if (MatcherOpcode == GIR_AddTempSubRegister)
  963.         SubReg = MatchTable[CurrentIdx++];
  964.  
  965.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  966.  
  967.       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
  968.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  969.                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
  970.                              << InsnID << "], TempRegisters[" << TempRegID
  971.                              << "]";
  972.                       if (SubReg)
  973.                         dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
  974.                       dbgs() << ", " << TempRegFlags << ")\n");
  975.       break;
  976.     }
  977.  
  978.     case GIR_AddImm: {
  979.       int64_t InsnID = MatchTable[CurrentIdx++];
  980.       int64_t Imm = MatchTable[CurrentIdx++];
  981.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  982.       OutMIs[InsnID].addImm(Imm);
  983.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  984.                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
  985.                              << "], " << Imm << ")\n");
  986.       break;
  987.     }
  988.  
  989.     case GIR_ComplexRenderer: {
  990.       int64_t InsnID = MatchTable[CurrentIdx++];
  991.       int64_t RendererID = MatchTable[CurrentIdx++];
  992.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  993.       for (const auto &RenderOpFn : State.Renderers[RendererID])
  994.         RenderOpFn(OutMIs[InsnID]);
  995.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  996.                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
  997.                              << InsnID << "], " << RendererID << ")\n");
  998.       break;
  999.     }
  1000.     case GIR_ComplexSubOperandRenderer: {
  1001.       int64_t InsnID = MatchTable[CurrentIdx++];
  1002.       int64_t RendererID = MatchTable[CurrentIdx++];
  1003.       int64_t RenderOpID = MatchTable[CurrentIdx++];
  1004.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1005.       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
  1006.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1007.                       dbgs() << CurrentIdx
  1008.                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
  1009.                              << InsnID << "], " << RendererID << ", "
  1010.                              << RenderOpID << ")\n");
  1011.       break;
  1012.     }
  1013.  
  1014.     case GIR_CopyConstantAsSImm: {
  1015.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  1016.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  1017.       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
  1018.       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
  1019.       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
  1020.         OutMIs[NewInsnID].addImm(
  1021.             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
  1022.       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
  1023.         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
  1024.       else
  1025.         llvm_unreachable("Expected Imm or CImm operand");
  1026.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1027.                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
  1028.                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
  1029.       break;
  1030.     }
  1031.  
  1032.     // TODO: Needs a test case once we have a pattern that uses this.
  1033.     case GIR_CopyFConstantAsFPImm: {
  1034.       int64_t NewInsnID = MatchTable[CurrentIdx++];
  1035.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  1036.       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
  1037.       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
  1038.       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
  1039.         OutMIs[NewInsnID].addFPImm(
  1040.             State.MIs[OldInsnID]->getOperand(1).getFPImm());
  1041.       else
  1042.         llvm_unreachable("Expected FPImm operand");
  1043.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1044.                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
  1045.                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
  1046.       break;
  1047.     }
  1048.  
  1049.     case GIR_CustomRenderer: {
  1050.       int64_t InsnID = MatchTable[CurrentIdx++];
  1051.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  1052.       int64_t RendererFnID = MatchTable[CurrentIdx++];
  1053.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1054.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1055.                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
  1056.                              << InsnID << "], MIs[" << OldInsnID << "], "
  1057.                              << RendererFnID << ")\n");
  1058.       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
  1059.         OutMIs[InsnID], *State.MIs[OldInsnID],
  1060.         -1); // Not a source operand of the old instruction.
  1061.       break;
  1062.     }
  1063.     case GIR_CustomOperandRenderer: {
  1064.       int64_t InsnID = MatchTable[CurrentIdx++];
  1065.       int64_t OldInsnID = MatchTable[CurrentIdx++];
  1066.       int64_t OpIdx = MatchTable[CurrentIdx++];
  1067.       int64_t RendererFnID = MatchTable[CurrentIdx++];
  1068.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1069.  
  1070.       DEBUG_WITH_TYPE(
  1071.         TgtInstructionSelector::getName(),
  1072.         dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
  1073.                << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
  1074.                << OpIdx << "), "
  1075.         << RendererFnID << ")\n");
  1076.       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
  1077.                                                      *State.MIs[OldInsnID],
  1078.                                                      OpIdx);
  1079.       break;
  1080.     }
  1081.     case GIR_ConstrainOperandRC: {
  1082.       int64_t InsnID = MatchTable[CurrentIdx++];
  1083.       int64_t OpIdx = MatchTable[CurrentIdx++];
  1084.       int64_t RCEnum = MatchTable[CurrentIdx++];
  1085.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1086.       MachineInstr &I = *OutMIs[InsnID].getInstr();
  1087.       MachineFunction &MF = *I.getParent()->getParent();
  1088.       MachineRegisterInfo &MRI = MF.getRegInfo();
  1089.       const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
  1090.       MachineOperand &MO = I.getOperand(OpIdx);
  1091.       constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
  1092.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1093.                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
  1094.                              << InsnID << "], " << OpIdx << ", " << RCEnum
  1095.                              << ")\n");
  1096.       break;
  1097.     }
  1098.  
  1099.     case GIR_ConstrainSelectedInstOperands: {
  1100.       int64_t InsnID = MatchTable[CurrentIdx++];
  1101.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1102.       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
  1103.                                        RBI);
  1104.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1105.                       dbgs() << CurrentIdx
  1106.                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
  1107.                              << InsnID << "])\n");
  1108.       break;
  1109.     }
  1110.  
  1111.     case GIR_MergeMemOperands: {
  1112.       int64_t InsnID = MatchTable[CurrentIdx++];
  1113.       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
  1114.  
  1115.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1116.                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
  1117.                              << InsnID << "]");
  1118.       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
  1119.       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
  1120.              GIU_MergeMemOperands_EndOfList) {
  1121.         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1122.                         dbgs() << ", MIs[" << MergeInsnID << "]");
  1123.         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
  1124.           OutMIs[InsnID].addMemOperand(MMO);
  1125.       }
  1126.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
  1127.       break;
  1128.     }
  1129.  
  1130.     case GIR_EraseFromParent: {
  1131.       int64_t InsnID = MatchTable[CurrentIdx++];
  1132.       assert(State.MIs[InsnID] &&
  1133.              "Attempted to erase an undefined instruction");
  1134.       State.MIs[InsnID]->eraseFromParent();
  1135.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1136.                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
  1137.                              << InsnID << "])\n");
  1138.       break;
  1139.     }
  1140.  
  1141.     case GIR_MakeTempReg: {
  1142.       int64_t TempRegID = MatchTable[CurrentIdx++];
  1143.       int64_t TypeID = MatchTable[CurrentIdx++];
  1144.  
  1145.       State.TempRegisters[TempRegID] =
  1146.           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
  1147.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1148.                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
  1149.                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
  1150.       break;
  1151.     }
  1152.  
  1153.     case GIR_Coverage: {
  1154.       int64_t RuleID = MatchTable[CurrentIdx++];
  1155.       CoverageInfo.setCovered(RuleID);
  1156.  
  1157.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1158.                       dbgs()
  1159.                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
  1160.       break;
  1161.     }
  1162.  
  1163.     case GIR_Done:
  1164.       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  1165.                       dbgs() << CurrentIdx << ": GIR_Done\n");
  1166.       propagateFlags(OutMIs);
  1167.       return true;
  1168.  
  1169.     default:
  1170.       llvm_unreachable("Unexpected command");
  1171.     }
  1172.   }
  1173. }
  1174.  
  1175. } // end namespace llvm
  1176.  
  1177. #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  1178.