Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

/// \file A pass to convert the target-illegal operations created by IR -> MIR

/// translation into ones the target expects to be able to select. This may

/// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> ->

/// G_ADD <4 x i16>.

///

/// The LegalizerHelper class is where most of the work happens, and is

/// designed to be callable from other passes that find themselves with an

/// illegal instruction.

//

//===----------------------------------------------------------------------===//

#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H

#define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H

#include "llvm/CodeGen/GlobalISel/CallLowering.h"

#include "llvm/CodeGen/RuntimeLibcalls.h"

#include "llvm/CodeGen/TargetOpcodes.h"

namespace llvm {

// Forward declarations.

class APInt;

class GAnyLoad;

class GLoadStore;

class GStore;

class GenericMachineInstr;

class MachineFunction;

class MachineIRBuilder;

class MachineInstr;

class MachineInstrBuilder;

struct MachinePointerInfo;

template <typename T> class SmallVectorImpl;

class LegalizerInfo;

class MachineRegisterInfo;

class GISelChangeObserver;

class LostDebugLocObserver;

class TargetLowering;

class LegalizerHelper {

public:

  /// Expose MIRBuilder so clients can set their own RecordInsertInstruction

  /// functions

  MachineIRBuilder &MIRBuilder;

  /// To keep track of changes made by the LegalizerHelper.

  GISelChangeObserver &Observer;

private:

  MachineRegisterInfo &MRI;

  const LegalizerInfo &LI;

  const TargetLowering &TLI;

public:

  enum LegalizeResult {

    /// Instruction was already legal and no change was made to the

    /// MachineFunction.

    AlreadyLegal,

    /// Instruction has been legalized and the MachineFunction changed.

    Legalized,

    /// Some kind of error has occurred and we could not legalize this

    /// instruction.

    UnableToLegalize,

  };

  /// Expose LegalizerInfo so the clients can re-use.

  const LegalizerInfo &getLegalizerInfo() const { return LI; }

  const TargetLowering &getTargetLowering() const { return TLI; }

  LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer,

                  MachineIRBuilder &B);

  LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,

                  GISelChangeObserver &Observer, MachineIRBuilder &B);

  /// Replace \p MI by a sequence of legal instructions that can implement the

  /// same operation. Note that this means \p MI may be deleted, so any iterator

  /// steps should be performed before calling this function. \p Helper should

  /// be initialized to the MachineFunction containing \p MI.

  ///

  /// Considered as an opaque blob, the legal code will use and define the same

  /// registers as \p MI.

  LegalizeResult legalizeInstrStep(MachineInstr &MI,

                                   LostDebugLocObserver &LocObserver);

  /// Legalize an instruction by emiting a runtime library call instead.

  LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver);

  /// Legalize an instruction by reducing the width of the underlying scalar

  /// type.

  LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);

  /// Legalize an instruction by performing the operation on a wider scalar type

  /// (for example a 16-bit addition can be safely performed at 32-bits

  /// precision, ignoring the unused bits).

  LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);

  /// Legalize an instruction by replacing the value type

  LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  /// Legalize an instruction by splitting it into simpler parts, hopefully

  /// understood by the target.

  LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  /// Legalize a vector instruction by splitting into multiple components, each

  /// acting on the same scalar type as the original but with fewer elements.

  LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,

                                     LLT NarrowTy);

  /// Legalize a vector instruction by increasing the number of vector elements

  /// involved and ignoring the added elements later.

  LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx,

                                    LLT MoreTy);

  /// Cast the given value to an LLT::scalar with an equivalent size. Returns

  /// the register to use if an instruction was inserted. Returns the original

  /// register if no coercion was necessary.

  //

  // This may also fail and return Register() if there is no legal way to cast.

  Register coerceToScalar(Register Val);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// Use by extending the operand's type to \p WideTy using the specified \p

  /// ExtOpcode for the extension instruction, and replacing the vreg of the

  /// operand in place.

  void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx,

                      unsigned ExtOpcode);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and

  /// replacing the vreg of the operand in place.

  void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// Def by extending the operand's type to \p WideTy and truncating it back

  /// with the \p TruncOpcode, and replacing the vreg of the operand in place.

  void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0,

                      unsigned TruncOpcode = TargetOpcode::G_TRUNC);

  // Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  // Def by truncating the operand's type to \p NarrowTy, replacing in place and

  // extending back with \p ExtOpcode.

  void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx,

                       unsigned ExtOpcode);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// Def by performing it with additional vector elements and extracting the

  /// result elements, and replacing the vreg of the operand in place.

  void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// Use by producing a vector with undefined high elements, extracting the

  /// original vector type, and replacing the vreg of the operand in place.

  void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// use by inserting a G_BITCAST to \p CastTy

  void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx);

  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a

  /// def by inserting a G_BITCAST from \p CastTy

  void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx);

private:

  LegalizeResult

  widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);

  LegalizeResult

  widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);

  LegalizeResult

  widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);

  LegalizeResult

  widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);

  LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,

                                           LLT WideTy);

  LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,

                                         LLT WideTy);

  LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,

                                 LLT WideTy);

  /// Helper function to split a wide generic register into bitwise blocks with

  /// the given Type (which implies the number of blocks needed). The generic

  /// registers created are appended to Ops, starting at bit 0 of Reg.

  void extractParts(Register Reg, LLT Ty, int NumParts,

                    SmallVectorImpl<Register> &VRegs);

  /// Version which handles irregular splits.

  bool extractParts(Register Reg, LLT RegTy, LLT MainTy,

                    LLT &LeftoverTy,

                    SmallVectorImpl<Register> &VRegs,

                    SmallVectorImpl<Register> &LeftoverVRegs);

  /// Version which handles irregular sub-vector splits.

  void extractVectorParts(Register Reg, unsigned NumElst,

                          SmallVectorImpl<Register> &VRegs);

  /// Helper function to build a wide generic register \p DstReg of type \p

  /// RegTy from smaller parts. This will produce a G_MERGE_VALUES,

  /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate

  /// for the types.

  ///

  /// \p PartRegs must be registers of type \p PartTy.

  ///

  /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the

  /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy.

  void insertParts(Register DstReg, LLT ResultTy,

                   LLT PartTy, ArrayRef<Register> PartRegs,

                   LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {});

  /// Merge \p PartRegs with different types into \p DstReg.

  void mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs);

  void appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg);

  /// Unmerge \p SrcReg into smaller sized values, and append them to \p

  /// Parts. The elements of \p Parts will be the greatest common divisor type

  /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and

  /// return the GCD type.

  LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,

                     LLT NarrowTy, Register SrcReg);

  /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of

  /// the unpacked registers to \p Parts. This version is if the common unmerge

  /// type is already known.

  void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy,

                      Register SrcReg);

  /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge

  /// from the least common multiple type, and convert as appropriate to \p

  /// DstReg.

  ///

  /// \p VRegs should each have type \p GCDTy. This type should be greatest

  /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source

  /// type.

  ///

  /// \p NarrowTy is the desired result merge source type. If the source value

  /// needs to be widened to evenly cover \p DstReg, inserts high bits

  /// corresponding to the extension opcode \p PadStrategy.

  ///

  /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces

  /// will replace it. Returns The complete LCMTy that \p VRegs will cover when

  /// merged.

  LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,

                          SmallVectorImpl<Register> &VRegs,

                          unsigned PadStrategy = TargetOpcode::G_ANYEXT);

  /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the

  /// low bits into \p DstReg. This is intended to use the outputs from

  /// buildLCMMergePieces after processing.

  void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,

                                ArrayRef<Register> RemergeRegs);

  /// Perform generic multiplication of values held in multiple registers.

  /// Generated instructions use only types NarrowTy and i1.

  /// Destination can be same or two times size of the source.

  void multiplyRegisters(SmallVectorImpl<Register> &DstRegs,

                         ArrayRef<Register> Src1Regs,

                         ArrayRef<Register> Src2Regs, LLT NarrowTy);

  void changeOpcode(MachineInstr &MI, unsigned NewOpcode);

  LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,

                                        LLT SrcTy, LLT NarrowTy,

                                        unsigned ScalarOpc);

  // Memcpy family legalization helpers.

  LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val,

                             uint64_t KnownLen, Align Alignment,

                             bool IsVolatile);

  LegalizeResult lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src,

                                   uint64_t KnownLen, Align DstAlign,

                                   Align SrcAlign, bool IsVolatile);

  LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,

                             uint64_t KnownLen, uint64_t Limit, Align DstAlign,

                             Align SrcAlign, bool IsVolatile);

  LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src,

                              uint64_t KnownLen, Align DstAlign, Align SrcAlign,

                              bool IsVolatile);

public:

  /// Return the alignment to use for a stack temporary object with the given

  /// type.

  Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const;

  /// Create a stack temporary based on the size in bytes and the alignment

  MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment,

                                           MachinePointerInfo &PtrInfo);

  /// Get a pointer to vector element \p Index located in memory for a vector of

  /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out

  /// of bounds the returned pointer is unspecified, but will be within the

  /// vector bounds.

  Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index);

  /// Handles most opcodes. Split \p MI into same instruction on sub-vectors or

  /// scalars with \p NumElts elements (1 for scalar). Supports uneven splits:

  /// there can be leftover sub-vector with fewer then \p NumElts or a leftover

  /// scalar. To avoid this use moreElements first and set MI number of elements

  /// to multiple of \p NumElts. Non-vector operands that should be used on all

  /// sub-instructions without split are listed in \p NonVecOpIndices.

  LegalizeResult fewerElementsVectorMultiEltType(

      GenericMachineInstr &MI, unsigned NumElts,

      std::initializer_list<unsigned> NonVecOpIndices = {});

  LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI,

                                        unsigned NumElts);

  LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,

                                       LLT MoreTy);

  LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx,

                                           LLT MoreTy);

  LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI,

                                                  unsigned TypeIdx,

                                                  LLT NarrowTy);

  LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,

                                          LLT NarrowTy);

  LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,

                                                           unsigned TypeIdx,

                                                           LLT NarrowTy);

  LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx,

                                      LLT NarrowTy);

  LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,

                                             LLT HalfTy, LLT ShiftAmtTy);

  LegalizeResult fewerElementsVectorReductions(MachineInstr &MI,

                                               unsigned TypeIdx, LLT NarrowTy);

  LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx,

                                            LLT NarrowTy);

  LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,

                                    LLT NarrowTy);

  LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty);

  LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);

  /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT.

  LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,

                                         LLT CastTy);

  /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT.

  LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,

                                        LLT CastTy);

  LegalizeResult lowerBitcast(MachineInstr &MI);

  LegalizeResult lowerLoad(GAnyLoad &MI);

  LegalizeResult lowerStore(GStore &MI);

  LegalizeResult lowerBitCount(MachineInstr &MI);

  LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI);

  LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI);

  LegalizeResult lowerFunnelShift(MachineInstr &MI);

  LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI);

  LegalizeResult lowerRotate(MachineInstr &MI);

  LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);

  LegalizeResult lowerUITOFP(MachineInstr &MI);

  LegalizeResult lowerSITOFP(MachineInstr &MI);

  LegalizeResult lowerFPTOUI(MachineInstr &MI);

  LegalizeResult lowerFPTOSI(MachineInstr &MI);

  LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI);

  LegalizeResult lowerFPTRUNC(MachineInstr &MI);

  LegalizeResult lowerFPOWI(MachineInstr &MI);

  LegalizeResult lowerISFPCLASS(MachineInstr &MI);

  LegalizeResult lowerMinMax(MachineInstr &MI);

  LegalizeResult lowerFCopySign(MachineInstr &MI);

  LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI);

  LegalizeResult lowerFMad(MachineInstr &MI);

  LegalizeResult lowerIntrinsicRound(MachineInstr &MI);

  LegalizeResult lowerFFloor(MachineInstr &MI);

  LegalizeResult lowerMergeValues(MachineInstr &MI);

  LegalizeResult lowerUnmergeValues(MachineInstr &MI);

  LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI);

  LegalizeResult lowerShuffleVector(MachineInstr &MI);

  LegalizeResult lowerDynStackAlloc(MachineInstr &MI);

  LegalizeResult lowerExtract(MachineInstr &MI);

  LegalizeResult lowerInsert(MachineInstr &MI);

  LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI);

  LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI);

  LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI);

  LegalizeResult lowerShlSat(MachineInstr &MI);

  LegalizeResult lowerBswap(MachineInstr &MI);

  LegalizeResult lowerBitreverse(MachineInstr &MI);

  LegalizeResult lowerReadWriteRegister(MachineInstr &MI);

  LegalizeResult lowerSMULH_UMULH(MachineInstr &MI);

  LegalizeResult lowerSelect(MachineInstr &MI);

  LegalizeResult lowerDIVREM(MachineInstr &MI);

  LegalizeResult lowerAbsToAddXor(MachineInstr &MI);

  LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI);

  LegalizeResult lowerVectorReduction(MachineInstr &MI);

  LegalizeResult lowerMemcpyInline(MachineInstr &MI);

  LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0);

};

/// Helper function that creates a libcall to the given \p Name using the given

/// calling convention \p CC.

LegalizerHelper::LegalizeResult

createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,

              const CallLowering::ArgInfo &Result,

              ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);

/// Helper function that creates the given libcall.

LegalizerHelper::LegalizeResult

createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,

              const CallLowering::ArgInfo &Result,

              ArrayRef<CallLowering::ArgInfo> Args);

/// Create a libcall to memcpy et al.

LegalizerHelper::LegalizeResult

createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,

                 MachineInstr &MI, LostDebugLocObserver &LocObserver);

} // End namespace llvm.

#endif