WebSVN – QNX 8.QNX8 LLVM/Clang compiler suite – Blame – //llvm-build/x86_64/include/llvm/CodeGen/SelectionDAGNodes.h

Rev	Author	Line No.	Line
14	pmbaty	1	//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----- C++ --===//
		2	//
		3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
		4	// See https://llvm.org/LICENSE.txt for license information.
		5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
		6	//
		7	//===----------------------------------------------------------------------===//
		8	//
		9	// This file declares the SDNode class and derived classes, which are used to
		10	// represent the nodes and operations present in a SelectionDAG. These nodes
		11	// and operations are machine code level operations, with some similarities to
		12	// the GCC RTL representation.
		13	//
		14	// Clients should include the SelectionDAG.h file instead of this file directly.
		15	//
		16	//===----------------------------------------------------------------------===//
		17
		18	#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
		19	#define LLVM_CODEGEN_SELECTIONDAGNODES_H
		20
		21	#include "llvm/ADT/APFloat.h"
		22	#include "llvm/ADT/ArrayRef.h"
		23	#include "llvm/ADT/BitVector.h"
		24	#include "llvm/ADT/FoldingSet.h"
		25	#include "llvm/ADT/GraphTraits.h"
		26	#include "llvm/ADT/SmallPtrSet.h"
		27	#include "llvm/ADT/SmallVector.h"
		28	#include "llvm/ADT/ilist_node.h"
		29	#include "llvm/ADT/iterator.h"
		30	#include "llvm/ADT/iterator_range.h"
		31	#include "llvm/CodeGen/ISDOpcodes.h"
		32	#include "llvm/CodeGen/MachineMemOperand.h"
		33	#include "llvm/CodeGen/Register.h"
		34	#include "llvm/CodeGen/ValueTypes.h"
		35	#include "llvm/IR/Constants.h"
		36	#include "llvm/IR/DebugLoc.h"
		37	#include "llvm/IR/Instruction.h"
		38	#include "llvm/IR/Instructions.h"
		39	#include "llvm/IR/Metadata.h"
		40	#include "llvm/IR/Operator.h"
		41	#include "llvm/Support/AlignOf.h"
		42	#include "llvm/Support/AtomicOrdering.h"
		43	#include "llvm/Support/Casting.h"
		44	#include "llvm/Support/ErrorHandling.h"
		45	#include "llvm/Support/MachineValueType.h"
		46	#include "llvm/Support/TypeSize.h"
		47	#include <algorithm>
		48	#include <cassert>
		49	#include <climits>
		50	#include <cstddef>
		51	#include <cstdint>
		52	#include <cstring>
		53	#include <iterator>
		54	#include <string>
		55	#include <tuple>
		56	#include <utility>
		57
		58	namespace llvm {
		59
		60	class APInt;
		61	class Constant;
		62	class GlobalValue;
		63	class MachineBasicBlock;
		64	class MachineConstantPoolValue;
		65	class MCSymbol;
		66	class raw_ostream;
		67	class SDNode;
		68	class SelectionDAG;
		69	class Type;
		70	class Value;
		71
		72	void checkForCycles(const SDNode N, const SelectionDAG DAG = nullptr,
		73	bool force = false);
		74
		75	/// This represents a list of ValueType's that has been intern'd by
		76	/// a SelectionDAG. Instances of this simple value class are returned by
		77	/// SelectionDAG::getVTList(...).
		78	///
		79	struct SDVTList {
		80	const EVT *VTs;
		81	unsigned int NumVTs;
		82	};
		83
		84	namespace ISD {
		85
		86	/// Node predicates
		87
		88	/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
		89	/// same constant or undefined, return true and return the constant value in
		90	/// \p SplatValue.
		91	bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
		92
		93	/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
		94	/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
		95	/// true, it only checks BUILD_VECTOR.
		96	bool isConstantSplatVectorAllOnes(const SDNode *N,
		97	bool BuildVectorOnly = false);
		98
		99	/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
		100	/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
		101	/// only checks BUILD_VECTOR.
		102	bool isConstantSplatVectorAllZeros(const SDNode *N,
		103	bool BuildVectorOnly = false);
		104
		105	/// Return true if the specified node is a BUILD_VECTOR where all of the
		106	/// elements are ~0 or undef.
		107	bool isBuildVectorAllOnes(const SDNode *N);
		108
		109	/// Return true if the specified node is a BUILD_VECTOR where all of the
		110	/// elements are 0 or undef.
		111	bool isBuildVectorAllZeros(const SDNode *N);
		112
		113	/// Return true if the specified node is a BUILD_VECTOR node of all
		114	/// ConstantSDNode or undef.
		115	bool isBuildVectorOfConstantSDNodes(const SDNode *N);
		116
		117	/// Return true if the specified node is a BUILD_VECTOR node of all
		118	/// ConstantFPSDNode or undef.
		119	bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
		120
		121	/// Returns true if the specified node is a vector where all elements can
		122	/// be truncated to the specified element size without a loss in meaning.
		123	bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed);
		124
		125	/// Return true if the node has at least one operand and all operands of the
		126	/// specified node are ISD::UNDEF.
		127	bool allOperandsUndef(const SDNode *N);
		128
		129	/// Return true if the specified node is FREEZE(UNDEF).
		130	bool isFreezeUndef(const SDNode *N);
		131
		132	} // end namespace ISD
		133
		134	//===----------------------------------------------------------------------===//
		135	/// Unlike LLVM values, Selection DAG nodes may return multiple
		136	/// values as the result of a computation. Many nodes return multiple values,
		137	/// from loads (which define a token and a return value) to ADDC (which returns
		138	/// a result and a carry value), to calls (which may return an arbitrary number
		139	/// of values).
		140	///
		141	/// As such, each use of a SelectionDAG computation must indicate the node that
		142	/// computes it as well as which return value to use from that node. This pair
		143	/// of information is represented with the SDValue value type.
		144	///
		145	class SDValue {
		146	friend struct DenseMapInfo<SDValue>;
		147
		148	SDNode *Node = nullptr; // The node defining the value we are using.
		149	unsigned ResNo = 0; // Which return value of the node we are using.
		150
		151	public:
		152	SDValue() = default;
		153	SDValue(SDNode *node, unsigned resno);
		154
		155	/// get the index which selects a specific result in the SDNode
		156	unsigned getResNo() const { return ResNo; }
		157
		158	/// get the SDNode which holds the desired result
		159	SDNode *getNode() const { return Node; }
		160
		161	/// set the SDNode
		162	void setNode(SDNode *N) { Node = N; }
		163
		164	inline SDNode *operator->() const { return Node; }
		165
		166	bool operator==(const SDValue &O) const {
		167	return Node == O.Node && ResNo == O.ResNo;
		168	}
		169	bool operator!=(const SDValue &O) const {
		170	return !operator==(O);
		171	}
		172	bool operator<(const SDValue &O) const {
		173	return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
		174	}
		175	explicit operator bool() const {
		176	return Node != nullptr;
		177	}
		178
		179	SDValue getValue(unsigned R) const {
		180	return SDValue(Node, R);
		181	}
		182
		183	/// Return true if this node is an operand of N.
		184	bool isOperandOf(const SDNode *N) const;
		185
		186	/// Return the ValueType of the referenced return value.
		187	inline EVT getValueType() const;
		188
		189	/// Return the simple ValueType of the referenced return value.
		190	MVT getSimpleValueType() const {
		191	return getValueType().getSimpleVT();
		192	}
		193
		194	/// Returns the size of the value in bits.
		195	///
		196	/// If the value type is a scalable vector type, the scalable property will
		197	/// be set and the runtime size will be a positive integer multiple of the
		198	/// base size.
		199	TypeSize getValueSizeInBits() const {
		200	return getValueType().getSizeInBits();
		201	}
		202
		203	uint64_t getScalarValueSizeInBits() const {
		204	return getValueType().getScalarType().getFixedSizeInBits();
		205	}
		206
		207	// Forwarding methods - These forward to the corresponding methods in SDNode.
		208	inline unsigned getOpcode() const;
		209	inline unsigned getNumOperands() const;
		210	inline const SDValue &getOperand(unsigned i) const;
		211	inline uint64_t getConstantOperandVal(unsigned i) const;
		212	inline const APInt &getConstantOperandAPInt(unsigned i) const;
		213	inline bool isTargetMemoryOpcode() const;
		214	inline bool isTargetOpcode() const;
		215	inline bool isMachineOpcode() const;
		216	inline bool isUndef() const;
		217	inline unsigned getMachineOpcode() const;
		218	inline const DebugLoc &getDebugLoc() const;
		219	inline void dump() const;
		220	inline void dump(const SelectionDAG *G) const;
		221	inline void dumpr() const;
		222	inline void dumpr(const SelectionDAG *G) const;
		223
		224	/// Return true if this operand (which must be a chain) reaches the
		225	/// specified operand without crossing any side-effecting instructions.
		226	/// In practice, this looks through token factors and non-volatile loads.
		227	/// In order to remain efficient, this only
		228	/// looks a couple of nodes in, it does not do an exhaustive search.
		229	bool reachesChainWithoutSideEffects(SDValue Dest,
		230	unsigned Depth = 2) const;
		231
		232	/// Return true if there are no nodes using value ResNo of Node.
		233	inline bool use_empty() const;
		234
		235	/// Return true if there is exactly one node using value ResNo of Node.
		236	inline bool hasOneUse() const;
		237	};
		238
		239	template<> struct DenseMapInfo<SDValue> {
		240	static inline SDValue getEmptyKey() {
		241	SDValue V;
		242	V.ResNo = -1U;
		243	return V;
		244	}
		245
		246	static inline SDValue getTombstoneKey() {
		247	SDValue V;
		248	V.ResNo = -2U;
		249	return V;
		250	}
		251
		252	static unsigned getHashValue(const SDValue &Val) {
		253	return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
		254	(unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
		255	}
		256
		257	static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
		258	return LHS == RHS;
		259	}
		260	};
		261
		262	/// Allow casting operators to work directly on
		263	/// SDValues as if they were SDNode*'s.
		264	template<> struct simplify_type<SDValue> {
		265	using SimpleType = SDNode *;
		266
		267	static SimpleType getSimplifiedValue(SDValue &Val) {
		268	return Val.getNode();
		269	}
		270	};
		271	template<> struct simplify_type<const SDValue> {
		272	using SimpleType = /const/ SDNode *;
		273
		274	static SimpleType getSimplifiedValue(const SDValue &Val) {
		275	return Val.getNode();
		276	}
		277	};
		278
		279	/// Represents a use of a SDNode. This class holds an SDValue,
		280	/// which records the SDNode being used and the result number, a
		281	/// pointer to the SDNode using the value, and Next and Prev pointers,
		282	/// which link together all the uses of an SDNode.
		283	///
		284	class SDUse {
		285	/// Val - The value being used.
		286	SDValue Val;
		287	/// User - The user of this value.
		288	SDNode *User = nullptr;
		289	/// Prev, Next - Pointers to the uses list of the SDNode referred by
		290	/// this operand.
		291	SDUse **Prev = nullptr;
		292	SDUse *Next = nullptr;
		293
		294	public:
		295	SDUse() = default;
		296	SDUse(const SDUse &U) = delete;
		297	SDUse &operator=(const SDUse &) = delete;
		298
		299	/// Normally SDUse will just implicitly convert to an SDValue that it holds.
		300	operator const SDValue&() const { return Val; }
		301
		302	/// If implicit conversion to SDValue doesn't work, the get() method returns
		303	/// the SDValue.
		304	const SDValue &get() const { return Val; }
		305
		306	/// This returns the SDNode that contains this Use.
		307	SDNode *getUser() { return User; }
		308	const SDNode *getUser() const { return User; }
		309
		310	/// Get the next SDUse in the use list.
		311	SDUse *getNext() const { return Next; }
		312
		313	/// Convenience function for get().getNode().
		314	SDNode *getNode() const { return Val.getNode(); }
		315	/// Convenience function for get().getResNo().
		316	unsigned getResNo() const { return Val.getResNo(); }
		317	/// Convenience function for get().getValueType().
		318	EVT getValueType() const { return Val.getValueType(); }
		319
		320	/// Convenience function for get().operator==
		321	bool operator==(const SDValue &V) const {
		322	return Val == V;
		323	}
		324
		325	/// Convenience function for get().operator!=
		326	bool operator!=(const SDValue &V) const {
		327	return Val != V;
		328	}
		329
		330	/// Convenience function for get().operator<
		331	bool operator<(const SDValue &V) const {
		332	return Val < V;
		333	}
		334
		335	private:
		336	friend class SelectionDAG;
		337	friend class SDNode;
		338	// TODO: unfriend HandleSDNode once we fix its operand handling.
		339	friend class HandleSDNode;
		340
		341	void setUser(SDNode *p) { User = p; }
		342
		343	/// Remove this use from its existing use list, assign it the
		344	/// given value, and add it to the new value's node's use list.
		345	inline void set(const SDValue &V);
		346	/// Like set, but only supports initializing a newly-allocated
		347	/// SDUse with a non-null value.
		348	inline void setInitial(const SDValue &V);
		349	/// Like set, but only sets the Node portion of the value,
		350	/// leaving the ResNo portion unmodified.
		351	inline void setNode(SDNode *N);
		352
		353	void addToList(SDUse **List) {
		354	Next = *List;
		355	if (Next) Next->Prev = &Next;
		356	Prev = List;
		357	*List = this;
		358	}
		359
		360	void removeFromList() {
		361	*Prev = Next;
		362	if (Next) Next->Prev = Prev;
		363	}
		364	};
		365
		366	/// simplify_type specializations - Allow casting operators to work directly on
		367	/// SDValues as if they were SDNode*'s.
		368	template<> struct simplify_type<SDUse> {
		369	using SimpleType = SDNode *;
		370
		371	static SimpleType getSimplifiedValue(SDUse &Val) {
		372	return Val.getNode();
		373	}
		374	};
		375
		376	/// These are IR-level optimization flags that may be propagated to SDNodes.
		377	/// TODO: This data structure should be shared by the IR optimizer and the
		378	/// the backend.
		379	struct SDNodeFlags {
		380	private:
		381	bool NoUnsignedWrap : 1;
		382	bool NoSignedWrap : 1;
		383	bool Exact : 1;
		384	bool NoNaNs : 1;
		385	bool NoInfs : 1;
		386	bool NoSignedZeros : 1;
		387	bool AllowReciprocal : 1;
		388	bool AllowContract : 1;
		389	bool ApproximateFuncs : 1;
		390	bool AllowReassociation : 1;
		391
		392	// We assume instructions do not raise floating-point exceptions by default,
		393	// and only those marked explicitly may do so. We could choose to represent
		394	// this via a positive "FPExcept" flags like on the MI level, but having a
		395	// negative "NoFPExcept" flag here (that defaults to true) makes the flag
		396	// intersection logic more straightforward.
		397	bool NoFPExcept : 1;
		398
		399	public:
		400	/// Default constructor turns off all optimization flags.
		401	SDNodeFlags()
		402	: NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
		403	NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
		404	AllowContract(false), ApproximateFuncs(false),
		405	AllowReassociation(false), NoFPExcept(false) {}
		406
		407	/// Propagate the fast-math-flags from an IR FPMathOperator.
		408	void copyFMF(const FPMathOperator &FPMO) {
		409	setNoNaNs(FPMO.hasNoNaNs());
		410	setNoInfs(FPMO.hasNoInfs());
		411	setNoSignedZeros(FPMO.hasNoSignedZeros());
		412	setAllowReciprocal(FPMO.hasAllowReciprocal());
		413	setAllowContract(FPMO.hasAllowContract());
		414	setApproximateFuncs(FPMO.hasApproxFunc());
		415	setAllowReassociation(FPMO.hasAllowReassoc());
		416	}
		417
		418	// These are mutators for each flag.
		419	void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
		420	void setNoSignedWrap(bool b) { NoSignedWrap = b; }
		421	void setExact(bool b) { Exact = b; }
		422	void setNoNaNs(bool b) { NoNaNs = b; }
		423	void setNoInfs(bool b) { NoInfs = b; }
		424	void setNoSignedZeros(bool b) { NoSignedZeros = b; }
		425	void setAllowReciprocal(bool b) { AllowReciprocal = b; }
		426	void setAllowContract(bool b) { AllowContract = b; }
		427	void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
		428	void setAllowReassociation(bool b) { AllowReassociation = b; }
		429	void setNoFPExcept(bool b) { NoFPExcept = b; }
		430
		431	// These are accessors for each flag.
		432	bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
		433	bool hasNoSignedWrap() const { return NoSignedWrap; }
		434	bool hasExact() const { return Exact; }
		435	bool hasNoNaNs() const { return NoNaNs; }
		436	bool hasNoInfs() const { return NoInfs; }
		437	bool hasNoSignedZeros() const { return NoSignedZeros; }
		438	bool hasAllowReciprocal() const { return AllowReciprocal; }
		439	bool hasAllowContract() const { return AllowContract; }
		440	bool hasApproximateFuncs() const { return ApproximateFuncs; }
		441	bool hasAllowReassociation() const { return AllowReassociation; }
		442	bool hasNoFPExcept() const { return NoFPExcept; }
		443
		444	/// Clear any flags in this flag set that aren't also set in Flags. All
		445	/// flags will be cleared if Flags are undefined.
		446	void intersectWith(const SDNodeFlags Flags) {
		447	NoUnsignedWrap &= Flags.NoUnsignedWrap;
		448	NoSignedWrap &= Flags.NoSignedWrap;
		449	Exact &= Flags.Exact;
		450	NoNaNs &= Flags.NoNaNs;
		451	NoInfs &= Flags.NoInfs;
		452	NoSignedZeros &= Flags.NoSignedZeros;
		453	AllowReciprocal &= Flags.AllowReciprocal;
		454	AllowContract &= Flags.AllowContract;
		455	ApproximateFuncs &= Flags.ApproximateFuncs;
		456	AllowReassociation &= Flags.AllowReassociation;
		457	NoFPExcept &= Flags.NoFPExcept;
		458	}
		459	};
		460
		461	/// Represents one node in the SelectionDAG.
		462	///
		463	class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
		464	private:
		465	/// The operation that this node performs.
		466	int32_t NodeType;
		467
		468	public:
		469	/// Unique and persistent id per SDNode in the DAG. Used for debug printing.
		470	/// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
		471	/// intentionally because it adds unneeded complexity without noticeable
		472	/// benefits (see discussion with @thakis in D120714).
		473	uint16_t PersistentId;
		474
		475	protected:
		476	// We define a set of mini-helper classes to help us interpret the bits in our
		477	// SubclassData. These are designed to fit within a uint16_t so they pack
		478	// with PersistentId.
		479
		480	#if defined(_AIX) && (!defined(__GNUC__) \|\| defined(__clang__))
		481	// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
		482	// and give the `pack` pragma push semantics.
		483	#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")
		484	#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")
		485	#else
		486	#define BEGIN_TWO_BYTE_PACK()
		487	#define END_TWO_BYTE_PACK()
		488	#endif
		489
		490	BEGIN_TWO_BYTE_PACK()
		491	class SDNodeBitfields {
		492	friend class SDNode;
		493	friend class MemIntrinsicSDNode;
		494	friend class MemSDNode;
		495	friend class SelectionDAG;
		496
		497	uint16_t HasDebugValue : 1;
		498	uint16_t IsMemIntrinsic : 1;
		499	uint16_t IsDivergent : 1;
		500	};
		501	enum { NumSDNodeBits = 3 };
		502
		503	class ConstantSDNodeBitfields {
		504	friend class ConstantSDNode;
		505
		506	uint16_t : NumSDNodeBits;
		507
		508	uint16_t IsOpaque : 1;
		509	};
		510
		511	class MemSDNodeBitfields {
		512	friend class MemSDNode;
		513	friend class MemIntrinsicSDNode;
		514	friend class AtomicSDNode;
		515
		516	uint16_t : NumSDNodeBits;
		517
		518	uint16_t IsVolatile : 1;
		519	uint16_t IsNonTemporal : 1;
		520	uint16_t IsDereferenceable : 1;
		521	uint16_t IsInvariant : 1;
		522	};
		523	enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
		524
		525	class LSBaseSDNodeBitfields {
		526	friend class LSBaseSDNode;
		527	friend class VPBaseLoadStoreSDNode;
		528	friend class MaskedLoadStoreSDNode;
		529	friend class MaskedGatherScatterSDNode;
		530	friend class VPGatherScatterSDNode;
		531
		532	uint16_t : NumMemSDNodeBits;
		533
		534	// This storage is shared between disparate class hierarchies to hold an
		535	// enumeration specific to the class hierarchy in use.
		536	// LSBaseSDNode => enum ISD::MemIndexedMode
		537	// VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
		538	// MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
		539	// VPGatherScatterSDNode => enum ISD::MemIndexType
		540	// MaskedGatherScatterSDNode => enum ISD::MemIndexType
		541	uint16_t AddressingMode : 3;
		542	};
		543	enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
		544
		545	class LoadSDNodeBitfields {
		546	friend class LoadSDNode;
		547	friend class VPLoadSDNode;
		548	friend class VPStridedLoadSDNode;
		549	friend class MaskedLoadSDNode;
		550	friend class MaskedGatherSDNode;
		551	friend class VPGatherSDNode;
		552
		553	uint16_t : NumLSBaseSDNodeBits;
		554
		555	uint16_t ExtTy : 2; // enum ISD::LoadExtType
		556	uint16_t IsExpanding : 1;
		557	};
		558
		559	class StoreSDNodeBitfields {
		560	friend class StoreSDNode;
		561	friend class VPStoreSDNode;
		562	friend class VPStridedStoreSDNode;
		563	friend class MaskedStoreSDNode;
		564	friend class MaskedScatterSDNode;
		565	friend class VPScatterSDNode;
		566
		567	uint16_t : NumLSBaseSDNodeBits;
		568
		569	uint16_t IsTruncating : 1;
		570	uint16_t IsCompressing : 1;
		571	};
		572
		573	union {
		574	char RawSDNodeBits[sizeof(uint16_t)];
		575	SDNodeBitfields SDNodeBits;
		576	ConstantSDNodeBitfields ConstantSDNodeBits;
		577	MemSDNodeBitfields MemSDNodeBits;
		578	LSBaseSDNodeBitfields LSBaseSDNodeBits;
		579	LoadSDNodeBitfields LoadSDNodeBits;
		580	StoreSDNodeBitfields StoreSDNodeBits;
		581	};
		582	END_TWO_BYTE_PACK()
		583	#undef BEGIN_TWO_BYTE_PACK
		584	#undef END_TWO_BYTE_PACK
		585
		586	// RawSDNodeBits must cover the entirety of the union. This means that all of
		587	// the union's members must have size <= RawSDNodeBits. We write the RHS as
		588	// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
		589	static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
		590	static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
		591	static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
		592	static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
		593	static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
		594	static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
		595
		596	private:
		597	friend class SelectionDAG;
		598	// TODO: unfriend HandleSDNode once we fix its operand handling.
		599	friend class HandleSDNode;
		600
		601	/// Unique id per SDNode in the DAG.
		602	int NodeId = -1;
		603
		604	/// The values that are used by this operation.
		605	SDUse *OperandList = nullptr;
		606
		607	/// The types of the values this node defines. SDNode's may
		608	/// define multiple values simultaneously.
		609	const EVT *ValueList;
		610
		611	/// List of uses for this SDNode.
		612	SDUse *UseList = nullptr;
		613
		614	/// The number of entries in the Operand/Value list.
		615	unsigned short NumOperands = 0;
		616	unsigned short NumValues;
		617
		618	// The ordering of the SDNodes. It roughly corresponds to the ordering of the
		619	// original LLVM instructions.
		620	// This is used for turning off scheduling, because we'll forgo
		621	// the normal scheduling algorithms and output the instructions according to
		622	// this ordering.
		623	unsigned IROrder;
		624
		625	/// Source line information.
		626	DebugLoc debugLoc;
		627
		628	/// Return a pointer to the specified value type.
		629	static const EVT *getValueTypeList(EVT VT);
		630
		631	SDNodeFlags Flags;
		632
		633	uint32_t CFIType = 0;
		634
		635	public:
		636	//===--------------------------------------------------------------------===//
		637	// Accessors
		638	//
		639
		640	/// Return the SelectionDAG opcode value for this node. For
		641	/// pre-isel nodes (those for which isMachineOpcode returns false), these
		642	/// are the opcode values in the ISD and <target>ISD namespaces. For
		643	/// post-isel opcodes, see getMachineOpcode.
		644	unsigned getOpcode() const { return (unsigned)NodeType; }
		645
		646	/// Test if this node has a target-specific opcode (in the
		647	/// \<target\>ISD namespace).
		648	bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
		649
		650	/// Test if this node has a target-specific opcode that may raise
		651	/// FP exceptions (in the \<target\>ISD namespace and greater than
		652	/// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
		653	/// opcode are currently automatically considered to possibly raise
		654	/// FP exceptions as well.
		655	bool isTargetStrictFPOpcode() const {
		656	return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
		657	}
		658
		659	/// Test if this node has a target-specific
		660	/// memory-referencing opcode (in the \<target\>ISD namespace and
		661	/// greater than FIRST_TARGET_MEMORY_OPCODE).
		662	bool isTargetMemoryOpcode() const {
		663	return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
		664	}
		665
		666	/// Return true if the type of the node type undefined.
		667	bool isUndef() const { return NodeType == ISD::UNDEF; }
		668
		669	/// Test if this node is a memory intrinsic (with valid pointer information).
		670	/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
		671	/// non-memory intrinsics (with chains) that are not really instances of
		672	/// MemSDNode. For such nodes, we need some extra state to determine the
		673	/// proper classof relationship.
		674	bool isMemIntrinsic() const {
		675	return (NodeType == ISD::INTRINSIC_W_CHAIN \|\|
		676	NodeType == ISD::INTRINSIC_VOID) &&
		677	SDNodeBits.IsMemIntrinsic;
		678	}
		679
		680	/// Test if this node is a strict floating point pseudo-op.
		681	bool isStrictFPOpcode() {
		682	switch (NodeType) {
		683	default:
		684	return false;
		685	case ISD::STRICT_FP16_TO_FP:
		686	case ISD::STRICT_FP_TO_FP16:
		687	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
		688	case ISD::STRICT_##DAGN:
		689	#include "llvm/IR/ConstrainedOps.def"
		690	return true;
		691	}
		692	}
		693
		694	/// Test if this node is a vector predication operation.
		695	bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }
		696
		697	/// Test if this node has a post-isel opcode, directly
		698	/// corresponding to a MachineInstr opcode.
		699	bool isMachineOpcode() const { return NodeType < 0; }
		700
		701	/// This may only be called if isMachineOpcode returns
		702	/// true. It returns the MachineInstr opcode value that the node's opcode
		703	/// corresponds to.
		704	unsigned getMachineOpcode() const {
		705	assert(isMachineOpcode() && "Not a MachineInstr opcode!");
		706	return ~NodeType;
		707	}
		708
		709	bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
		710	void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
		711
		712	bool isDivergent() const { return SDNodeBits.IsDivergent; }
		713
		714	/// Return true if there are no uses of this node.
		715	bool use_empty() const { return UseList == nullptr; }
		716
		717	/// Return true if there is exactly one use of this node.
		718	bool hasOneUse() const { return hasSingleElement(uses()); }
		719
		720	/// Return the number of uses of this node. This method takes
		721	/// time proportional to the number of uses.
		722	size_t use_size() const { return std::distance(use_begin(), use_end()); }
		723
		724	/// Return the unique node id.
		725	int getNodeId() const { return NodeId; }
		726
		727	/// Set unique node id.
		728	void setNodeId(int Id) { NodeId = Id; }
		729
		730	/// Return the node ordering.
		731	unsigned getIROrder() const { return IROrder; }
		732
		733	/// Set the node ordering.
		734	void setIROrder(unsigned Order) { IROrder = Order; }
		735
		736	/// Return the source location info.
		737	const DebugLoc &getDebugLoc() const { return debugLoc; }
		738
		739	/// Set source location info. Try to avoid this, putting
		740	/// it in the constructor is preferable.
		741	void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
		742
		743	/// This class provides iterator support for SDUse
		744	/// operands that use a specific SDNode.
		745	class use_iterator {
		746	friend class SDNode;
		747
		748	SDUse *Op = nullptr;
		749
		750	explicit use_iterator(SDUse *op) : Op(op) {}
		751
		752	public:
		753	using iterator_category = std::forward_iterator_tag;
		754	using value_type = SDUse;
		755	using difference_type = std::ptrdiff_t;
		756	using pointer = value_type *;
		757	using reference = value_type &;
		758
		759	use_iterator() = default;
		760	use_iterator(const use_iterator &I) = default;
		761	use_iterator &operator=(const use_iterator &) = default;
		762
		763	bool operator==(const use_iterator &x) const { return Op == x.Op; }
		764	bool operator!=(const use_iterator &x) const {
		765	return !operator==(x);
		766	}
		767
		768	/// Return true if this iterator is at the end of uses list.
		769	bool atEnd() const { return Op == nullptr; }
		770
		771	// Iterator traversal: forward iteration only.
		772	use_iterator &operator++() { // Preincrement
		773	assert(Op && "Cannot increment end iterator!");
		774	Op = Op->getNext();
		775	return *this;
		776	}
		777
		778	use_iterator operator++(int) { // Postincrement
		779	use_iterator tmp = this; ++this; return tmp;
		780	}
		781
		782	/// Retrieve a pointer to the current user node.
		783	SDNode operator() const {
		784	assert(Op && "Cannot dereference end iterator!");
		785	return Op->getUser();
		786	}
		787
		788	SDNode operator->() const { return operator(); }
		789
		790	SDUse &getUse() const { return *Op; }
		791
		792	/// Retrieve the operand # of this use in its user.
		793	unsigned getOperandNo() const {
		794	assert(Op && "Cannot dereference end iterator!");
		795	return (unsigned)(Op - Op->getUser()->OperandList);
		796	}
		797	};
		798
		799	/// Provide iteration support to walk over all uses of an SDNode.
		800	use_iterator use_begin() const {
		801	return use_iterator(UseList);
		802	}
		803
		804	static use_iterator use_end() { return use_iterator(nullptr); }
		805
		806	inline iterator_range<use_iterator> uses() {
		807	return make_range(use_begin(), use_end());
		808	}
		809	inline iterator_range<use_iterator> uses() const {
		810	return make_range(use_begin(), use_end());
		811	}
		812
		813	/// Return true if there are exactly NUSES uses of the indicated value.
		814	/// This method ignores uses of other values defined by this operation.
		815	bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
		816
		817	/// Return true if there are any use of the indicated value.
		818	/// This method ignores uses of other values defined by this operation.
		819	bool hasAnyUseOfValue(unsigned Value) const;
		820
		821	/// Return true if this node is the only use of N.
		822	bool isOnlyUserOf(const SDNode *N) const;
		823
		824	/// Return true if this node is an operand of N.
		825	bool isOperandOf(const SDNode *N) const;
		826
		827	/// Return true if this node is a predecessor of N.
		828	/// NOTE: Implemented on top of hasPredecessor and every bit as
		829	/// expensive. Use carefully.
		830	bool isPredecessorOf(const SDNode *N) const {
		831	return N->hasPredecessor(this);
		832	}
		833
		834	/// Return true if N is a predecessor of this node.
		835	/// N is either an operand of this node, or can be reached by recursively
		836	/// traversing up the operands.
		837	/// NOTE: This is an expensive method. Use it carefully.
		838	bool hasPredecessor(const SDNode *N) const;
		839
		840	/// Returns true if N is a predecessor of any node in Worklist. This
		841	/// helper keeps Visited and Worklist sets externally to allow unions
		842	/// searches to be performed in parallel, caching of results across
		843	/// queries and incremental addition to Worklist. Stops early if N is
		844	/// found but will resume. Remember to clear Visited and Worklists
		845	/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
		846	/// giving up. The TopologicalPrune flag signals that positive NodeIds are
		847	/// topologically ordered (Operands have strictly smaller node id) and search
		848	/// can be pruned leveraging this.
		849	static bool hasPredecessorHelper(const SDNode *N,
		850	SmallPtrSetImpl<const SDNode *> &Visited,
		851	SmallVectorImpl<const SDNode *> &Worklist,
		852	unsigned int MaxSteps = 0,
		853	bool TopologicalPrune = false) {
		854	SmallVector<const SDNode *, 8> DeferredNodes;
		855	if (Visited.count(N))
		856	return true;
		857
		858	// Node Id's are assigned in three places: As a topological
		859	// ordering (> 0), during legalization (results in values set to
		860	// 0), new nodes (set to -1). If N has a topolgical id then we
		861	// know that all nodes with ids smaller than it cannot be
		862	// successors and we need not check them. Filter out all node
		863	// that can't be matches. We add them to the worklist before exit
		864	// in case of multiple calls. Note that during selection the topological id
		865	// may be violated if a node's predecessor is selected before it. We mark
		866	// this at selection negating the id of unselected successors and
		867	// restricting topological pruning to positive ids.
		868
		869	int NId = N->getNodeId();
		870	// If we Invalidated the Id, reconstruct original NId.
		871	if (NId < -1)
		872	NId = -(NId + 1);
		873
		874	bool Found = false;
		875	while (!Worklist.empty()) {
		876	const SDNode *M = Worklist.pop_back_val();
		877	int MId = M->getNodeId();
		878	if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
		879	(MId > 0) && (MId < NId)) {
		880	DeferredNodes.push_back(M);
		881	continue;
		882	}
		883	for (const SDValue &OpV : M->op_values()) {
		884	SDNode *Op = OpV.getNode();
		885	if (Visited.insert(Op).second)
		886	Worklist.push_back(Op);
		887	if (Op == N)
		888	Found = true;
		889	}
		890	if (Found)
		891	break;
		892	if (MaxSteps != 0 && Visited.size() >= MaxSteps)
		893	break;
		894	}
		895	// Push deferred nodes back on worklist.
		896	Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
		897	// If we bailed early, conservatively return found.
		898	if (MaxSteps != 0 && Visited.size() >= MaxSteps)
		899	return true;
		900	return Found;
		901	}
		902
		903	/// Return true if all the users of N are contained in Nodes.
		904	/// NOTE: Requires at least one match, but doesn't require them all.
		905	static bool areOnlyUsersOf(ArrayRef<const SDNode > Nodes, const SDNode N);
		906
		907	/// Return the number of values used by this operation.
		908	unsigned getNumOperands() const { return NumOperands; }
		909
		910	/// Return the maximum number of operands that a SDNode can hold.
		911	static constexpr size_t getMaxNumOperands() {
		912	return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
		913	}
		914
		915	/// Helper method returns the integer value of a ConstantSDNode operand.
		916	inline uint64_t getConstantOperandVal(unsigned Num) const;
		917
		918	/// Helper method returns the APInt of a ConstantSDNode operand.
		919	inline const APInt &getConstantOperandAPInt(unsigned Num) const;
		920
		921	const SDValue &getOperand(unsigned Num) const {
		922	assert(Num < NumOperands && "Invalid child # of SDNode!");
		923	return OperandList[Num];
		924	}
		925
		926	using op_iterator = SDUse *;
		927
		928	op_iterator op_begin() const { return OperandList; }
		929	op_iterator op_end() const { return OperandList+NumOperands; }
		930	ArrayRef<SDUse> ops() const { return ArrayRef(op_begin(), op_end()); }
		931
		932	/// Iterator for directly iterating over the operand SDValue's.
		933	struct value_op_iterator
		934	: iterator_adaptor_base<value_op_iterator, op_iterator,
		935	std::random_access_iterator_tag, SDValue,
		936	ptrdiff_t, value_op_iterator *,
		937	value_op_iterator *> {
		938	explicit value_op_iterator(SDUse *U = nullptr)
		939	: iterator_adaptor_base(U) {}
		940
		941	const SDValue &operator*() const { return I->get(); }
		942	};
		943
		944	iterator_range<value_op_iterator> op_values() const {
		945	return make_range(value_op_iterator(op_begin()),
		946	value_op_iterator(op_end()));
		947	}
		948
		949	SDVTList getVTList() const {
		950	SDVTList X = { ValueList, NumValues };
		951	return X;
		952	}
		953
		954	/// If this node has a glue operand, return the node
		955	/// to which the glue operand points. Otherwise return NULL.
		956	SDNode *getGluedNode() const {
		957	if (getNumOperands() != 0 &&
		958	getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
		959	return getOperand(getNumOperands()-1).getNode();
		960	return nullptr;
		961	}
		962
		963	/// If this node has a glue value with a user, return
		964	/// the user (there is at most one). Otherwise return NULL.
		965	SDNode *getGluedUser() const {
		966	for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
		967	if (UI.getUse().get().getValueType() == MVT::Glue)
		968	return *UI;
		969	return nullptr;
		970	}
		971
		972	SDNodeFlags getFlags() const { return Flags; }
		973	void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
		974
		975	/// Clear any flags in this node that aren't also set in Flags.
		976	/// If Flags is not in a defined state then this has no effect.
		977	void intersectFlagsWith(const SDNodeFlags Flags);
		978
		979	void setCFIType(uint32_t Type) { CFIType = Type; }
		980	uint32_t getCFIType() const { return CFIType; }
		981
		982	/// Return the number of values defined/returned by this operator.
		983	unsigned getNumValues() const { return NumValues; }
		984
		985	/// Return the type of a specified result.
		986	EVT getValueType(unsigned ResNo) const {
		987	assert(ResNo < NumValues && "Illegal result number!");
		988	return ValueList[ResNo];
		989	}
		990
		991	/// Return the type of a specified result as a simple type.
		992	MVT getSimpleValueType(unsigned ResNo) const {
		993	return getValueType(ResNo).getSimpleVT();
		994	}
		995
		996	/// Returns MVT::getSizeInBits(getValueType(ResNo)).
		997	///
		998	/// If the value type is a scalable vector type, the scalable property will
		999	/// be set and the runtime size will be a positive integer multiple of the
		1000	/// base size.
		1001	TypeSize getValueSizeInBits(unsigned ResNo) const {
		1002	return getValueType(ResNo).getSizeInBits();
		1003	}
		1004
		1005	using value_iterator = const EVT *;
		1006
		1007	value_iterator value_begin() const { return ValueList; }
		1008	value_iterator value_end() const { return ValueList+NumValues; }
		1009	iterator_range<value_iterator> values() const {
		1010	return llvm::make_range(value_begin(), value_end());
		1011	}
		1012
		1013	/// Return the opcode of this operation for printing.
		1014	std::string getOperationName(const SelectionDAG *G = nullptr) const;
		1015	static const char* getIndexedModeName(ISD::MemIndexedMode AM);
		1016	void print_types(raw_ostream &OS, const SelectionDAG *G) const;
		1017	void print_details(raw_ostream &OS, const SelectionDAG *G) const;
		1018	void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
		1019	void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
		1020
		1021	/// Print a SelectionDAG node and all children down to
		1022	/// the leaves. The given SelectionDAG allows target-specific nodes
		1023	/// to be printed in human-readable form. Unlike printr, this will
		1024	/// print the whole DAG, including children that appear multiple
		1025	/// times.
		1026	///
		1027	void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
		1028
		1029	/// Print a SelectionDAG node and children up to
		1030	/// depth "depth." The given SelectionDAG allows target-specific
		1031	/// nodes to be printed in human-readable form. Unlike printr, this
		1032	/// will print children that appear multiple times wherever they are
		1033	/// used.
		1034	///
		1035	void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
		1036	unsigned depth = 100) const;
		1037
		1038	/// Dump this node, for debugging.
		1039	void dump() const;
		1040
		1041	/// Dump (recursively) this node and its use-def subgraph.
		1042	void dumpr() const;
		1043
		1044	/// Dump this node, for debugging.
		1045	/// The given SelectionDAG allows target-specific nodes to be printed
		1046	/// in human-readable form.
		1047	void dump(const SelectionDAG *G) const;
		1048
		1049	/// Dump (recursively) this node and its use-def subgraph.
		1050	/// The given SelectionDAG allows target-specific nodes to be printed
		1051	/// in human-readable form.
		1052	void dumpr(const SelectionDAG *G) const;
		1053
		1054	/// printrFull to dbgs(). The given SelectionDAG allows
		1055	/// target-specific nodes to be printed in human-readable form.
		1056	/// Unlike dumpr, this will print the whole DAG, including children
		1057	/// that appear multiple times.
		1058	void dumprFull(const SelectionDAG *G = nullptr) const;
		1059
		1060	/// printrWithDepth to dbgs(). The given
		1061	/// SelectionDAG allows target-specific nodes to be printed in
		1062	/// human-readable form. Unlike dumpr, this will print children
		1063	/// that appear multiple times wherever they are used.
		1064	///
		1065	void dumprWithDepth(const SelectionDAG *G = nullptr,
		1066	unsigned depth = 100) const;
		1067
		1068	/// Gather unique data for the node.
		1069	void Profile(FoldingSetNodeID &ID) const;
		1070
		1071	/// This method should only be used by the SDUse class.
		1072	void addUse(SDUse &U) { U.addToList(&UseList); }
		1073
		1074	protected:
		1075	static SDVTList getSDVTList(EVT VT) {
		1076	SDVTList Ret = { getValueTypeList(VT), 1 };
		1077	return Ret;
		1078	}
		1079
		1080	/// Create an SDNode.
		1081	///
		1082	/// SDNodes are created without any operands, and never own the operand
		1083	/// storage. To add operands, see SelectionDAG::createOperands.
		1084	SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
		1085	: NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
		1086	IROrder(Order), debugLoc(std::move(dl)) {
		1087	memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
		1088	assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
		1089	assert(NumValues == VTs.NumVTs &&
		1090	"NumValues wasn't wide enough for its operands!");
		1091	}
		1092
		1093	/// Release the operands and set this node to have zero operands.
		1094	void DropOperands();
		1095	};
		1096
		1097	/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
		1098	/// into SDNode creation functions.
		1099	/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
		1100	/// from the original Instruction, and IROrder is the ordinal position of
		1101	/// the instruction.
		1102	/// When an SDNode is created after the DAG is being built, both DebugLoc and
		1103	/// the IROrder are propagated from the original SDNode.
		1104	/// So SDLoc class provides two constructors besides the default one, one to
		1105	/// be used by the DAGBuilder, the other to be used by others.
		1106	class SDLoc {
		1107	private:
		1108	DebugLoc DL;
		1109	int IROrder = 0;
		1110
		1111	public:
		1112	SDLoc() = default;
		1113	SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
		1114	SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
		1115	SDLoc(const Instruction *I, int Order) : IROrder(Order) {
		1116	assert(Order >= 0 && "bad IROrder");
		1117	if (I)
		1118	DL = I->getDebugLoc();
		1119	}
		1120
		1121	unsigned getIROrder() const { return IROrder; }
		1122	const DebugLoc &getDebugLoc() const { return DL; }
		1123	};
		1124
		1125	// Define inline functions from the SDValue class.
		1126
		1127	inline SDValue::SDValue(SDNode *node, unsigned resno)
		1128	: Node(node), ResNo(resno) {
		1129	// Explicitly check for !ResNo to avoid use-after-free, because there are
		1130	// callers that use SDValue(N, 0) with a deleted N to indicate successful
		1131	// combines.
		1132	assert((!Node \|\| !ResNo \|\| ResNo < Node->getNumValues()) &&
		1133	"Invalid result number for the given node!");
		1134	assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.");
		1135	}
		1136
		1137	inline unsigned SDValue::getOpcode() const {
		1138	return Node->getOpcode();
		1139	}
		1140
		1141	inline EVT SDValue::getValueType() const {
		1142	return Node->getValueType(ResNo);
		1143	}
		1144
		1145	inline unsigned SDValue::getNumOperands() const {
		1146	return Node->getNumOperands();
		1147	}
		1148
		1149	inline const SDValue &SDValue::getOperand(unsigned i) const {
		1150	return Node->getOperand(i);
		1151	}
		1152
		1153	inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
		1154	return Node->getConstantOperandVal(i);
		1155	}
		1156
		1157	inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
		1158	return Node->getConstantOperandAPInt(i);
		1159	}
		1160
		1161	inline bool SDValue::isTargetOpcode() const {
		1162	return Node->isTargetOpcode();
		1163	}
		1164
		1165	inline bool SDValue::isTargetMemoryOpcode() const {
		1166	return Node->isTargetMemoryOpcode();
		1167	}
		1168
		1169	inline bool SDValue::isMachineOpcode() const {
		1170	return Node->isMachineOpcode();
		1171	}
		1172
		1173	inline unsigned SDValue::getMachineOpcode() const {
		1174	return Node->getMachineOpcode();
		1175	}
		1176
		1177	inline bool SDValue::isUndef() const {
		1178	return Node->isUndef();
		1179	}
		1180
		1181	inline bool SDValue::use_empty() const {
		1182	return !Node->hasAnyUseOfValue(ResNo);
		1183	}
		1184
		1185	inline bool SDValue::hasOneUse() const {
		1186	return Node->hasNUsesOfValue(1, ResNo);
		1187	}
		1188
		1189	inline const DebugLoc &SDValue::getDebugLoc() const {
		1190	return Node->getDebugLoc();
		1191	}
		1192
		1193	inline void SDValue::dump() const {
		1194	return Node->dump();
		1195	}
		1196
		1197	inline void SDValue::dump(const SelectionDAG *G) const {
		1198	return Node->dump(G);
		1199	}
		1200
		1201	inline void SDValue::dumpr() const {
		1202	return Node->dumpr();
		1203	}
		1204
		1205	inline void SDValue::dumpr(const SelectionDAG *G) const {
		1206	return Node->dumpr(G);
		1207	}
		1208
		1209	// Define inline functions from the SDUse class.
		1210
		1211	inline void SDUse::set(const SDValue &V) {
		1212	if (Val.getNode()) removeFromList();
		1213	Val = V;
		1214	if (V.getNode())
		1215	V->addUse(*this);
		1216	}
		1217
		1218	inline void SDUse::setInitial(const SDValue &V) {
		1219	Val = V;
		1220	V->addUse(*this);
		1221	}
		1222
		1223	inline void SDUse::setNode(SDNode *N) {
		1224	if (Val.getNode()) removeFromList();
		1225	Val.setNode(N);
		1226	if (N) N->addUse(*this);
		1227	}
		1228
		1229	/// This class is used to form a handle around another node that
		1230	/// is persistent and is updated across invocations of replaceAllUsesWith on its
		1231	/// operand. This node should be directly created by end-users and not added to
		1232	/// the AllNodes list.
		1233	class HandleSDNode : public SDNode {
		1234	SDUse Op;
		1235
		1236	public:
		1237	explicit HandleSDNode(SDValue X)
		1238	: SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
		1239	// HandleSDNodes are never inserted into the DAG, so they won't be
		1240	// auto-numbered. Use ID 65535 as a sentinel.
		1241	PersistentId = 0xffff;
		1242
		1243	// Manually set up the operand list. This node type is special in that it's
		1244	// always stack allocated and SelectionDAG does not manage its operands.
		1245	// TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
		1246	// be so special.
		1247	Op.setUser(this);
		1248	Op.setInitial(X);
		1249	NumOperands = 1;
		1250	OperandList = &Op;
		1251	}
		1252	~HandleSDNode();
		1253
		1254	const SDValue &getValue() const { return Op; }
		1255	};
		1256
		1257	class AddrSpaceCastSDNode : public SDNode {
		1258	private:
		1259	unsigned SrcAddrSpace;
		1260	unsigned DestAddrSpace;
		1261
		1262	public:
		1263	AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
		1264	unsigned SrcAS, unsigned DestAS);
		1265
		1266	unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
		1267	unsigned getDestAddressSpace() const { return DestAddrSpace; }
		1268
		1269	static bool classof(const SDNode *N) {
		1270	return N->getOpcode() == ISD::ADDRSPACECAST;
		1271	}
		1272	};
		1273
		1274	/// This is an abstract virtual class for memory operations.
		1275	class MemSDNode : public SDNode {
		1276	private:
		1277	// VT of in-memory value.
		1278	EVT MemoryVT;
		1279
		1280	protected:
		1281	/// Memory reference information.
		1282	MachineMemOperand *MMO;
		1283
		1284	public:
		1285	MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
		1286	EVT memvt, MachineMemOperand *MMO);
		1287
		1288	bool readMem() const { return MMO->isLoad(); }
		1289	bool writeMem() const { return MMO->isStore(); }
		1290
		1291	/// Returns alignment and volatility of the memory access
		1292	Align getOriginalAlign() const { return MMO->getBaseAlign(); }
		1293	Align getAlign() const { return MMO->getAlign(); }
		1294
		1295	/// Return the SubclassData value, without HasDebugValue. This contains an
		1296	/// encoding of the volatile flag, as well as bits used by subclasses. This
		1297	/// function should only be used to compute a FoldingSetNodeID value.
		1298	/// The HasDebugValue bit is masked out because CSE map needs to match
		1299	/// nodes with debug info with nodes without debug info. Same is about
		1300	/// isDivergent bit.
		1301	unsigned getRawSubclassData() const {
		1302	uint16_t Data;
		1303	union {
		1304	char RawSDNodeBits[sizeof(uint16_t)];
		1305	SDNodeBitfields SDNodeBits;
		1306	};
		1307	memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
		1308	SDNodeBits.HasDebugValue = 0;
		1309	SDNodeBits.IsDivergent = false;
		1310	memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
		1311	return Data;
		1312	}
		1313
		1314	bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
		1315	bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
		1316	bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
		1317	bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
		1318
		1319	// Returns the offset from the location of the access.
		1320	int64_t getSrcValueOffset() const { return MMO->getOffset(); }
		1321
		1322	/// Returns the AA info that describes the dereference.
		1323	AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
		1324
		1325	/// Returns the Ranges that describes the dereference.
		1326	const MDNode *getRanges() const { return MMO->getRanges(); }
		1327
		1328	/// Returns the synchronization scope ID for this memory operation.
		1329	SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
		1330
		1331	/// Return the atomic ordering requirements for this memory operation. For
		1332	/// cmpxchg atomic operations, return the atomic ordering requirements when
		1333	/// store occurs.
		1334	AtomicOrdering getSuccessOrdering() const {
		1335	return MMO->getSuccessOrdering();
		1336	}
		1337
		1338	/// Return a single atomic ordering that is at least as strong as both the
		1339	/// success and failure orderings for an atomic operation. (For operations
		1340	/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
		1341	AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
		1342
		1343	/// Return true if the memory operation ordering is Unordered or higher.
		1344	bool isAtomic() const { return MMO->isAtomic(); }
		1345
		1346	/// Returns true if the memory operation doesn't imply any ordering
		1347	/// constraints on surrounding memory operations beyond the normal memory
		1348	/// aliasing rules.
		1349	bool isUnordered() const { return MMO->isUnordered(); }
		1350
		1351	/// Returns true if the memory operation is neither atomic or volatile.
		1352	bool isSimple() const { return !isAtomic() && !isVolatile(); }
		1353
		1354	/// Return the type of the in-memory value.
		1355	EVT getMemoryVT() const { return MemoryVT; }
		1356
		1357	/// Return a MachineMemOperand object describing the memory
		1358	/// reference performed by operation.
		1359	MachineMemOperand *getMemOperand() const { return MMO; }
		1360
		1361	const MachinePointerInfo &getPointerInfo() const {
		1362	return MMO->getPointerInfo();
		1363	}
		1364
		1365	/// Return the address space for the associated pointer
		1366	unsigned getAddressSpace() const {
		1367	return getPointerInfo().getAddrSpace();
		1368	}
		1369
		1370	/// Update this MemSDNode's MachineMemOperand information
		1371	/// to reflect the alignment of NewMMO, if it has a greater alignment.
		1372	/// This must only be used when the new alignment applies to all users of
		1373	/// this MachineMemOperand.
		1374	void refineAlignment(const MachineMemOperand *NewMMO) {
		1375	MMO->refineAlignment(NewMMO);
		1376	}
		1377
		1378	const SDValue &getChain() const { return getOperand(0); }
		1379
		1380	const SDValue &getBasePtr() const {
		1381	switch (getOpcode()) {
		1382	case ISD::STORE:
		1383	case ISD::VP_STORE:
		1384	case ISD::MSTORE:
		1385	case ISD::VP_SCATTER:
		1386	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
		1387	return getOperand(2);
		1388	case ISD::MGATHER:
		1389	case ISD::MSCATTER:
		1390	return getOperand(3);
		1391	default:
		1392	return getOperand(1);
		1393	}
		1394	}
		1395
		1396	// Methods to support isa and dyn_cast
		1397	static bool classof(const SDNode *N) {
		1398	// For some targets, we lower some target intrinsics to a MemIntrinsicNode
		1399	// with either an intrinsic or a target opcode.
		1400	switch (N->getOpcode()) {
		1401	case ISD::LOAD:
		1402	case ISD::STORE:
		1403	case ISD::PREFETCH:
		1404	case ISD::ATOMIC_CMP_SWAP:
		1405	case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
		1406	case ISD::ATOMIC_SWAP:
		1407	case ISD::ATOMIC_LOAD_ADD:
		1408	case ISD::ATOMIC_LOAD_SUB:
		1409	case ISD::ATOMIC_LOAD_AND:
		1410	case ISD::ATOMIC_LOAD_CLR:
		1411	case ISD::ATOMIC_LOAD_OR:
		1412	case ISD::ATOMIC_LOAD_XOR:
		1413	case ISD::ATOMIC_LOAD_NAND:
		1414	case ISD::ATOMIC_LOAD_MIN:
		1415	case ISD::ATOMIC_LOAD_MAX:
		1416	case ISD::ATOMIC_LOAD_UMIN:
		1417	case ISD::ATOMIC_LOAD_UMAX:
		1418	case ISD::ATOMIC_LOAD_FADD:
		1419	case ISD::ATOMIC_LOAD_FSUB:
		1420	case ISD::ATOMIC_LOAD_FMAX:
		1421	case ISD::ATOMIC_LOAD_FMIN:
		1422	case ISD::ATOMIC_LOAD_UINC_WRAP:
		1423	case ISD::ATOMIC_LOAD_UDEC_WRAP:
		1424	case ISD::ATOMIC_LOAD:
		1425	case ISD::ATOMIC_STORE:
		1426	case ISD::MLOAD:
		1427	case ISD::MSTORE:
		1428	case ISD::MGATHER:
		1429	case ISD::MSCATTER:
		1430	case ISD::VP_LOAD:
		1431	case ISD::VP_STORE:
		1432	case ISD::VP_GATHER:
		1433	case ISD::VP_SCATTER:
		1434	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
		1435	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
		1436	return true;
		1437	default:
		1438	return N->isMemIntrinsic() \|\| N->isTargetMemoryOpcode();
		1439	}
		1440	}
		1441	};
		1442
		1443	/// This is an SDNode representing atomic operations.
		1444	class AtomicSDNode : public MemSDNode {
		1445	public:
		1446	AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
		1447	EVT MemVT, MachineMemOperand *MMO)
		1448	: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
		1449	assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) \|\|
		1450	MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
		1451	}
		1452
		1453	const SDValue &getBasePtr() const { return getOperand(1); }
		1454	const SDValue &getVal() const { return getOperand(2); }
		1455
		1456	/// Returns true if this SDNode represents cmpxchg atomic operation, false
		1457	/// otherwise.
		1458	bool isCompareAndSwap() const {
		1459	unsigned Op = getOpcode();
		1460	return Op == ISD::ATOMIC_CMP_SWAP \|\|
		1461	Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
		1462	}
		1463
		1464	/// For cmpxchg atomic operations, return the atomic ordering requirements
		1465	/// when store does not occur.
		1466	AtomicOrdering getFailureOrdering() const {
		1467	assert(isCompareAndSwap() && "Must be cmpxchg operation");
		1468	return MMO->getFailureOrdering();
		1469	}
		1470
		1471	// Methods to support isa and dyn_cast
		1472	static bool classof(const SDNode *N) {
		1473	return N->getOpcode() == ISD::ATOMIC_CMP_SWAP \|\|
		1474	N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS \|\|
		1475	N->getOpcode() == ISD::ATOMIC_SWAP \|\|
		1476	N->getOpcode() == ISD::ATOMIC_LOAD_ADD \|\|
		1477	N->getOpcode() == ISD::ATOMIC_LOAD_SUB \|\|
		1478	N->getOpcode() == ISD::ATOMIC_LOAD_AND \|\|
		1479	N->getOpcode() == ISD::ATOMIC_LOAD_CLR \|\|
		1480	N->getOpcode() == ISD::ATOMIC_LOAD_OR \|\|
		1481	N->getOpcode() == ISD::ATOMIC_LOAD_XOR \|\|
		1482	N->getOpcode() == ISD::ATOMIC_LOAD_NAND \|\|
		1483	N->getOpcode() == ISD::ATOMIC_LOAD_MIN \|\|
		1484	N->getOpcode() == ISD::ATOMIC_LOAD_MAX \|\|
		1485	N->getOpcode() == ISD::ATOMIC_LOAD_UMIN \|\|
		1486	N->getOpcode() == ISD::ATOMIC_LOAD_UMAX \|\|
		1487	N->getOpcode() == ISD::ATOMIC_LOAD_FADD \|\|
		1488	N->getOpcode() == ISD::ATOMIC_LOAD_FSUB \|\|
		1489	N->getOpcode() == ISD::ATOMIC_LOAD_FMAX \|\|
		1490	N->getOpcode() == ISD::ATOMIC_LOAD_FMIN \|\|
		1491	N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP \|\|
		1492	N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP \|\|
		1493	N->getOpcode() == ISD::ATOMIC_LOAD \|\|
		1494	N->getOpcode() == ISD::ATOMIC_STORE;
		1495	}
		1496	};
		1497
		1498	/// This SDNode is used for target intrinsics that touch
		1499	/// memory and need an associated MachineMemOperand. Its opcode may be
		1500	/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
		1501	/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
		1502	class MemIntrinsicSDNode : public MemSDNode {
		1503	public:
		1504	MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
		1505	SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
		1506	: MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
		1507	SDNodeBits.IsMemIntrinsic = true;
		1508	}
		1509
		1510	// Methods to support isa and dyn_cast
		1511	static bool classof(const SDNode *N) {
		1512	// We lower some target intrinsics to their target opcode
		1513	// early a node with a target opcode can be of this class
		1514	return N->isMemIntrinsic() \|\|
		1515	N->getOpcode() == ISD::PREFETCH \|\|
		1516	N->isTargetMemoryOpcode();
		1517	}
		1518	};
		1519
		1520	/// This SDNode is used to implement the code generator
		1521	/// support for the llvm IR shufflevector instruction. It combines elements
		1522	/// from two input vectors into a new input vector, with the selection and
		1523	/// ordering of elements determined by an array of integers, referred to as
		1524	/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
		1525	/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
		1526	/// An index of -1 is treated as undef, such that the code generator may put
		1527	/// any value in the corresponding element of the result.
		1528	class ShuffleVectorSDNode : public SDNode {
		1529	// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
		1530	// is freed when the SelectionDAG object is destroyed.
		1531	const int *Mask;
		1532
		1533	protected:
		1534	friend class SelectionDAG;
		1535
		1536	ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
		1537	: SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
		1538
		1539	public:
		1540	ArrayRef<int> getMask() const {
		1541	EVT VT = getValueType(0);
		1542	return ArrayRef(Mask, VT.getVectorNumElements());
		1543	}
		1544
		1545	int getMaskElt(unsigned Idx) const {
		1546	assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
		1547	return Mask[Idx];
		1548	}
		1549
		1550	bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
		1551
		1552	int getSplatIndex() const {
		1553	assert(isSplat() && "Cannot get splat index for non-splat!");
		1554	EVT VT = getValueType(0);
		1555	for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
		1556	if (Mask[i] >= 0)
		1557	return Mask[i];
		1558
		1559	// We can choose any index value here and be correct because all elements
		1560	// are undefined. Return 0 for better potential for callers to simplify.
		1561	return 0;
		1562	}
		1563
		1564	static bool isSplatMask(const int *Mask, EVT VT);
		1565
		1566	/// Change values in a shuffle permute mask assuming
		1567	/// the two vector operands have swapped position.
		1568	static void commuteMask(MutableArrayRef<int> Mask) {
		1569	unsigned NumElems = Mask.size();
		1570	for (unsigned i = 0; i != NumElems; ++i) {
		1571	int idx = Mask[i];
		1572	if (idx < 0)
		1573	continue;
		1574	else if (idx < (int)NumElems)
		1575	Mask[i] = idx + NumElems;
		1576	else
		1577	Mask[i] = idx - NumElems;
		1578	}
		1579	}
		1580
		1581	static bool classof(const SDNode *N) {
		1582	return N->getOpcode() == ISD::VECTOR_SHUFFLE;
		1583	}
		1584	};
		1585
		1586	class ConstantSDNode : public SDNode {
		1587	friend class SelectionDAG;
		1588
		1589	const ConstantInt *Value;
		1590
		1591	ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
		1592	: SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
		1593	getSDVTList(VT)),
		1594	Value(val) {
		1595	ConstantSDNodeBits.IsOpaque = isOpaque;
		1596	}
		1597
		1598	public:
		1599	const ConstantInt *getConstantIntValue() const { return Value; }
		1600	const APInt &getAPIntValue() const { return Value->getValue(); }
		1601	uint64_t getZExtValue() const { return Value->getZExtValue(); }
		1602	int64_t getSExtValue() const { return Value->getSExtValue(); }
		1603	uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
		1604	return Value->getLimitedValue(Limit);
		1605	}
		1606	MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
		1607	Align getAlignValue() const { return Value->getAlignValue(); }
		1608
		1609	bool isOne() const { return Value->isOne(); }
		1610	bool isZero() const { return Value->isZero(); }
		1611	// NOTE: This is soft-deprecated. Please use `isZero()` instead.
		1612	bool isNullValue() const { return isZero(); }
		1613	bool isAllOnes() const { return Value->isMinusOne(); }
		1614	// NOTE: This is soft-deprecated. Please use `isAllOnes()` instead.
		1615	bool isAllOnesValue() const { return isAllOnes(); }
		1616	bool isMaxSignedValue() const { return Value->isMaxValue(true); }
		1617	bool isMinSignedValue() const { return Value->isMinValue(true); }
		1618
		1619	bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
		1620
		1621	static bool classof(const SDNode *N) {
		1622	return N->getOpcode() == ISD::Constant \|\|
		1623	N->getOpcode() == ISD::TargetConstant;
		1624	}
		1625	};
		1626
		1627	uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
		1628	return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
		1629	}
		1630
		1631	const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
		1632	return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
		1633	}
		1634
		1635	class ConstantFPSDNode : public SDNode {
		1636	friend class SelectionDAG;
		1637
		1638	const ConstantFP *Value;
		1639
		1640	ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
		1641	: SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
		1642	DebugLoc(), getSDVTList(VT)),
		1643	Value(val) {}
		1644
		1645	public:
		1646	const APFloat& getValueAPF() const { return Value->getValueAPF(); }
		1647	const ConstantFP *getConstantFPValue() const { return Value; }
		1648
		1649	/// Return true if the value is positive or negative zero.
		1650	bool isZero() const { return Value->isZero(); }
		1651
		1652	/// Return true if the value is a NaN.
		1653	bool isNaN() const { return Value->isNaN(); }
		1654
		1655	/// Return true if the value is an infinity
		1656	bool isInfinity() const { return Value->isInfinity(); }
		1657
		1658	/// Return true if the value is negative.
		1659	bool isNegative() const { return Value->isNegative(); }
		1660
		1661	/// We don't rely on operator== working on double values, as
		1662	/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
		1663	/// As such, this method can be used to do an exact bit-for-bit comparison of
		1664	/// two floating point values.
		1665
		1666	/// We leave the version with the double argument here because it's just so
		1667	/// convenient to write "2.0" and the like. Without this function we'd
		1668	/// have to duplicate its logic everywhere it's called.
		1669	bool isExactlyValue(double V) const {
		1670	return Value->getValueAPF().isExactlyValue(V);
		1671	}
		1672	bool isExactlyValue(const APFloat& V) const;
		1673
		1674	static bool isValueValidForType(EVT VT, const APFloat& Val);
		1675
		1676	static bool classof(const SDNode *N) {
		1677	return N->getOpcode() == ISD::ConstantFP \|\|
		1678	N->getOpcode() == ISD::TargetConstantFP;
		1679	}
		1680	};
		1681
		1682	/// Returns true if \p V is a constant integer zero.
		1683	bool isNullConstant(SDValue V);
		1684
		1685	/// Returns true if \p V is an FP constant with a value of positive zero.
		1686	bool isNullFPConstant(SDValue V);
		1687
		1688	/// Returns true if \p V is an integer constant with all bits set.
		1689	bool isAllOnesConstant(SDValue V);
		1690
		1691	/// Returns true if \p V is a constant integer one.
		1692	bool isOneConstant(SDValue V);
		1693
		1694	/// Returns true if \p V is a constant min signed integer value.
		1695	bool isMinSignedConstant(SDValue V);
		1696
		1697	/// Returns true if \p V is a neutral element of Opc with Flags.
		1698	/// When OperandNo is 0, it checks that V is a left identity. Otherwise, it
		1699	/// checks that V is a right identity.
		1700	bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V,
		1701	unsigned OperandNo);
		1702
		1703	/// Return the non-bitcasted source operand of \p V if it exists.
		1704	/// If \p V is not a bitcasted value, it is returned as-is.
		1705	SDValue peekThroughBitcasts(SDValue V);
		1706
		1707	/// Return the non-bitcasted and one-use source operand of \p V if it exists.
		1708	/// If \p V is not a bitcasted one-use value, it is returned as-is.
		1709	SDValue peekThroughOneUseBitcasts(SDValue V);
		1710
		1711	/// Return the non-extracted vector source operand of \p V if it exists.
		1712	/// If \p V is not an extracted subvector, it is returned as-is.
		1713	SDValue peekThroughExtractSubvectors(SDValue V);
		1714
		1715	/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
		1716	/// constant is canonicalized to be operand 1.
		1717	bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
		1718
		1719	/// If \p V is a bitwise not, returns the inverted operand. Otherwise returns
		1720	/// an empty SDValue. Only bits set in \p Mask are required to be inverted,
		1721	/// other bits may be arbitrary.
		1722	SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs);
		1723
		1724	/// Returns the SDNode if it is a constant splat BuildVector or constant int.
		1725	ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
		1726	bool AllowTruncation = false);
		1727
		1728	/// Returns the SDNode if it is a demanded constant splat BuildVector or
		1729	/// constant int.
		1730	ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
		1731	bool AllowUndefs = false,
		1732	bool AllowTruncation = false);
		1733
		1734	/// Returns the SDNode if it is a constant splat BuildVector or constant float.
		1735	ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
		1736
		1737	/// Returns the SDNode if it is a demanded constant splat BuildVector or
		1738	/// constant float.
		1739	ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
		1740	bool AllowUndefs = false);
		1741
		1742	/// Return true if the value is a constant 0 integer or a splatted vector of
		1743	/// a constant 0 integer (with no undefs by default).
		1744	/// Build vector implicit truncation is not an issue for null values.
		1745	bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
		1746
		1747	/// Return true if the value is a constant 1 integer or a splatted vector of a
		1748	/// constant 1 integer (with no undefs).
		1749	/// Build vector implicit truncation is allowed, but the truncated bits need to
		1750	/// be zero.
		1751	bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
		1752
		1753	/// Return true if the value is a constant -1 integer or a splatted vector of a
		1754	/// constant -1 integer (with no undefs).
		1755	/// Does not permit build vector implicit truncation.
		1756	bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
		1757
		1758	/// Return true if \p V is either a integer or FP constant.
		1759	inline bool isIntOrFPConstant(SDValue V) {
		1760	return isa<ConstantSDNode>(V) \|\| isa<ConstantFPSDNode>(V);
		1761	}
		1762
		1763	class GlobalAddressSDNode : public SDNode {
		1764	friend class SelectionDAG;
		1765
		1766	const GlobalValue *TheGlobal;
		1767	int64_t Offset;
		1768	unsigned TargetFlags;
		1769
		1770	GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
		1771	const GlobalValue *GA, EVT VT, int64_t o,
		1772	unsigned TF);
		1773
		1774	public:
		1775	const GlobalValue *getGlobal() const { return TheGlobal; }
		1776	int64_t getOffset() const { return Offset; }
		1777	unsigned getTargetFlags() const { return TargetFlags; }
		1778	// Return the address space this GlobalAddress belongs to.
		1779	unsigned getAddressSpace() const;
		1780
		1781	static bool classof(const SDNode *N) {
		1782	return N->getOpcode() == ISD::GlobalAddress \|\|
		1783	N->getOpcode() == ISD::TargetGlobalAddress \|\|
		1784	N->getOpcode() == ISD::GlobalTLSAddress \|\|
		1785	N->getOpcode() == ISD::TargetGlobalTLSAddress;
		1786	}
		1787	};
		1788
		1789	class FrameIndexSDNode : public SDNode {
		1790	friend class SelectionDAG;
		1791
		1792	int FI;
		1793
		1794	FrameIndexSDNode(int fi, EVT VT, bool isTarg)
		1795	: SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
		1796	0, DebugLoc(), getSDVTList(VT)), FI(fi) {
		1797	}
		1798
		1799	public:
		1800	int getIndex() const { return FI; }
		1801
		1802	static bool classof(const SDNode *N) {
		1803	return N->getOpcode() == ISD::FrameIndex \|\|
		1804	N->getOpcode() == ISD::TargetFrameIndex;
		1805	}
		1806	};
		1807
		1808	/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
		1809	/// the offet and size that are started/ended in the underlying FrameIndex.
		1810	class LifetimeSDNode : public SDNode {
		1811	friend class SelectionDAG;
		1812	int64_t Size;
		1813	int64_t Offset; // -1 if offset is unknown.
		1814
		1815	LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
		1816	SDVTList VTs, int64_t Size, int64_t Offset)
		1817	: SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
		1818	public:
		1819	int64_t getFrameIndex() const {
		1820	return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
		1821	}
		1822
		1823	bool hasOffset() const { return Offset >= 0; }
		1824	int64_t getOffset() const {
		1825	assert(hasOffset() && "offset is unknown");
		1826	return Offset;
		1827	}
		1828	int64_t getSize() const {
		1829	assert(hasOffset() && "offset is unknown");
		1830	return Size;
		1831	}
		1832
		1833	// Methods to support isa and dyn_cast
		1834	static bool classof(const SDNode *N) {
		1835	return N->getOpcode() == ISD::LIFETIME_START \|\|
		1836	N->getOpcode() == ISD::LIFETIME_END;
		1837	}
		1838	};
		1839
		1840	/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
		1841	/// the index of the basic block being probed. A pseudo probe serves as a place
		1842	/// holder and will be removed at the end of compilation. It does not have any
		1843	/// operand because we do not want the instruction selection to deal with any.
		1844	class PseudoProbeSDNode : public SDNode {
		1845	friend class SelectionDAG;
		1846	uint64_t Guid;
		1847	uint64_t Index;
		1848	uint32_t Attributes;
		1849
		1850	PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
		1851	SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
		1852	: SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
		1853	Attributes(Attr) {}
		1854
		1855	public:
		1856	uint64_t getGuid() const { return Guid; }
		1857	uint64_t getIndex() const { return Index; }
		1858	uint32_t getAttributes() const { return Attributes; }
		1859
		1860	// Methods to support isa and dyn_cast
		1861	static bool classof(const SDNode *N) {
		1862	return N->getOpcode() == ISD::PSEUDO_PROBE;
		1863	}
		1864	};
		1865
		1866	class JumpTableSDNode : public SDNode {
		1867	friend class SelectionDAG;
		1868
		1869	int JTI;
		1870	unsigned TargetFlags;
		1871
		1872	JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
		1873	: SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
		1874	0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
		1875	}
		1876
		1877	public:
		1878	int getIndex() const { return JTI; }
		1879	unsigned getTargetFlags() const { return TargetFlags; }
		1880
		1881	static bool classof(const SDNode *N) {
		1882	return N->getOpcode() == ISD::JumpTable \|\|
		1883	N->getOpcode() == ISD::TargetJumpTable;
		1884	}
		1885	};
		1886
		1887	class ConstantPoolSDNode : public SDNode {
		1888	friend class SelectionDAG;
		1889
		1890	union {
		1891	const Constant *ConstVal;
		1892	MachineConstantPoolValue *MachineCPVal;
		1893	} Val;
		1894	int Offset; // It's a MachineConstantPoolValue if top bit is set.
		1895	Align Alignment; // Minimum alignment requirement of CP.
		1896	unsigned TargetFlags;
		1897
		1898	ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
		1899	Align Alignment, unsigned TF)
		1900	: SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
		1901	DebugLoc(), getSDVTList(VT)),
		1902	Offset(o), Alignment(Alignment), TargetFlags(TF) {
		1903	assert(Offset >= 0 && "Offset is too large");
		1904	Val.ConstVal = c;
		1905	}
		1906
		1907	ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
		1908	Align Alignment, unsigned TF)
		1909	: SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
		1910	DebugLoc(), getSDVTList(VT)),
		1911	Offset(o), Alignment(Alignment), TargetFlags(TF) {
		1912	assert(Offset >= 0 && "Offset is too large");
		1913	Val.MachineCPVal = v;
		1914	Offset \|= 1 << (sizeof(unsigned)*CHAR_BIT-1);
		1915	}
		1916
		1917	public:
		1918	bool isMachineConstantPoolEntry() const {
		1919	return Offset < 0;
		1920	}
		1921
		1922	const Constant *getConstVal() const {
		1923	assert(!isMachineConstantPoolEntry() && "Wrong constantpool type");
		1924	return Val.ConstVal;
		1925	}
		1926
		1927	MachineConstantPoolValue *getMachineCPVal() const {
		1928	assert(isMachineConstantPoolEntry() && "Wrong constantpool type");
		1929	return Val.MachineCPVal;
		1930	}
		1931
		1932	int getOffset() const {
		1933	return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1));
		1934	}
		1935
		1936	// Return the alignment of this constant pool object, which is either 0 (for
		1937	// default alignment) or the desired value.
		1938	Align getAlign() const { return Alignment; }
		1939	unsigned getTargetFlags() const { return TargetFlags; }
		1940
		1941	Type *getType() const;
		1942
		1943	static bool classof(const SDNode *N) {
		1944	return N->getOpcode() == ISD::ConstantPool \|\|
		1945	N->getOpcode() == ISD::TargetConstantPool;
		1946	}
		1947	};
		1948
		1949	/// Completely target-dependent object reference.
		1950	class TargetIndexSDNode : public SDNode {
		1951	friend class SelectionDAG;
		1952
		1953	unsigned TargetFlags;
		1954	int Index;
		1955	int64_t Offset;
		1956
		1957	public:
		1958	TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
		1959	: SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
		1960	TargetFlags(TF), Index(Idx), Offset(Ofs) {}
		1961
		1962	unsigned getTargetFlags() const { return TargetFlags; }
		1963	int getIndex() const { return Index; }
		1964	int64_t getOffset() const { return Offset; }
		1965
		1966	static bool classof(const SDNode *N) {
		1967	return N->getOpcode() == ISD::TargetIndex;
		1968	}
		1969	};
		1970
		1971	class BasicBlockSDNode : public SDNode {
		1972	friend class SelectionDAG;
		1973
		1974	MachineBasicBlock *MBB;
		1975
		1976	/// Debug info is meaningful and potentially useful here, but we create
		1977	/// blocks out of order when they're jumped to, which makes it a bit
		1978	/// harder. Let's see if we need it first.
		1979	explicit BasicBlockSDNode(MachineBasicBlock *mbb)
		1980	: SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
		1981	{}
		1982
		1983	public:
		1984	MachineBasicBlock *getBasicBlock() const { return MBB; }
		1985
		1986	static bool classof(const SDNode *N) {
		1987	return N->getOpcode() == ISD::BasicBlock;
		1988	}
		1989	};
		1990
		1991	/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
		1992	class BuildVectorSDNode : public SDNode {
		1993	public:
		1994	// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
		1995	explicit BuildVectorSDNode() = delete;
		1996
		1997	/// Check if this is a constant splat, and if so, find the
		1998	/// smallest element size that splats the vector. If MinSplatBits is
		1999	/// nonzero, the element size must be at least that large. Note that the
		2000	/// splat element may be the entire vector (i.e., a one element vector).
		2001	/// Returns the splat element value in SplatValue. Any undefined bits in
		2002	/// that value are zero, and the corresponding bits in the SplatUndef mask
		2003	/// are set. The SplatBitSize value is set to the splat element size in
		2004	/// bits. HasAnyUndefs is set to true if any bits in the vector are
		2005	/// undefined. isBigEndian describes the endianness of the target.
		2006	bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
		2007	unsigned &SplatBitSize, bool &HasAnyUndefs,
		2008	unsigned MinSplatBits = 0,
		2009	bool isBigEndian = false) const;
		2010
		2011	/// Returns the demanded splatted value or a null value if this is not a
		2012	/// splat.
		2013	///
		2014	/// The DemandedElts mask indicates the elements that must be in the splat.
		2015	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2016	/// the vector width and set the bits where elements are undef.
		2017	SDValue getSplatValue(const APInt &DemandedElts,
		2018	BitVector *UndefElements = nullptr) const;
		2019
		2020	/// Returns the splatted value or a null value if this is not a splat.
		2021	///
		2022	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2023	/// the vector width and set the bits where elements are undef.
		2024	SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
		2025
		2026	/// Find the shortest repeating sequence of values in the build vector.
		2027	///
		2028	/// e.g. { u, X, u, X, u, u, X, u } -> { X }
		2029	/// { X, Y, u, Y, u, u, X, u } -> { X, Y }
		2030	///
		2031	/// Currently this must be a power-of-2 build vector.
		2032	/// The DemandedElts mask indicates the elements that must be present,
		2033	/// undemanded elements in Sequence may be null (SDValue()). If passed a
		2034	/// non-null UndefElements bitvector, it will resize it to match the original
		2035	/// vector width and set the bits where elements are undef. If result is
		2036	/// false, Sequence will be empty.
		2037	bool getRepeatedSequence(const APInt &DemandedElts,
		2038	SmallVectorImpl<SDValue> &Sequence,
		2039	BitVector *UndefElements = nullptr) const;
		2040
		2041	/// Find the shortest repeating sequence of values in the build vector.
		2042	///
		2043	/// e.g. { u, X, u, X, u, u, X, u } -> { X }
		2044	/// { X, Y, u, Y, u, u, X, u } -> { X, Y }
		2045	///
		2046	/// Currently this must be a power-of-2 build vector.
		2047	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2048	/// the original vector width and set the bits where elements are undef.
		2049	/// If result is false, Sequence will be empty.
		2050	bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
		2051	BitVector *UndefElements = nullptr) const;
		2052
		2053	/// Returns the demanded splatted constant or null if this is not a constant
		2054	/// splat.
		2055	///
		2056	/// The DemandedElts mask indicates the elements that must be in the splat.
		2057	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2058	/// the vector width and set the bits where elements are undef.
		2059	ConstantSDNode *
		2060	getConstantSplatNode(const APInt &DemandedElts,
		2061	BitVector *UndefElements = nullptr) const;
		2062
		2063	/// Returns the splatted constant or null if this is not a constant
		2064	/// splat.
		2065	///
		2066	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2067	/// the vector width and set the bits where elements are undef.
		2068	ConstantSDNode *
		2069	getConstantSplatNode(BitVector *UndefElements = nullptr) const;
		2070
		2071	/// Returns the demanded splatted constant FP or null if this is not a
		2072	/// constant FP splat.
		2073	///
		2074	/// The DemandedElts mask indicates the elements that must be in the splat.
		2075	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2076	/// the vector width and set the bits where elements are undef.
		2077	ConstantFPSDNode *
		2078	getConstantFPSplatNode(const APInt &DemandedElts,
		2079	BitVector *UndefElements = nullptr) const;
		2080
		2081	/// Returns the splatted constant FP or null if this is not a constant
		2082	/// FP splat.
		2083	///
		2084	/// If passed a non-null UndefElements bitvector, it will resize it to match
		2085	/// the vector width and set the bits where elements are undef.
		2086	ConstantFPSDNode *
		2087	getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
		2088
		2089	/// If this is a constant FP splat and the splatted constant FP is an
		2090	/// exact power or 2, return the log base 2 integer value. Otherwise,
		2091	/// return -1.
		2092	///
		2093	/// The BitWidth specifies the necessary bit precision.
		2094	int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
		2095	uint32_t BitWidth) const;
		2096
		2097	/// Extract the raw bit data from a build vector of Undef, Constant or
		2098	/// ConstantFP node elements. Each raw bit element will be \p
		2099	/// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
		2100	/// undefined elements are flagged in \p UndefElements.
		2101	bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
		2102	SmallVectorImpl<APInt> &RawBitElements,
		2103	BitVector &UndefElements) const;
		2104
		2105	bool isConstant() const;
		2106
		2107	/// If this BuildVector is constant and represents the numerical series
		2108	/// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer,
		2109	/// the value "<a,n>" is returned.
		2110	std::optional<std::pair<APInt, APInt>> isConstantSequence() const;
		2111
		2112	/// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
		2113	/// Undef elements are treated as zero, and entirely undefined elements are
		2114	/// flagged in \p DstUndefElements.
		2115	static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
		2116	SmallVectorImpl<APInt> &DstBitElements,
		2117	ArrayRef<APInt> SrcBitElements,
		2118	BitVector &DstUndefElements,
		2119	const BitVector &SrcUndefElements);
		2120
		2121	static bool classof(const SDNode *N) {
		2122	return N->getOpcode() == ISD::BUILD_VECTOR;
		2123	}
		2124	};
		2125
		2126	/// An SDNode that holds an arbitrary LLVM IR Value. This is
		2127	/// used when the SelectionDAG needs to make a simple reference to something
		2128	/// in the LLVM IR representation.
		2129	///
		2130	class SrcValueSDNode : public SDNode {
		2131	friend class SelectionDAG;
		2132
		2133	const Value *V;
		2134
		2135	/// Create a SrcValue for a general value.
		2136	explicit SrcValueSDNode(const Value *v)
		2137	: SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
		2138
		2139	public:
		2140	/// Return the contained Value.
		2141	const Value *getValue() const { return V; }
		2142
		2143	static bool classof(const SDNode *N) {
		2144	return N->getOpcode() == ISD::SRCVALUE;
		2145	}
		2146	};
		2147
		2148	class MDNodeSDNode : public SDNode {
		2149	friend class SelectionDAG;
		2150
		2151	const MDNode *MD;
		2152
		2153	explicit MDNodeSDNode(const MDNode *md)
		2154	: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
		2155	{}
		2156
		2157	public:
		2158	const MDNode *getMD() const { return MD; }
		2159
		2160	static bool classof(const SDNode *N) {
		2161	return N->getOpcode() == ISD::MDNODE_SDNODE;
		2162	}
		2163	};
		2164
		2165	class RegisterSDNode : public SDNode {
		2166	friend class SelectionDAG;
		2167
		2168	Register Reg;
		2169
		2170	RegisterSDNode(Register reg, EVT VT)
		2171	: SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
		2172
		2173	public:
		2174	Register getReg() const { return Reg; }
		2175
		2176	static bool classof(const SDNode *N) {
		2177	return N->getOpcode() == ISD::Register;
		2178	}
		2179	};
		2180
		2181	class RegisterMaskSDNode : public SDNode {
		2182	friend class SelectionDAG;
		2183
		2184	// The memory for RegMask is not owned by the node.
		2185	const uint32_t *RegMask;
		2186
		2187	RegisterMaskSDNode(const uint32_t *mask)
		2188	: SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
		2189	RegMask(mask) {}
		2190
		2191	public:
		2192	const uint32_t *getRegMask() const { return RegMask; }
		2193
		2194	static bool classof(const SDNode *N) {
		2195	return N->getOpcode() == ISD::RegisterMask;
		2196	}
		2197	};
		2198
		2199	class BlockAddressSDNode : public SDNode {
		2200	friend class SelectionDAG;
		2201
		2202	const BlockAddress *BA;
		2203	int64_t Offset;
		2204	unsigned TargetFlags;
		2205
		2206	BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
		2207	int64_t o, unsigned Flags)
		2208	: SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
		2209	BA(ba), Offset(o), TargetFlags(Flags) {}
		2210
		2211	public:
		2212	const BlockAddress *getBlockAddress() const { return BA; }
		2213	int64_t getOffset() const { return Offset; }
		2214	unsigned getTargetFlags() const { return TargetFlags; }
		2215
		2216	static bool classof(const SDNode *N) {
		2217	return N->getOpcode() == ISD::BlockAddress \|\|
		2218	N->getOpcode() == ISD::TargetBlockAddress;
		2219	}
		2220	};
		2221
		2222	class LabelSDNode : public SDNode {
		2223	friend class SelectionDAG;
		2224
		2225	MCSymbol *Label;
		2226
		2227	LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
		2228	: SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
		2229	assert(LabelSDNode::classof(this) && "not a label opcode");
		2230	}
		2231
		2232	public:
		2233	MCSymbol *getLabel() const { return Label; }
		2234
		2235	static bool classof(const SDNode *N) {
		2236	return N->getOpcode() == ISD::EH_LABEL \|\|
		2237	N->getOpcode() == ISD::ANNOTATION_LABEL;
		2238	}
		2239	};
		2240
		2241	class ExternalSymbolSDNode : public SDNode {
		2242	friend class SelectionDAG;
		2243
		2244	const char *Symbol;
		2245	unsigned TargetFlags;
		2246
		2247	ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
		2248	: SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
		2249	DebugLoc(), getSDVTList(VT)),
		2250	Symbol(Sym), TargetFlags(TF) {}
		2251
		2252	public:
		2253	const char *getSymbol() const { return Symbol; }
		2254	unsigned getTargetFlags() const { return TargetFlags; }
		2255
		2256	static bool classof(const SDNode *N) {
		2257	return N->getOpcode() == ISD::ExternalSymbol \|\|
		2258	N->getOpcode() == ISD::TargetExternalSymbol;
		2259	}
		2260	};
		2261
		2262	class MCSymbolSDNode : public SDNode {
		2263	friend class SelectionDAG;
		2264
		2265	MCSymbol *Symbol;
		2266
		2267	MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
		2268	: SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
		2269
		2270	public:
		2271	MCSymbol *getMCSymbol() const { return Symbol; }
		2272
		2273	static bool classof(const SDNode *N) {
		2274	return N->getOpcode() == ISD::MCSymbol;
		2275	}
		2276	};
		2277
		2278	class CondCodeSDNode : public SDNode {
		2279	friend class SelectionDAG;
		2280
		2281	ISD::CondCode Condition;
		2282
		2283	explicit CondCodeSDNode(ISD::CondCode Cond)
		2284	: SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
		2285	Condition(Cond) {}
		2286
		2287	public:
		2288	ISD::CondCode get() const { return Condition; }
		2289
		2290	static bool classof(const SDNode *N) {
		2291	return N->getOpcode() == ISD::CONDCODE;
		2292	}
		2293	};
		2294
		2295	/// This class is used to represent EVT's, which are used
		2296	/// to parameterize some operations.
		2297	class VTSDNode : public SDNode {
		2298	friend class SelectionDAG;
		2299
		2300	EVT ValueType;
		2301
		2302	explicit VTSDNode(EVT VT)
		2303	: SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
		2304	ValueType(VT) {}
		2305
		2306	public:
		2307	EVT getVT() const { return ValueType; }
		2308
		2309	static bool classof(const SDNode *N) {
		2310	return N->getOpcode() == ISD::VALUETYPE;
		2311	}
		2312	};
		2313
		2314	/// Base class for LoadSDNode and StoreSDNode
		2315	class LSBaseSDNode : public MemSDNode {
		2316	public:
		2317	LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
		2318	SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
		2319	MachineMemOperand *MMO)
		2320	: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
		2321	LSBaseSDNodeBits.AddressingMode = AM;
		2322	assert(getAddressingMode() == AM && "Value truncated");
		2323	}
		2324
		2325	const SDValue &getOffset() const {
		2326	return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
		2327	}
		2328
		2329	/// Return the addressing mode for this load or store:
		2330	/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
		2331	ISD::MemIndexedMode getAddressingMode() const {
		2332	return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
		2333	}
		2334
		2335	/// Return true if this is a pre/post inc/dec load/store.
		2336	bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
		2337
		2338	/// Return true if this is NOT a pre/post inc/dec load/store.
		2339	bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
		2340
		2341	static bool classof(const SDNode *N) {
		2342	return N->getOpcode() == ISD::LOAD \|\|
		2343	N->getOpcode() == ISD::STORE;
		2344	}
		2345	};
		2346
		2347	/// This class is used to represent ISD::LOAD nodes.
		2348	class LoadSDNode : public LSBaseSDNode {
		2349	friend class SelectionDAG;
		2350
		2351	LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2352	ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
		2353	MachineMemOperand *MMO)
		2354	: LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
		2355	LoadSDNodeBits.ExtTy = ETy;
		2356	assert(readMem() && "Load MachineMemOperand is not a load!");
		2357	assert(!writeMem() && "Load MachineMemOperand is a store!");
		2358	}
		2359
		2360	public:
		2361	/// Return whether this is a plain node,
		2362	/// or one of the varieties of value-extending loads.
		2363	ISD::LoadExtType getExtensionType() const {
		2364	return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
		2365	}
		2366
		2367	const SDValue &getBasePtr() const { return getOperand(1); }
		2368	const SDValue &getOffset() const { return getOperand(2); }
		2369
		2370	static bool classof(const SDNode *N) {
		2371	return N->getOpcode() == ISD::LOAD;
		2372	}
		2373	};
		2374
		2375	/// This class is used to represent ISD::STORE nodes.
		2376	class StoreSDNode : public LSBaseSDNode {
		2377	friend class SelectionDAG;
		2378
		2379	StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2380	ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
		2381	MachineMemOperand *MMO)
		2382	: LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
		2383	StoreSDNodeBits.IsTruncating = isTrunc;
		2384	assert(!readMem() && "Store MachineMemOperand is a load!");
		2385	assert(writeMem() && "Store MachineMemOperand is not a store!");
		2386	}
		2387
		2388	public:
		2389	/// Return true if the op does a truncation before store.
		2390	/// For integers this is the same as doing a TRUNCATE and storing the result.
		2391	/// For floats, it is the same as doing an FP_ROUND and storing the result.
		2392	bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
		2393	void setTruncatingStore(bool Truncating) {
		2394	StoreSDNodeBits.IsTruncating = Truncating;
		2395	}
		2396
		2397	const SDValue &getValue() const { return getOperand(1); }
		2398	const SDValue &getBasePtr() const { return getOperand(2); }
		2399	const SDValue &getOffset() const { return getOperand(3); }
		2400
		2401	static bool classof(const SDNode *N) {
		2402	return N->getOpcode() == ISD::STORE;
		2403	}
		2404	};
		2405
		2406	/// This base class is used to represent VP_LOAD, VP_STORE,
		2407	/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
		2408	class VPBaseLoadStoreSDNode : public MemSDNode {
		2409	public:
		2410	friend class SelectionDAG;
		2411
		2412	VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
		2413	const DebugLoc &DL, SDVTList VTs,
		2414	ISD::MemIndexedMode AM, EVT MemVT,
		2415	MachineMemOperand *MMO)
		2416	: MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
		2417	LSBaseSDNodeBits.AddressingMode = AM;
		2418	assert(getAddressingMode() == AM && "Value truncated");
		2419	}
		2420
		2421	// VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL)
		2422	// VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
		2423	// VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL)
		2424	// VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
		2425	// Mask is a vector of i1 elements;
		2426	// the type of EVL is TLI.getVPExplicitVectorLengthTy().
		2427	const SDValue &getOffset() const {
		2428	return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD \|\|
		2429	getOpcode() == ISD::VP_LOAD)
		2430	? 2
		2431	: 3);
		2432	}
		2433	const SDValue &getBasePtr() const {
		2434	return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD \|\|
		2435	getOpcode() == ISD::VP_LOAD)
		2436	? 1
		2437	: 2);
		2438	}
		2439	const SDValue &getMask() const {
		2440	switch (getOpcode()) {
		2441	default:
		2442	llvm_unreachable("Invalid opcode");
		2443	case ISD::VP_LOAD:
		2444	return getOperand(3);
		2445	case ISD::VP_STORE:
		2446	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
		2447	return getOperand(4);
		2448	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
		2449	return getOperand(5);
		2450	}
		2451	}
		2452	const SDValue &getVectorLength() const {
		2453	switch (getOpcode()) {
		2454	default:
		2455	llvm_unreachable("Invalid opcode");
		2456	case ISD::VP_LOAD:
		2457	return getOperand(4);
		2458	case ISD::VP_STORE:
		2459	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
		2460	return getOperand(5);
		2461	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
		2462	return getOperand(6);
		2463	}
		2464	}
		2465
		2466	/// Return the addressing mode for this load or store:
		2467	/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
		2468	ISD::MemIndexedMode getAddressingMode() const {
		2469	return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
		2470	}
		2471
		2472	/// Return true if this is a pre/post inc/dec load/store.
		2473	bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
		2474
		2475	/// Return true if this is NOT a pre/post inc/dec load/store.
		2476	bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
		2477
		2478	static bool classof(const SDNode *N) {
		2479	return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD \|\|
		2480	N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE \|\|
		2481	N->getOpcode() == ISD::VP_LOAD \|\| N->getOpcode() == ISD::VP_STORE;
		2482	}
		2483	};
		2484
		2485	/// This class is used to represent a VP_LOAD node
		2486	class VPLoadSDNode : public VPBaseLoadStoreSDNode {
		2487	public:
		2488	friend class SelectionDAG;
		2489
		2490	VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2491	ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
		2492	EVT MemVT, MachineMemOperand *MMO)
		2493	: VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
		2494	LoadSDNodeBits.ExtTy = ETy;
		2495	LoadSDNodeBits.IsExpanding = isExpanding;
		2496	}
		2497
		2498	ISD::LoadExtType getExtensionType() const {
		2499	return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
		2500	}
		2501
		2502	const SDValue &getBasePtr() const { return getOperand(1); }
		2503	const SDValue &getOffset() const { return getOperand(2); }
		2504	const SDValue &getMask() const { return getOperand(3); }
		2505	const SDValue &getVectorLength() const { return getOperand(4); }
		2506
		2507	static bool classof(const SDNode *N) {
		2508	return N->getOpcode() == ISD::VP_LOAD;
		2509	}
		2510	bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
		2511	};
		2512
		2513	/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
		2514	class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
		2515	public:
		2516	friend class SelectionDAG;
		2517
		2518	VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
		2519	ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
		2520	bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
		2521	: VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
		2522	AM, MemVT, MMO) {
		2523	LoadSDNodeBits.ExtTy = ETy;
		2524	LoadSDNodeBits.IsExpanding = IsExpanding;
		2525	}
		2526
		2527	ISD::LoadExtType getExtensionType() const {
		2528	return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
		2529	}
		2530
		2531	const SDValue &getBasePtr() const { return getOperand(1); }
		2532	const SDValue &getOffset() const { return getOperand(2); }
		2533	const SDValue &getStride() const { return getOperand(3); }
		2534	const SDValue &getMask() const { return getOperand(4); }
		2535	const SDValue &getVectorLength() const { return getOperand(5); }
		2536
		2537	static bool classof(const SDNode *N) {
		2538	return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
		2539	}
		2540	bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
		2541	};
		2542
		2543	/// This class is used to represent a VP_STORE node
		2544	class VPStoreSDNode : public VPBaseLoadStoreSDNode {
		2545	public:
		2546	friend class SelectionDAG;
		2547
		2548	VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2549	ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
		2550	EVT MemVT, MachineMemOperand *MMO)
		2551	: VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
		2552	StoreSDNodeBits.IsTruncating = isTrunc;
		2553	StoreSDNodeBits.IsCompressing = isCompressing;
		2554	}
		2555
		2556	/// Return true if this is a truncating store.
		2557	/// For integers this is the same as doing a TRUNCATE and storing the result.
		2558	/// For floats, it is the same as doing an FP_ROUND and storing the result.
		2559	bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
		2560
		2561	/// Returns true if the op does a compression to the vector before storing.
		2562	/// The node contiguously stores the active elements (integers or floats)
		2563	/// in src (those with their respective bit set in writemask k) to unaligned
		2564	/// memory at base_addr.
		2565	bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
		2566
		2567	const SDValue &getValue() const { return getOperand(1); }
		2568	const SDValue &getBasePtr() const { return getOperand(2); }
		2569	const SDValue &getOffset() const { return getOperand(3); }
		2570	const SDValue &getMask() const { return getOperand(4); }
		2571	const SDValue &getVectorLength() const { return getOperand(5); }
		2572
		2573	static bool classof(const SDNode *N) {
		2574	return N->getOpcode() == ISD::VP_STORE;
		2575	}
		2576	};
		2577
		2578	/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
		2579	class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
		2580	public:
		2581	friend class SelectionDAG;
		2582
		2583	VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
		2584	ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
		2585	EVT MemVT, MachineMemOperand *MMO)
		2586	: VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
		2587	VTs, AM, MemVT, MMO) {
		2588	StoreSDNodeBits.IsTruncating = IsTrunc;
		2589	StoreSDNodeBits.IsCompressing = IsCompressing;
		2590	}
		2591
		2592	/// Return true if this is a truncating store.
		2593	/// For integers this is the same as doing a TRUNCATE and storing the result.
		2594	/// For floats, it is the same as doing an FP_ROUND and storing the result.
		2595	bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
		2596
		2597	/// Returns true if the op does a compression to the vector before storing.
		2598	/// The node contiguously stores the active elements (integers or floats)
		2599	/// in src (those with their respective bit set in writemask k) to unaligned
		2600	/// memory at base_addr.
		2601	bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
		2602
		2603	const SDValue &getValue() const { return getOperand(1); }
		2604	const SDValue &getBasePtr() const { return getOperand(2); }
		2605	const SDValue &getOffset() const { return getOperand(3); }
		2606	const SDValue &getStride() const { return getOperand(4); }
		2607	const SDValue &getMask() const { return getOperand(5); }
		2608	const SDValue &getVectorLength() const { return getOperand(6); }
		2609
		2610	static bool classof(const SDNode *N) {
		2611	return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
		2612	}
		2613	};
		2614
		2615	/// This base class is used to represent MLOAD and MSTORE nodes
		2616	class MaskedLoadStoreSDNode : public MemSDNode {
		2617	public:
		2618	friend class SelectionDAG;
		2619
		2620	MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
		2621	const DebugLoc &dl, SDVTList VTs,
		2622	ISD::MemIndexedMode AM, EVT MemVT,
		2623	MachineMemOperand *MMO)
		2624	: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
		2625	LSBaseSDNodeBits.AddressingMode = AM;
		2626	assert(getAddressingMode() == AM && "Value truncated");
		2627	}
		2628
		2629	// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
		2630	// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
		2631	// Mask is a vector of i1 elements
		2632	const SDValue &getOffset() const {
		2633	return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
		2634	}
		2635	const SDValue &getMask() const {
		2636	return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
		2637	}
		2638
		2639	/// Return the addressing mode for this load or store:
		2640	/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
		2641	ISD::MemIndexedMode getAddressingMode() const {
		2642	return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
		2643	}
		2644
		2645	/// Return true if this is a pre/post inc/dec load/store.
		2646	bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
		2647
		2648	/// Return true if this is NOT a pre/post inc/dec load/store.
		2649	bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
		2650
		2651	static bool classof(const SDNode *N) {
		2652	return N->getOpcode() == ISD::MLOAD \|\|
		2653	N->getOpcode() == ISD::MSTORE;
		2654	}
		2655	};
		2656
		2657	/// This class is used to represent an MLOAD node
		2658	class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
		2659	public:
		2660	friend class SelectionDAG;
		2661
		2662	MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2663	ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
		2664	bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
		2665	: MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
		2666	LoadSDNodeBits.ExtTy = ETy;
		2667	LoadSDNodeBits.IsExpanding = IsExpanding;
		2668	}
		2669
		2670	ISD::LoadExtType getExtensionType() const {
		2671	return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
		2672	}
		2673
		2674	const SDValue &getBasePtr() const { return getOperand(1); }
		2675	const SDValue &getOffset() const { return getOperand(2); }
		2676	const SDValue &getMask() const { return getOperand(3); }
		2677	const SDValue &getPassThru() const { return getOperand(4); }
		2678
		2679	static bool classof(const SDNode *N) {
		2680	return N->getOpcode() == ISD::MLOAD;
		2681	}
		2682
		2683	bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
		2684	};
		2685
		2686	/// This class is used to represent an MSTORE node
		2687	class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
		2688	public:
		2689	friend class SelectionDAG;
		2690
		2691	MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2692	ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
		2693	EVT MemVT, MachineMemOperand *MMO)
		2694	: MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
		2695	StoreSDNodeBits.IsTruncating = isTrunc;
		2696	StoreSDNodeBits.IsCompressing = isCompressing;
		2697	}
		2698
		2699	/// Return true if the op does a truncation before store.
		2700	/// For integers this is the same as doing a TRUNCATE and storing the result.
		2701	/// For floats, it is the same as doing an FP_ROUND and storing the result.
		2702	bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
		2703
		2704	/// Returns true if the op does a compression to the vector before storing.
		2705	/// The node contiguously stores the active elements (integers or floats)
		2706	/// in src (those with their respective bit set in writemask k) to unaligned
		2707	/// memory at base_addr.
		2708	bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
		2709
		2710	const SDValue &getValue() const { return getOperand(1); }
		2711	const SDValue &getBasePtr() const { return getOperand(2); }
		2712	const SDValue &getOffset() const { return getOperand(3); }
		2713	const SDValue &getMask() const { return getOperand(4); }
		2714
		2715	static bool classof(const SDNode *N) {
		2716	return N->getOpcode() == ISD::MSTORE;
		2717	}
		2718	};
		2719
		2720	/// This is a base class used to represent
		2721	/// VP_GATHER and VP_SCATTER nodes
		2722	///
		2723	class VPGatherScatterSDNode : public MemSDNode {
		2724	public:
		2725	friend class SelectionDAG;
		2726
		2727	VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
		2728	const DebugLoc &dl, SDVTList VTs, EVT MemVT,
		2729	MachineMemOperand *MMO, ISD::MemIndexType IndexType)
		2730	: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
		2731	LSBaseSDNodeBits.AddressingMode = IndexType;
		2732	assert(getIndexType() == IndexType && "Value truncated");
		2733	}
		2734
		2735	/// How is Index applied to BasePtr when computing addresses.
		2736	ISD::MemIndexType getIndexType() const {
		2737	return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
		2738	}
		2739	bool isIndexScaled() const {
		2740	return !cast<ConstantSDNode>(getScale())->isOne();
		2741	}
		2742	bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }
		2743
		2744	// In the both nodes address is Op1, mask is Op2:
		2745	// VPGatherSDNode (Chain, base, index, scale, mask, vlen)
		2746	// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
		2747	// Mask is a vector of i1 elements
		2748	const SDValue &getBasePtr() const {
		2749	return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
		2750	}
		2751	const SDValue &getIndex() const {
		2752	return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
		2753	}
		2754	const SDValue &getScale() const {
		2755	return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
		2756	}
		2757	const SDValue &getMask() const {
		2758	return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
		2759	}
		2760	const SDValue &getVectorLength() const {
		2761	return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
		2762	}
		2763
		2764	static bool classof(const SDNode *N) {
		2765	return N->getOpcode() == ISD::VP_GATHER \|\|
		2766	N->getOpcode() == ISD::VP_SCATTER;
		2767	}
		2768	};
		2769
		2770	/// This class is used to represent an VP_GATHER node
		2771	///
		2772	class VPGatherSDNode : public VPGatherScatterSDNode {
		2773	public:
		2774	friend class SelectionDAG;
		2775
		2776	VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
		2777	MachineMemOperand *MMO, ISD::MemIndexType IndexType)
		2778	: VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
		2779	IndexType) {}
		2780
		2781	static bool classof(const SDNode *N) {
		2782	return N->getOpcode() == ISD::VP_GATHER;
		2783	}
		2784	};
		2785
		2786	/// This class is used to represent an VP_SCATTER node
		2787	///
		2788	class VPScatterSDNode : public VPGatherScatterSDNode {
		2789	public:
		2790	friend class SelectionDAG;
		2791
		2792	VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
		2793	MachineMemOperand *MMO, ISD::MemIndexType IndexType)
		2794	: VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
		2795	IndexType) {}
		2796
		2797	const SDValue &getValue() const { return getOperand(1); }
		2798
		2799	static bool classof(const SDNode *N) {
		2800	return N->getOpcode() == ISD::VP_SCATTER;
		2801	}
		2802	};
		2803
		2804	/// This is a base class used to represent
		2805	/// MGATHER and MSCATTER nodes
		2806	///
		2807	class MaskedGatherScatterSDNode : public MemSDNode {
		2808	public:
		2809	friend class SelectionDAG;
		2810
		2811	MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
		2812	const DebugLoc &dl, SDVTList VTs, EVT MemVT,
		2813	MachineMemOperand *MMO, ISD::MemIndexType IndexType)
		2814	: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
		2815	LSBaseSDNodeBits.AddressingMode = IndexType;
		2816	assert(getIndexType() == IndexType && "Value truncated");
		2817	}
		2818
		2819	/// How is Index applied to BasePtr when computing addresses.
		2820	ISD::MemIndexType getIndexType() const {
		2821	return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
		2822	}
		2823	bool isIndexScaled() const {
		2824	return !cast<ConstantSDNode>(getScale())->isOne();
		2825	}
		2826	bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }
		2827
		2828	// In the both nodes address is Op1, mask is Op2:
		2829	// MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
		2830	// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
		2831	// Mask is a vector of i1 elements
		2832	const SDValue &getBasePtr() const { return getOperand(3); }
		2833	const SDValue &getIndex() const { return getOperand(4); }
		2834	const SDValue &getMask() const { return getOperand(2); }
		2835	const SDValue &getScale() const { return getOperand(5); }
		2836
		2837	static bool classof(const SDNode *N) {
		2838	return N->getOpcode() == ISD::MGATHER \|\|
		2839	N->getOpcode() == ISD::MSCATTER;
		2840	}
		2841	};
		2842
		2843	/// This class is used to represent an MGATHER node
		2844	///
		2845	class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
		2846	public:
		2847	friend class SelectionDAG;
		2848
		2849	MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2850	EVT MemVT, MachineMemOperand *MMO,
		2851	ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
		2852	: MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
		2853	IndexType) {
		2854	LoadSDNodeBits.ExtTy = ETy;
		2855	}
		2856
		2857	const SDValue &getPassThru() const { return getOperand(1); }
		2858
		2859	ISD::LoadExtType getExtensionType() const {
		2860	return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
		2861	}
		2862
		2863	static bool classof(const SDNode *N) {
		2864	return N->getOpcode() == ISD::MGATHER;
		2865	}
		2866	};
		2867
		2868	/// This class is used to represent an MSCATTER node
		2869	///
		2870	class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
		2871	public:
		2872	friend class SelectionDAG;
		2873
		2874	MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
		2875	EVT MemVT, MachineMemOperand *MMO,
		2876	ISD::MemIndexType IndexType, bool IsTrunc)
		2877	: MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
		2878	IndexType) {
		2879	StoreSDNodeBits.IsTruncating = IsTrunc;
		2880	}
		2881
		2882	/// Return true if the op does a truncation before store.
		2883	/// For integers this is the same as doing a TRUNCATE and storing the result.
		2884	/// For floats, it is the same as doing an FP_ROUND and storing the result.
		2885	bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
		2886
		2887	const SDValue &getValue() const { return getOperand(1); }
		2888
		2889	static bool classof(const SDNode *N) {
		2890	return N->getOpcode() == ISD::MSCATTER;
		2891	}
		2892	};
		2893
		2894	/// An SDNode that represents everything that will be needed
		2895	/// to construct a MachineInstr. These nodes are created during the
		2896	/// instruction selection proper phase.
		2897	///
		2898	/// Note that the only supported way to set the `memoperands` is by calling the
		2899	/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
		2900	/// inside the DAG rather than in the node.
		2901	class MachineSDNode : public SDNode {
		2902	private:
		2903	friend class SelectionDAG;
		2904
		2905	MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
		2906	: SDNode(Opc, Order, DL, VTs) {}
		2907
		2908	// We use a pointer union between a single `MachineMemOperand` pointer and
		2909	// a pointer to an array of `MachineMemOperand` pointers. This is null when
		2910	// the number of these is zero, the single pointer variant used when the
		2911	// number is one, and the array is used for larger numbers.
		2912	//
		2913	// The array is allocated via the `SelectionDAG`'s allocator and so will
		2914	// always live until the DAG is cleaned up and doesn't require ownership here.
		2915	//
		2916	// We can't use something simpler like `TinyPtrVector` here because `SDNode`
		2917	// subclasses aren't managed in a conforming C++ manner. See the comments on
		2918	// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
		2919	// constraint here is that these don't manage memory with their constructor or
		2920	// destructor and can be initialized to a good state even if they start off
		2921	// uninitialized.
		2922	PointerUnion<MachineMemOperand , MachineMemOperand *> MemRefs = {};
		2923
		2924	// Note that this could be folded into the above `MemRefs` member if doing so
		2925	// is advantageous at some point. We don't need to store this in most cases.
		2926	// However, at the moment this doesn't appear to make the allocation any
		2927	// smaller and makes the code somewhat simpler to read.
		2928	int NumMemRefs = 0;
		2929
		2930	public:
		2931	using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
		2932
		2933	ArrayRef<MachineMemOperand *> memoperands() const {
		2934	// Special case the common cases.
		2935	if (NumMemRefs == 0)
		2936	return {};
		2937	if (NumMemRefs == 1)
		2938	return ArrayRef(MemRefs.getAddrOfPtr1(), 1);
		2939
		2940	// Otherwise we have an actual array.
		2941	return ArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
		2942	}
		2943	mmo_iterator memoperands_begin() const { return memoperands().begin(); }
		2944	mmo_iterator memoperands_end() const { return memoperands().end(); }
		2945	bool memoperands_empty() const { return memoperands().empty(); }
		2946
		2947	/// Clear out the memory reference descriptor list.
		2948	void clearMemRefs() {
		2949	MemRefs = nullptr;
		2950	NumMemRefs = 0;
		2951	}
		2952
		2953	static bool classof(const SDNode *N) {
		2954	return N->isMachineOpcode();
		2955	}
		2956	};
		2957
		2958	/// An SDNode that records if a register contains a value that is guaranteed to
		2959	/// be aligned accordingly.
		2960	class AssertAlignSDNode : public SDNode {
		2961	Align Alignment;
		2962
		2963	public:
		2964	AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
		2965	: SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
		2966
		2967	Align getAlign() const { return Alignment; }
		2968
		2969	static bool classof(const SDNode *N) {
		2970	return N->getOpcode() == ISD::AssertAlign;
		2971	}
		2972	};
		2973
		2974	class SDNodeIterator {
		2975	const SDNode *Node;
		2976	unsigned Operand;
		2977
		2978	SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
		2979
		2980	public:
		2981	using iterator_category = std::forward_iterator_tag;
		2982	using value_type = SDNode;
		2983	using difference_type = std::ptrdiff_t;
		2984	using pointer = value_type *;
		2985	using reference = value_type &;
		2986
		2987	bool operator==(const SDNodeIterator& x) const {
		2988	return Operand == x.Operand;
		2989	}
		2990	bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
		2991
		2992	pointer operator*() const {
		2993	return Node->getOperand(Operand).getNode();
		2994	}
		2995	pointer operator->() const { return operator*(); }
		2996
		2997	SDNodeIterator& operator++() { // Preincrement
		2998	++Operand;
		2999	return *this;
		3000	}
		3001	SDNodeIterator operator++(int) { // Postincrement
		3002	SDNodeIterator tmp = this; ++this; return tmp;
		3003	}
		3004	size_t operator-(SDNodeIterator Other) const {
		3005	assert(Node == Other.Node &&
		3006	"Cannot compare iterators of two different nodes!");
		3007	return Operand - Other.Operand;
		3008	}
		3009
		3010	static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
		3011	static SDNodeIterator end (const SDNode *N) {
		3012	return SDNodeIterator(N, N->getNumOperands());
		3013	}
		3014
		3015	unsigned getOperand() const { return Operand; }
		3016	const SDNode *getNode() const { return Node; }
		3017	};
		3018
		3019	template <> struct GraphTraits<SDNode*> {
		3020	using NodeRef = SDNode *;
		3021	using ChildIteratorType = SDNodeIterator;
		3022
		3023	static NodeRef getEntryNode(SDNode *N) { return N; }
		3024
		3025	static ChildIteratorType child_begin(NodeRef N) {
		3026	return SDNodeIterator::begin(N);
		3027	}
		3028
		3029	static ChildIteratorType child_end(NodeRef N) {
		3030	return SDNodeIterator::end(N);
		3031	}
		3032	};
		3033
		3034	/// A representation of the largest SDNode, for use in sizeof().
		3035	///
		3036	/// This needs to be a union because the largest node differs on 32 bit systems
		3037	/// with 4 and 8 byte pointer alignment, respectively.
		3038	using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
		3039	BlockAddressSDNode,
		3040	GlobalAddressSDNode,
		3041	PseudoProbeSDNode>;
		3042
		3043	/// The SDNode class with the greatest alignment requirement.
		3044	using MostAlignedSDNode = GlobalAddressSDNode;
		3045
		3046	namespace ISD {
		3047
		3048	/// Returns true if the specified node is a non-extending and unindexed load.
		3049	inline bool isNormalLoad(const SDNode *N) {
		3050	const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
		3051	return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
		3052	Ld->getAddressingMode() == ISD::UNINDEXED;
		3053	}
		3054
		3055	/// Returns true if the specified node is a non-extending load.
		3056	inline bool isNON_EXTLoad(const SDNode *N) {
		3057	return isa<LoadSDNode>(N) &&
		3058	cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
		3059	}
		3060
		3061	/// Returns true if the specified node is a EXTLOAD.
		3062	inline bool isEXTLoad(const SDNode *N) {
		3063	return isa<LoadSDNode>(N) &&
		3064	cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
		3065	}
		3066
		3067	/// Returns true if the specified node is a SEXTLOAD.
		3068	inline bool isSEXTLoad(const SDNode *N) {
		3069	return isa<LoadSDNode>(N) &&
		3070	cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
		3071	}
		3072
		3073	/// Returns true if the specified node is a ZEXTLOAD.
		3074	inline bool isZEXTLoad(const SDNode *N) {
		3075	return isa<LoadSDNode>(N) &&
		3076	cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
		3077	}
		3078
		3079	/// Returns true if the specified node is an unindexed load.
		3080	inline bool isUNINDEXEDLoad(const SDNode *N) {
		3081	return isa<LoadSDNode>(N) &&
		3082	cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
		3083	}
		3084
		3085	/// Returns true if the specified node is a non-truncating
		3086	/// and unindexed store.
		3087	inline bool isNormalStore(const SDNode *N) {
		3088	const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
		3089	return St && !St->isTruncatingStore() &&
		3090	St->getAddressingMode() == ISD::UNINDEXED;
		3091	}
		3092
		3093	/// Returns true if the specified node is an unindexed store.
		3094	inline bool isUNINDEXEDStore(const SDNode *N) {
		3095	return isa<StoreSDNode>(N) &&
		3096	cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
		3097	}
		3098
		3099	/// Attempt to match a unary predicate against a scalar/splat constant or
		3100	/// every element of a constant BUILD_VECTOR.
		3101	/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
		3102	bool matchUnaryPredicate(SDValue Op,
		3103	std::function<bool(ConstantSDNode *)> Match,
		3104	bool AllowUndefs = false);
		3105
		3106	/// Attempt to match a binary predicate against a pair of scalar/splat
		3107	/// constants or every element of a pair of constant BUILD_VECTORs.
		3108	/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
		3109	/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
		3110	bool matchBinaryPredicate(
		3111	SDValue LHS, SDValue RHS,
		3112	std::function<bool(ConstantSDNode , ConstantSDNode )> Match,
		3113	bool AllowUndefs = false, bool AllowTypeMismatch = false);
		3114
		3115	/// Returns true if the specified value is the overflow result from one
		3116	/// of the overflow intrinsic nodes.
		3117	inline bool isOverflowIntrOpRes(SDValue Op) {
		3118	unsigned Opc = Op.getOpcode();
		3119	return (Op.getResNo() == 1 &&
		3120	(Opc == ISD::SADDO \|\| Opc == ISD::UADDO \|\| Opc == ISD::SSUBO \|\|
		3121	Opc == ISD::USUBO \|\| Opc == ISD::SMULO \|\| Opc == ISD::UMULO));
		3122	}
		3123
		3124	} // end namespace ISD
		3125
		3126	} // end namespace llvm
		3127
		3128	#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H

Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

QNX 8.QNX8 LLVM/Clang compiler suite//llvm-build/x86_64/include/llvm/CodeGen/SelectionDAGNodes.h – Rev 14